// Verify the four wave-3 watch-out fixes on live. import { chromium } from "playwright"; const URL = "https://canvas.flow-master.ai/"; const results = []; function pass(n, d = "") { results.push({ n, ok: true, d }); console.log(`✅ PASS ${n}${d ? " — " + d : ""}`); } function fail(n, d = "") { results.push({ n, ok: false, d }); console.log(`❌ FAIL ${n}${d ? " — " + d : ""}`); } const b = await chromium.launch({ headless: true }); const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); const p = await ctx.newPage(); const calls = []; const consoleMsgs = []; p.on("response", (r) => { const u = r.url(); if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") }); }); p.on("console", (m) => consoleMsgs.push({ type: m.type(), text: m.text() })); // Skip onboarding tour; KEEP mission primer fresh await ctx.addInitScript(() => { localStorage.setItem("fm.canvas.tour.seen", String(Date.now())); }); console.log("\n=== watch-out 1: Wizard config soft-warn telemetry ==="); await p.goto(URL, { waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); await p.waitForSelector(".topbar", { timeout: 30000 }); await p.waitForTimeout(2000); // Open Studio + create draft + check for the soft-warn toast (if PUT fails) OR // successful flow without errors. await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); await p.waitForTimeout(2000); await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft")); await p.reload({ waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); await p.waitForSelector("textarea", { timeout: 10000 }); await p.locator("textarea").first().fill("Wave3 verification: laptop procurement flow."); const draftStart = calls.length; await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click(); await p.waitForTimeout(6000); const newCalls = calls.slice(draftStart); const flowPosts = newCalls.filter(c => c.method === "POST" && c.url === "/api/ea2/flow"); const flowPuts = newCalls.filter(c => c.method === "PUT" && c.url.startsWith("/api/ea2/flow/")); console.log(` POST /api/ea2/flow:`, flowPosts.map(c => c.status)); console.log(` PUT /api/ea2/flow/:`, flowPuts.map(c => c.status)); const draftOk = flowPosts.some(c => c.status === 201); if (draftOk) pass("wave3_studio_draft_lands_via_split_post", `POST 201 + PUT ${flowPuts[0]?.status ?? "missing"}`); else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flowPosts)}`); console.log("\n=== watch-out 2: LLM telemetry shows in console ==="); await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); await p.waitForSelector(".agent-composer textarea", { timeout: 15000 }); await p.waitForTimeout(2000); const txt = p.locator(".agent-composer textarea").first(); // Genuinely non-tool prompt so routeAgentInput falls through to llmFallback. await txt.fill("tell me a haiku about flowcharts"); await txt.press("Enter"); await p.waitForTimeout(14000); const llmConsoleEntries = consoleMsgs.filter(m => m.type === "warning" || m.type === "info" ).filter(m => m.text.includes("[llm]")); console.log(` console [llm] entries:`, llmConsoleEntries.length); for (const c of llmConsoleEntries.slice(0, 3)) console.log(` [${c.type}] ${c.text.slice(0, 180)}`); if (llmConsoleEntries.length > 0) pass("wave3_llm_telemetry_logged", `${llmConsoleEntries.length} [llm] entries`); else fail("wave3_llm_telemetry_logged", "no [llm] console entries found"); console.log("\n=== watch-out 3: Catalog dedupe (no Laptop Procurement x3) ==="); await txt.fill("list processes"); await txt.press("Enter"); await p.waitForTimeout(8000); const replies = await p.locator(".agent-turn-body").allTextContents(); const lastList = replies.slice(-1)[0] || ""; const laptopMatches = (lastList.match(/laptop\s+procurement/gi) || []).length; console.log(` "Laptop Procurement" occurrences:`, laptopMatches); console.log(` full reply: ${lastList.slice(0, 400)}`); if (laptopMatches <= 1) pass("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`); else fail("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`); console.log("\n=== watch-out 4: Mission primer + no 'snapshot' wording ==="); // Clear primer-dismissed flag in case it's set await p.evaluate(() => localStorage.removeItem("fm.canvas.mission.primer.dismissed")); await p.locator(".tab").filter({ hasText: /Mission/i }).first().click(); await p.waitForTimeout(3500); const primerVisible = await p.locator(".mc-primer").count(); const sceneText = (await p.evaluate(() => document.querySelector(".scene")?.innerText || "")).toLowerCase(); const hasSnapshotInText = sceneText.includes("snapshot"); console.log(` .mc-primer count:`, primerVisible); console.log(` 'snapshot' anywhere in Mission text:`, hasSnapshotInText); if (primerVisible >= 1) pass("wave3_mission_primer_visible"); else fail("wave3_mission_primer_visible"); if (!hasSnapshotInText) pass("wave3_mission_no_snapshot_text"); else fail("wave3_mission_no_snapshot_text"); await p.screenshot({ path: "/tmp/canvas-evidence/wave3-mission.png", fullPage: false }); await b.close(); const passed = results.filter(r => r.ok).length; console.log(`\n=== ${passed}/${results.length} watch-out checks passed ===`); console.log(`total api calls: ${calls.length}`); const byStatus = {}; for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1; console.log(`by status: ${JSON.stringify(byStatus)}`); const fivexx = calls.filter(c => c.status >= 500).length; console.log(`5xx count: ${fivexx}`); process.exit(passed === results.length ? 0 : 1);