canvas-frontend/qa/_wave3_verify.mjs

// Verify the four wave-3 watch-out fixes on live.
import { chromium } from "playwright";

const URL = "https://canvas.flow-master.ai/";

const results = [];
function pass(n, d = "") { results.push({ n, ok: true, d }); console.log(`✅ PASS  ${n}${d ? "  — " + d : ""}`); }
function fail(n, d = "") { results.push({ n, ok: false, d }); console.log(`❌ FAIL  ${n}${d ? "  — " + d : ""}`); }

const b = await chromium.launch({ headless: true });
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
const p = await ctx.newPage();

const calls = [];
const consoleMsgs = [];
p.on("response", (r) => {
  const u = r.url();
  if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") });
});
p.on("console", (m) => consoleMsgs.push({ type: m.type(), text: m.text() }));

// Skip onboarding tour; KEEP mission primer fresh
await ctx.addInitScript(() => {
  localStorage.setItem("fm.canvas.tour.seen", String(Date.now()));
});

console.log("\n=== watch-out 1: Wizard config soft-warn telemetry ===");
await p.goto(URL, { waitUntil: "domcontentloaded" });
await p.waitForTimeout(2500);
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
await p.waitForSelector(".topbar", { timeout: 30000 });
await p.waitForTimeout(2000);

// Open Studio + create draft + check for the soft-warn toast (if PUT fails) OR
// successful flow without errors.
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
await p.waitForTimeout(2000);
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
await p.reload({ waitUntil: "domcontentloaded" });
await p.waitForTimeout(2500);
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
await p.waitForSelector("textarea", { timeout: 10000 });
await p.locator("textarea").first().fill("Wave3 verification: laptop procurement flow.");
const draftStart = calls.length;
await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click();
await p.waitForTimeout(6000);
const newCalls = calls.slice(draftStart);
const flowPosts = newCalls.filter(c => c.method === "POST" && c.url === "/api/ea2/flow");
const flowPuts = newCalls.filter(c => c.method === "PUT" && c.url.startsWith("/api/ea2/flow/"));
console.log(`  POST /api/ea2/flow:`, flowPosts.map(c => c.status));
console.log(`  PUT /api/ea2/flow/<key>:`, flowPuts.map(c => c.status));
const draftOk = flowPosts.some(c => c.status === 201);
if (draftOk) pass("wave3_studio_draft_lands_via_split_post", `POST 201 + PUT ${flowPuts[0]?.status ?? "missing"}`);
else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flowPosts)}`);

console.log("\n=== watch-out 2: LLM telemetry shows in console ===");
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
await p.waitForTimeout(2000);
const txt = p.locator(".agent-composer textarea").first();
// Genuinely non-tool prompt so routeAgentInput falls through to llmFallback.
await txt.fill("tell me a haiku about flowcharts");
await txt.press("Enter");
await p.waitForTimeout(14000);
const llmConsoleEntries = consoleMsgs.filter(m =>
  m.type === "warning" || m.type === "info"
).filter(m => m.text.includes("[llm]"));
console.log(`  console [llm] entries:`, llmConsoleEntries.length);
for (const c of llmConsoleEntries.slice(0, 3)) console.log(`    [${c.type}] ${c.text.slice(0, 180)}`);
if (llmConsoleEntries.length > 0) pass("wave3_llm_telemetry_logged", `${llmConsoleEntries.length} [llm] entries`);
else fail("wave3_llm_telemetry_logged", "no [llm] console entries found");

console.log("\n=== watch-out 3: Catalog dedupe (no Laptop Procurement x3) ===");
await txt.fill("list processes");
await txt.press("Enter");
await p.waitForTimeout(8000);
const replies = await p.locator(".agent-turn-body").allTextContents();
const lastList = replies.slice(-1)[0] || "";
const laptopMatches = (lastList.match(/laptop\s+procurement/gi) || []).length;
console.log(`  "Laptop Procurement" occurrences:`, laptopMatches);
console.log(`  full reply: ${lastList.slice(0, 400)}`);
if (laptopMatches <= 1) pass("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`);
else fail("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`);

console.log("\n=== watch-out 4: Mission primer + no 'snapshot' wording ===");
// Clear primer-dismissed flag in case it's set
await p.evaluate(() => localStorage.removeItem("fm.canvas.mission.primer.dismissed"));
await p.locator(".tab").filter({ hasText: /Mission/i }).first().click();
await p.waitForTimeout(3500);
const primerVisible = await p.locator(".mc-primer").count();
const sceneText = (await p.evaluate(() => document.querySelector(".scene")?.innerText || "")).toLowerCase();
const hasSnapshotInText = sceneText.includes("snapshot");
console.log(`  .mc-primer count:`, primerVisible);
console.log(`  'snapshot' anywhere in Mission text:`, hasSnapshotInText);
if (primerVisible >= 1) pass("wave3_mission_primer_visible");
else fail("wave3_mission_primer_visible");
if (!hasSnapshotInText) pass("wave3_mission_no_snapshot_text");
else fail("wave3_mission_no_snapshot_text");

await p.screenshot({ path: "/tmp/canvas-evidence/wave3-mission.png", fullPage: false });
await b.close();

const passed = results.filter(r => r.ok).length;
console.log(`\n=== ${passed}/${results.length} watch-out checks passed ===`);
console.log(`total api calls: ${calls.length}`);
const byStatus = {};
for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1;
console.log(`by status: ${JSON.stringify(byStatus)}`);
const fivexx = calls.filter(c => c.status >= 500).length;
console.log(`5xx count: ${fivexx}`);
process.exit(passed === results.length ? 0 : 1);