// Wave 4: Oracle's gaps — real LLM answer, multi-persona, perf timings, visual proof. import { chromium } from "playwright"; import { writeFileSync, mkdirSync } from "fs"; const URL = "https://canvas.flow-master.ai/"; const OUT = "/tmp/canvas-evidence/wave4"; mkdirSync(OUT, { recursive: true }); const r = []; const pass = (n, d = "") => { r.push({ n, ok: true, d }); console.log(`PASS ${n}${d ? " " + d : ""}`); }; const fail = (n, d = "") => { r.push({ n, ok: false, d }); console.log(`FAIL ${n}${d ? " " + d : ""}`); }; const b = await chromium.launch({ headless: true }); async function login(personaName) { const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); const p = await ctx.newPage(); const msgs = []; const calls = []; p.on("console", m => msgs.push({ type: m.type(), text: m.text().slice(0, 240) })); p.on("response", res => { if (res.url().includes("/api/")) calls.push({ status: res.status(), method: res.request().method(), url: res.url().replace(/https?:\/\/[^/]+/, "") }); }); await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now()))); const t0 = Date.now(); await p.goto(URL, { waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".persona-chip").filter({ hasText: new RegExp(personaName) }).click(); await p.waitForSelector(".topbar", { timeout: 30000 }); return { ctx, p, msgs, calls, loginMs: Date.now() - t0 }; } // ============= GAP 1: Real LLM-backed answer on canvas ============= console.log("\n=== Gap 1: Real LLM-backed answer ==="); { const { p, msgs, calls } = await login("Mariana"); await p.waitForTimeout(2000); await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); await p.waitForSelector(".agent-composer textarea", { timeout: 15000 }); await p.waitForTimeout(2000); const t0 = Date.now(); await p.locator(".agent-composer textarea").first().fill("In one sentence, what does FlowMaster do?"); await p.locator(".agent-composer textarea").first().press("Enter"); await p.waitForTimeout(15000); const elapsed = Date.now() - t0; const replies = await p.locator(".agent-turn-body").allTextContents(); const last = replies.slice(-1)[0] || ""; console.log(` elapsed: ${elapsed}ms`); console.log(` last reply: ${last.slice(0, 300)}`); const fallback = /trouble|temporarily|fallback|can do for you immediately/i.test(last); const gatewayOk = msgs.find(m => /\[llm\] (primary|fallback-dev) ok in \d+ms · provider=/.test(m.text)); if (gatewayOk) console.log(` telemetry: ${gatewayOk.text.slice(0, 200)}`); if (gatewayOk && !fallback && last.length > 30) pass("gap1_real_llm_answer", `${gatewayOk.text.match(/provider=\w+/)?.[0]} · ${last.length}ch`); else fail("gap1_real_llm_answer", `fallback=${fallback}, gateway_ok=${!!gatewayOk}, len=${last.length}`); await p.screenshot({ path: `${OUT}/g1-llm-answer.png`, fullPage: false }); // Oracle wave-4 action #5: regression check — if both primary and // fallback gateways fail, the assistant must NOT produce a real reply. // Equivalent: a real LLM reply implies at least one gateway succeeded. // We assert the inverse by inspecting telemetry: a successful 'gap1' // run produces exactly one "ok" telemetry line; the cascade-exhausted // failure mode emits "all gateways exhausted". If we ever see both // primary AND fallback-dev fail (cascade exhausted) AND a real reply, // the test fails because the contract is broken. const cascadeExhausted = msgs.some(m => /\[llm\] all gateways exhausted/.test(m.text)); if (!cascadeExhausted) pass("gap1_cascade_did_not_exhaust"); else fail("gap1_cascade_did_not_exhaust", "both primary and fallback-dev failed"); await p.context().close(); } // ============= GAP 2: Multi-persona coverage ============= console.log("\n=== Gap 2: Multi-persona coverage (Aisha + Rohan) ==="); for (const persona of ["Aisha", "Rohan"]) { console.log(` ${persona}:`); const { p, calls } = await login(persona); await p.waitForTimeout(2500); const tabs = await p.locator(".tab").allTextContents(); const has6 = ["Mission", "Approvals", "Runs", "Studio", "Chat", "Assistant"].every(t => tabs.some(x => x.includes(t))); if (has6) pass(`gap2_${persona.toLowerCase()}_topbar_6_tabs`); else fail(`gap2_${persona.toLowerCase()}_topbar_6_tabs`, `tabs=${tabs.join("|")}`); await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); await p.waitForTimeout(2000); await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft")); await p.reload({ waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); await p.waitForSelector("textarea", { timeout: 10000 }); await p.locator("textarea").first().fill(`${persona} test draft.`); await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click(); await p.waitForTimeout(7000); const fivexx = calls.filter(c => c.status >= 500 && !c.url.includes("/llm/")).length; const post201 = calls.some(c => c.method === "POST" && c.url === "/api/ea2/flow" && c.status === 201); if (post201 && fivexx === 0) pass(`gap2_${persona.toLowerCase()}_studio_no_5xx`); else fail(`gap2_${persona.toLowerCase()}_studio_no_5xx`, `post201=${post201}, fivexx=${fivexx}`); await p.locator(".tab").filter({ hasText: /Chat/i }).first().click(); await p.waitForTimeout(2500); const chatVisible = await p.locator(".chat-surface, .chat-list, .threadlist, .chat-empty, .chat-err").count(); if (chatVisible > 0) pass(`gap2_${persona.toLowerCase()}_chat_surface_visible`); else fail(`gap2_${persona.toLowerCase()}_chat_surface_visible`); await p.screenshot({ path: `${OUT}/g2-${persona.toLowerCase()}-topbar.png`, clip: { x: 0, y: 0, width: 1440, height: 64 } }); await p.context().close(); } // ============= GAP 3: Perf timings (Mission + EA2 sync + Assistant) ============= console.log("\n=== Gap 3: Performance timings ==="); { const { p, calls, loginMs } = await login("Mariana"); const t0 = Date.now(); await p.waitForSelector(".mc, .mc-strip, .mc-hero", { timeout: 15000 }); await p.waitForTimeout(3000); const missionInitialMs = Date.now() - t0; const ea2Calls = calls.filter(c => c.url.startsWith("/api/ea2/")); console.log(` login: ${loginMs}ms`); console.log(` Mission first paint: ${missionInitialMs}ms`); console.log(` EA2 calls during initial sync: ${ea2Calls.length} (statuses: ${[...new Set(ea2Calls.map(c => c.status))].join(",")})`); if (missionInitialMs < 12000) pass("gap3_mission_initial_paint_under_12s", `${missionInitialMs}ms`); else fail("gap3_mission_initial_paint_under_12s", `${missionInitialMs}ms`); await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); await p.waitForSelector(".agent-composer textarea", { timeout: 15000 }); await p.waitForTimeout(1500); const t1 = Date.now(); await p.locator(".agent-composer textarea").first().fill("list processes"); await p.locator(".agent-composer textarea").first().press("Enter"); await p.waitForFunction(() => document.querySelectorAll(".agent-turn-body").length >= 3, { timeout: 20000 }); const assistantListMs = Date.now() - t1; console.log(` Assistant 'list processes' end-to-end: ${assistantListMs}ms`); if (assistantListMs < 10000) pass("gap3_assistant_list_under_10s", `${assistantListMs}ms`); else fail("gap3_assistant_list_under_10s", `${assistantListMs}ms`); writeFileSync(`${OUT}/perf-timings.json`, JSON.stringify({ loginMs, missionInitialMs, assistantListMs, ea2Calls: ea2Calls.length }, null, 2)); await p.context().close(); } // ============= GAP 4: Topbar visual + responsive + focus ============= console.log("\n=== Gap 4: Topbar visual + responsive + focus ==="); for (const width of [1920, 1440, 1024, 768]) { const ctx2 = await b.newContext({ viewport: { width, height: 900 } }); const p = await ctx2.newPage(); await ctx2.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now()))); await p.goto(URL, { waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); await p.waitForSelector(".topbar", { timeout: 30000 }); await p.waitForTimeout(1500); const topbarBox = await p.locator(".topbar").boundingBox(); const tabsOverflow = await p.evaluate(() => { const tb = document.querySelector(".topbar"); return tb ? { scrollWidth: tb.scrollWidth, clientWidth: tb.clientWidth, overflowed: tb.scrollWidth > tb.clientWidth } : null; }); console.log(` ${width}px: topbar h=${topbarBox?.height}px, overflowed=${tabsOverflow?.overflowed}`); if (tabsOverflow && !tabsOverflow.overflowed) pass(`gap4_topbar_no_overflow_at_${width}`); else fail(`gap4_topbar_no_overflow_at_${width}`, JSON.stringify(tabsOverflow)); await p.screenshot({ path: `${OUT}/g4-topbar-${width}.png`, clip: { x: 0, y: 0, width, height: 80 } }); await ctx2.close(); } { const { p } = await login("Mariana"); await p.waitForTimeout(2000); await p.keyboard.press("Tab"); await p.keyboard.press("Tab"); await p.keyboard.press("Tab"); await p.waitForTimeout(300); const focused = await p.evaluate(() => document.activeElement?.tagName); console.log(` Tab focus reached: ${focused}`); if (focused === "BUTTON" || focused === "A") pass("gap4_topbar_keyboard_reachable", focused); else fail("gap4_topbar_keyboard_reachable", focused || "null"); await p.screenshot({ path: `${OUT}/g4-keyboard-focus.png`, clip: { x: 0, y: 0, width: 1440, height: 80 } }); await p.context().close(); } await b.close(); const passed = r.filter(x => x.ok).length; console.log(`\n=== ${passed}/${r.length} wave-4 gap checks passed ===`); writeFileSync(`${OUT}/results.json`, JSON.stringify(r, null, 2)); process.exit(passed === r.length ? 0 : 1);