// Full end-to-end live verification of the user's specific bugs. import { chromium } from "playwright"; import { writeFileSync, mkdirSync } from "node:fs"; const URL = "https://canvas.flow-master.ai/"; const OUT = "/tmp/canvas-evidence/oracle-e2e"; mkdirSync(OUT, { recursive: true }); const results = []; function pass(name, detail = "") { results.push({ name, ok: true, detail }); console.log(`✅ PASS ${name}${detail ? " — " + detail : ""}`); } function fail(name, detail = "") { results.push({ name, ok: false, detail }); console.log(`❌ FAIL ${name}${detail ? " — " + detail : ""}`); } const calls = []; const consoleErrs = []; const b = await chromium.launch({ headless: true }); const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); const p = await ctx.newPage(); p.on("response", (r) => { const u = r.url(); if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") }); }); p.on("console", (m) => { if (m.type() === "error") consoleErrs.push(m.text().slice(0, 200)); }); await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now()))); // 1. Login → mission await p.goto(URL, { waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); await p.waitForSelector(".topbar", { timeout: 30000 }); await p.waitForTimeout(2000); const shell = await p.evaluate(() => document.querySelector("[class*='shell-']")?.className); if (shell?.includes("mission")) pass("login_lands_on_mission", shell); else fail("login_lands_on_mission", shell); await p.screenshot({ path: `${OUT}/01-mission.png` }); // 2. Topbar count const tabCount = await p.locator(".tab").count(); const tabs = await p.locator(".tab").allTextContents(); if (tabCount === 6) pass("topbar_has_6_tabs", tabs.map(t => t.trim()).join(", ")); else fail("topbar_has_6_tabs", `count=${tabCount}, tabs=${tabs.join(",")}`); // 3. User menu opens with 5 items await p.locator(".user-avatar").click(); await p.waitForTimeout(700); const menuItems = await p.locator(".user-menu-item").allTextContents(); if (menuItems.length >= 4) pass("user_menu_has_items", menuItems.map(m => m.trim().slice(0, 30)).join(" | ")); else fail("user_menu_has_items", JSON.stringify(menuItems)); await p.screenshot({ path: `${OUT}/02-user-menu.png` }); await p.keyboard.press("Escape"); await p.waitForTimeout(500); // 4. Open Studio → fill → start drafting → confirm structure await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); await p.waitForTimeout(2500); // Clear any stale draft from localStorage await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft")); await p.reload({ waitUntil: "domcontentloaded" }); await p.waitForTimeout(2500); await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); await p.waitForSelector("textarea", { timeout: 10000 }).catch(() => {}); await p.waitForTimeout(1500); const ta = p.locator("textarea").first(); if (await ta.count()) { await ta.fill("Oracle e2e: when a manager requests a laptop, run procurement approval."); const draftBtn = p.locator("button.btn-primary").filter({ hasText: /draft/i }).first(); if (await draftBtn.count()) { await draftBtn.click(); await p.waitForTimeout(8000); const studioCallsBefore = calls.length; const confirmBtn = p.locator("button.btn-primary").filter({ hasText: /Confirm structure/i }).first(); if (await confirmBtn.count()) { await confirmBtn.click(); await p.waitForTimeout(12000); const studioFails = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 400); const studioOks = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 200 && c.status < 300); if (studioFails.length === 0 && studioOks.length > 0) pass("studio_confirm_structure_no_502", `${studioOks.length} OK applies`); else fail("studio_confirm_structure_no_502", `${studioFails.length} fails, ${studioOks.length} OK`); await p.screenshot({ path: `${OUT}/03-studio-after-confirm.png` }); } else fail("studio_confirm_structure_button_visible"); } else fail("studio_start_drafting_button"); } else fail("studio_intake_visible"); // 5. Assistant list processes await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); await p.waitForTimeout(2500); const agentInput = p.locator(".agent-input, input[placeholder*='Ask']").first(); await agentInput.fill("list processes"); await agentInput.press("Enter"); await p.waitForTimeout(6000); const replies = await p.locator(".agent-msg-text, .agent-msg-body").allTextContents(); const lastReply = replies.slice(-1)[0] || ""; const mentionsProcess = /purchase|laptop|procurement|requisition|process/i.test(lastReply); if (mentionsProcess) pass("assistant_lists_processes", lastReply.slice(0, 100)); else fail("assistant_lists_processes", lastReply.slice(0, 150)); await p.screenshot({ path: `${OUT}/04-assistant.png` }); // 6. Chat opens await p.locator(".tab").filter({ hasText: /Chat/i }).first().click(); await p.waitForTimeout(4000); const chatErr = await p.locator(".chat-err").count(); const chatEmpty = await p.locator(".chat-empty").count(); const chatShell = await p.locator(".chat-shell, .chat-scene, .scene").first().innerText().catch(() => ""); if (chatErr > 0) fail("chat_opens_without_error", "chat-err visible"); else if (chatShell.includes("Conversations") || chatShell.includes("Talk to your team") || chatShell.includes("No conversations")) pass("chat_opens_without_error", "chat surface visible"); else fail("chat_opens_without_error", chatShell.slice(0, 200)); await p.screenshot({ path: `${OUT}/05-chat.png` }); // 7. No snapshot mode anywhere const hasSnapshot = await p.evaluate(() => document.body.innerText.toLowerCase().includes("snapshot")); if (!hasSnapshot) pass("no_snapshot_text_anywhere"); else fail("no_snapshot_text_anywhere", "found 'snapshot' in body"); // 8. LLM gateway: ask the assistant something non-tool await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); await p.waitForTimeout(1500); const llmInput = p.locator(".agent-input, input[placeholder*='Ask']").first(); await llmInput.fill("What is the difference between a flow and a view in EA2?"); await llmInput.press("Enter"); await p.waitForTimeout(8000); const llmReplies = await p.locator(".agent-msg-text, .agent-msg-body").allTextContents(); const llmLast = llmReplies.slice(-1)[0] || ""; const llmCallsList = calls.filter(c => c.url.includes("/api/v1/llm/generate")); if (llmCallsList.length > 0) { const status = llmCallsList.slice(-1)[0].status; if (status === 200) pass("llm_gateway_responds", `200 with ${llmLast.length} char reply`); else if (status === 503 || status === 500) { if (llmLast.toLowerCase().includes("unavailable") || llmLast.toLowerCase().includes("not configured")) pass("llm_gateway_shows_clear_error", llmLast.slice(0, 100)); else fail("llm_gateway_shows_clear_error", `status=${status} reply=${llmLast.slice(0, 100)}`); } else fail("llm_gateway_responds", `unexpected status ${status}`); } else fail("llm_gateway_responds", "no calls to /api/v1/llm/generate"); await p.screenshot({ path: `${OUT}/06-llm-reply.png` }); // 9. Network health summary const byStatus = {}; for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1; const fivexx = calls.filter(c => c.status >= 500).length; const fast502 = calls.filter(c => c.status === 502).length; if (fast502 === 0) pass("no_502_anywhere", `5xx total=${fivexx}`); else fail("no_502_anywhere", `${fast502} 502s seen`); await b.close(); // Write writeFileSync(`${OUT}/results.json`, JSON.stringify(results, null, 2)); writeFileSync(`${OUT}/network.json`, JSON.stringify(calls, null, 2)); writeFileSync(`${OUT}/console-errors.json`, JSON.stringify(consoleErrs, null, 2)); const passed = results.filter(r => r.ok).length; console.log(`\n=== ${passed}/${results.length} checks passed ===`); console.log(`api calls: ${calls.length} by status: ${JSON.stringify(byStatus)}`); process.exit(passed === results.length ? 0 : 1);