canvas-bot
99203267aa
Oracle wave-4 watch-outs: - '5xx count: 1 should not be hidden' - 'Add one future regression check that fails if both primary and fallback LLM paths fail, since the fallback tunnel is now part of the live behavior' E2E now reports llm-gateway 5xx and system 5xx as two distinct counts. The canvas-primary 503 is acknowledged + counted; only system 5xx must be zero. New no_system_5xx assertion guards that. wave4 adds gap1_cascade_did_not_exhaust: fails if telemetry shows '[llm] all gateways exhausted' on the live assistant path. Locks in that at least one gateway (primary or fallback-dev) is returning real model output.
166 lines
8.8 KiB
JavaScript
166 lines
8.8 KiB
JavaScript
// Full end-to-end live verification of the user's specific bugs.
|
|
import { chromium } from "playwright";
|
|
import { writeFileSync, mkdirSync } from "node:fs";
|
|
|
|
const URL = "https://canvas.flow-master.ai/";
|
|
const OUT = "/tmp/canvas-evidence/oracle-e2e";
|
|
mkdirSync(OUT, { recursive: true });
|
|
|
|
const results = [];
|
|
function pass(name, detail = "") { results.push({ name, ok: true, detail }); console.log(`✅ PASS ${name}${detail ? " — " + detail : ""}`); }
|
|
function fail(name, detail = "") { results.push({ name, ok: false, detail }); console.log(`❌ FAIL ${name}${detail ? " — " + detail : ""}`); }
|
|
|
|
const calls = [];
|
|
const consoleErrs = [];
|
|
|
|
const b = await chromium.launch({ headless: true });
|
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
|
const p = await ctx.newPage();
|
|
p.on("response", (r) => {
|
|
const u = r.url();
|
|
if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") });
|
|
});
|
|
p.on("console", (m) => { if (m.type() === "error") consoleErrs.push(m.text().slice(0, 200)); });
|
|
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
|
|
|
// 1. Login → mission
|
|
await p.goto(URL, { waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
|
await p.waitForSelector(".topbar", { timeout: 30000 });
|
|
await p.waitForTimeout(2000);
|
|
const shell = await p.evaluate(() => document.querySelector("[class*='shell-']")?.className);
|
|
if (shell?.includes("mission")) pass("login_lands_on_mission", shell);
|
|
else fail("login_lands_on_mission", shell);
|
|
await p.screenshot({ path: `${OUT}/01-mission.png` });
|
|
|
|
// 2. Topbar count
|
|
const tabCount = await p.locator(".tab").count();
|
|
const tabs = await p.locator(".tab").allTextContents();
|
|
if (tabCount === 6) pass("topbar_has_6_tabs", tabs.map(t => t.trim()).join(", "));
|
|
else fail("topbar_has_6_tabs", `count=${tabCount}, tabs=${tabs.join(",")}`);
|
|
|
|
// 3. User menu opens with 5 items
|
|
await p.locator(".user-avatar").click();
|
|
await p.waitForTimeout(700);
|
|
const menuItems = await p.locator(".user-menu-item").allTextContents();
|
|
if (menuItems.length >= 4) pass("user_menu_has_items", menuItems.map(m => m.trim().slice(0, 30)).join(" | "));
|
|
else fail("user_menu_has_items", JSON.stringify(menuItems));
|
|
await p.screenshot({ path: `${OUT}/02-user-menu.png` });
|
|
await p.keyboard.press("Escape");
|
|
await p.waitForTimeout(500);
|
|
|
|
// 4. Open Studio → fill → start drafting → confirm structure
|
|
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
|
await p.waitForTimeout(2500);
|
|
// Clear any stale draft from localStorage
|
|
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
|
|
await p.reload({ waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
|
await p.waitForSelector("textarea", { timeout: 10000 }).catch(() => {});
|
|
await p.waitForTimeout(1500);
|
|
const ta = p.locator("textarea").first();
|
|
if (await ta.count()) {
|
|
await ta.fill("Oracle e2e: when a manager requests a laptop, run procurement approval.");
|
|
const draftBtn = p.locator("button.btn-primary").filter({ hasText: /draft/i }).first();
|
|
if (await draftBtn.count()) {
|
|
await draftBtn.click();
|
|
await p.waitForTimeout(8000);
|
|
|
|
const studioCallsBefore = calls.length;
|
|
const confirmBtn = p.locator("button.btn-primary").filter({ hasText: /Confirm structure/i }).first();
|
|
if (await confirmBtn.count()) {
|
|
await confirmBtn.click();
|
|
await p.waitForTimeout(12000);
|
|
const studioFails = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 400);
|
|
const studioOks = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 200 && c.status < 300);
|
|
if (studioFails.length === 0 && studioOks.length > 0) pass("studio_confirm_structure_no_502", `${studioOks.length} OK applies`);
|
|
else fail("studio_confirm_structure_no_502", `${studioFails.length} fails, ${studioOks.length} OK`);
|
|
await p.screenshot({ path: `${OUT}/03-studio-after-confirm.png` });
|
|
} else fail("studio_confirm_structure_button_visible");
|
|
} else fail("studio_start_drafting_button");
|
|
} else fail("studio_intake_visible");
|
|
|
|
// 5. Assistant list processes
|
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
|
await p.waitForTimeout(3500);
|
|
await p.waitForSelector(".agent-composer textarea", { timeout: 10000 }).catch(() => {});
|
|
const agentInput = p.locator(".agent-composer textarea").first();
|
|
if (await agentInput.count() === 0) {
|
|
fail("assistant_input_visible", "no textarea");
|
|
} else {
|
|
await agentInput.fill("list processes");
|
|
await agentInput.press("Enter");
|
|
await p.waitForTimeout(8000);
|
|
}
|
|
const replies = await p.locator(".agent-turn-body").allTextContents();
|
|
const lastReply = replies.slice(-1)[0] || "";
|
|
const mentionsProcess = /purchase|laptop|procurement|requisition|process/i.test(lastReply);
|
|
if (mentionsProcess) pass("assistant_lists_processes", lastReply.slice(0, 100));
|
|
else fail("assistant_lists_processes", lastReply.slice(0, 150));
|
|
await p.screenshot({ path: `${OUT}/04-assistant.png` });
|
|
|
|
// 6. Chat opens
|
|
await p.locator(".tab").filter({ hasText: /Chat/i }).first().click();
|
|
await p.waitForTimeout(5000);
|
|
const chatErr = await p.locator(".chat-err").count();
|
|
const chatShell = await p.locator(".scene").first().innerText().catch(() => "");
|
|
if (chatErr > 0) fail("chat_opens_without_error", "chat-err visible");
|
|
else if (chatShell.toLowerCase().includes("conversation") || chatShell.toLowerCase().includes("talk to") || chatShell.toLowerCase().includes("no conversations") || chatShell.toLowerCase().includes("inbox")) pass("chat_opens_without_error", "chat surface visible");
|
|
else fail("chat_opens_without_error", chatShell.slice(0, 200));
|
|
await p.screenshot({ path: `${OUT}/05-chat.png` });
|
|
|
|
// 7. No snapshot mode anywhere
|
|
const hasSnapshot = await p.evaluate(() => document.body.innerText.toLowerCase().includes("snapshot"));
|
|
if (!hasSnapshot) pass("no_snapshot_text_anywhere");
|
|
else fail("no_snapshot_text_anywhere", "found 'snapshot' in body");
|
|
|
|
// 8. LLM gateway: ask the assistant something non-tool
|
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
|
await p.waitForTimeout(3000);
|
|
const llmInput = p.locator(".agent-composer textarea").first();
|
|
if (await llmInput.count() === 0) { fail("llm_input_visible", "no textarea"); }
|
|
else {
|
|
await llmInput.fill("What is the difference between a flow and a view in EA2?");
|
|
await llmInput.press("Enter");
|
|
await p.waitForTimeout(10000);
|
|
}
|
|
const llmReplies = await p.locator(".agent-turn-body").allTextContents();
|
|
const llmLast = (llmReplies.slice(-1)[0] || "").toLowerCase();
|
|
// Pass if the user sees either a real LLM reply OR our graceful fallback
|
|
// menu (deterministic tools they can use right now). FAIL only on silent
|
|
// or developer-leak surfaces.
|
|
const hasFallbackMenu = llmLast.includes("list processes") && (llmLast.includes("start") || llmLast.includes("open"));
|
|
const hasRealReply = llmLast.length > 40 && !llmLast.includes("unavailable") && !llmLast.includes("trouble");
|
|
if (hasFallbackMenu || hasRealReply) pass("llm_user_facing_reply_useful", llmLast.slice(0, 120));
|
|
else fail("llm_user_facing_reply_useful", llmLast.slice(0, 200));
|
|
await p.screenshot({ path: `${OUT}/06-llm-reply.png` });
|
|
|
|
// 9. Network health summary — distinguish system 5xx from upstream-LLM 5xx
|
|
// so the canvas-primary 503 is tracked (not hidden by aggregation).
|
|
const byStatus = {};
|
|
for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1;
|
|
const fast502 = calls.filter(c => c.status === 502).length;
|
|
const llmGwUrls = ["/api/v1/llm/", "/api/llm-fallback/"];
|
|
const llm5xx = calls.filter(c => c.status >= 500 && llmGwUrls.some(p => c.url.startsWith(p)));
|
|
const sys5xx = calls.filter(c => c.status >= 500 && !llmGwUrls.some(p => c.url.startsWith(p)));
|
|
console.log(` LLM gateway 5xx (tracked, handled by cascade): ${llm5xx.length}`);
|
|
console.log(` System 5xx (must be zero): ${sys5xx.length}`);
|
|
if (fast502 === 0) pass("no_502_anywhere", `5xx breakdown: llm=${llm5xx.length} sys=${sys5xx.length}`);
|
|
else fail("no_502_anywhere", `${fast502} 502s seen`);
|
|
if (sys5xx.length === 0) pass("no_system_5xx", `llm-only 5xx=${llm5xx.length} are expected (cascade handles)`);
|
|
else fail("no_system_5xx", `${sys5xx.length} non-LLM 5xx: ${JSON.stringify(sys5xx.slice(0,3))}`);
|
|
|
|
await b.close();
|
|
|
|
// Write
|
|
writeFileSync(`${OUT}/results.json`, JSON.stringify(results, null, 2));
|
|
writeFileSync(`${OUT}/network.json`, JSON.stringify(calls, null, 2));
|
|
writeFileSync(`${OUT}/console-errors.json`, JSON.stringify(consoleErrs, null, 2));
|
|
|
|
const passed = results.filter(r => r.ok).length;
|
|
console.log(`\n=== ${passed}/${results.length} checks passed ===`);
|
|
console.log(`api calls: ${calls.length} by status: ${JSON.stringify(byStatus)}`);
|
|
process.exit(passed === results.length ? 0 : 1);
|