canvas-bot
9cbee11756
Three Oracle-flagged gaps:
1. Studio createDraft 500 (was masked by retries). Empirical curl
reproducer proved POST /api/ea2/flow returns 201 with the minimal
shape but intermittently 500s when config.wizard.* is in the
initial body. Split: POST creates the draft minimally, then a
follow-up PUT attaches the wizard config best-effort. Draft
creation now lands reliably.
2. LLM 'temporarily unavailable' was a dead end for the user. Now
the assistant returns ok:true with a concrete menu of tools the
user CAN use right now ('list processes', 'start <name>',
'open <hub>', 'tell <name> that ...', 'remember ...', 'recall ...').
The reply renders as a normal assistant message, not a red error.
3. Topbar UI/UX Pro Max pass:
- Added aria-label to the ⌘K and avatar buttons (icon-only a11y)
- Removed the duplicate topbar-mid chips on Mission (Mission scene
already shows family/defName/version inline). Cleaner topbar.
- Added focus-visible outlines (2px amber, 2px offset) on all
topbar action buttons + user-menu items
- Hover/active scale on the avatar (1.04 / 0.97) with 160ms ease
- Tabular numerals for the ⌘K kbd and notification badge
- prefers-reduced-motion respected
30/30 vitest pass. tsc + vite build green.
161 lines
8.2 KiB
JavaScript
161 lines
8.2 KiB
JavaScript
// Full end-to-end live verification of the user's specific bugs.
|
|
import { chromium } from "playwright";
|
|
import { writeFileSync, mkdirSync } from "node:fs";
|
|
|
|
const URL = "https://canvas.flow-master.ai/";
|
|
const OUT = "/tmp/canvas-evidence/oracle-e2e";
|
|
mkdirSync(OUT, { recursive: true });
|
|
|
|
const results = [];
|
|
function pass(name, detail = "") { results.push({ name, ok: true, detail }); console.log(`✅ PASS ${name}${detail ? " — " + detail : ""}`); }
|
|
function fail(name, detail = "") { results.push({ name, ok: false, detail }); console.log(`❌ FAIL ${name}${detail ? " — " + detail : ""}`); }
|
|
|
|
const calls = [];
|
|
const consoleErrs = [];
|
|
|
|
const b = await chromium.launch({ headless: true });
|
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
|
const p = await ctx.newPage();
|
|
p.on("response", (r) => {
|
|
const u = r.url();
|
|
if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") });
|
|
});
|
|
p.on("console", (m) => { if (m.type() === "error") consoleErrs.push(m.text().slice(0, 200)); });
|
|
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
|
|
|
// 1. Login → mission
|
|
await p.goto(URL, { waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
|
await p.waitForSelector(".topbar", { timeout: 30000 });
|
|
await p.waitForTimeout(2000);
|
|
const shell = await p.evaluate(() => document.querySelector("[class*='shell-']")?.className);
|
|
if (shell?.includes("mission")) pass("login_lands_on_mission", shell);
|
|
else fail("login_lands_on_mission", shell);
|
|
await p.screenshot({ path: `${OUT}/01-mission.png` });
|
|
|
|
// 2. Topbar count
|
|
const tabCount = await p.locator(".tab").count();
|
|
const tabs = await p.locator(".tab").allTextContents();
|
|
if (tabCount === 6) pass("topbar_has_6_tabs", tabs.map(t => t.trim()).join(", "));
|
|
else fail("topbar_has_6_tabs", `count=${tabCount}, tabs=${tabs.join(",")}`);
|
|
|
|
// 3. User menu opens with 5 items
|
|
await p.locator(".user-avatar").click();
|
|
await p.waitForTimeout(700);
|
|
const menuItems = await p.locator(".user-menu-item").allTextContents();
|
|
if (menuItems.length >= 4) pass("user_menu_has_items", menuItems.map(m => m.trim().slice(0, 30)).join(" | "));
|
|
else fail("user_menu_has_items", JSON.stringify(menuItems));
|
|
await p.screenshot({ path: `${OUT}/02-user-menu.png` });
|
|
await p.keyboard.press("Escape");
|
|
await p.waitForTimeout(500);
|
|
|
|
// 4. Open Studio → fill → start drafting → confirm structure
|
|
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
|
await p.waitForTimeout(2500);
|
|
// Clear any stale draft from localStorage
|
|
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
|
|
await p.reload({ waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
|
await p.waitForSelector("textarea", { timeout: 10000 }).catch(() => {});
|
|
await p.waitForTimeout(1500);
|
|
const ta = p.locator("textarea").first();
|
|
if (await ta.count()) {
|
|
await ta.fill("Oracle e2e: when a manager requests a laptop, run procurement approval.");
|
|
const draftBtn = p.locator("button.btn-primary").filter({ hasText: /draft/i }).first();
|
|
if (await draftBtn.count()) {
|
|
await draftBtn.click();
|
|
await p.waitForTimeout(8000);
|
|
|
|
const studioCallsBefore = calls.length;
|
|
const confirmBtn = p.locator("button.btn-primary").filter({ hasText: /Confirm structure/i }).first();
|
|
if (await confirmBtn.count()) {
|
|
await confirmBtn.click();
|
|
await p.waitForTimeout(12000);
|
|
const studioFails = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 400);
|
|
const studioOks = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 200 && c.status < 300);
|
|
if (studioFails.length === 0 && studioOks.length > 0) pass("studio_confirm_structure_no_502", `${studioOks.length} OK applies`);
|
|
else fail("studio_confirm_structure_no_502", `${studioFails.length} fails, ${studioOks.length} OK`);
|
|
await p.screenshot({ path: `${OUT}/03-studio-after-confirm.png` });
|
|
} else fail("studio_confirm_structure_button_visible");
|
|
} else fail("studio_start_drafting_button");
|
|
} else fail("studio_intake_visible");
|
|
|
|
// 5. Assistant list processes
|
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
|
await p.waitForTimeout(3500);
|
|
await p.waitForSelector(".agent-composer textarea", { timeout: 10000 }).catch(() => {});
|
|
const agentInput = p.locator(".agent-composer textarea").first();
|
|
if (await agentInput.count() === 0) {
|
|
fail("assistant_input_visible", "no textarea");
|
|
} else {
|
|
await agentInput.fill("list processes");
|
|
await agentInput.press("Enter");
|
|
await p.waitForTimeout(8000);
|
|
}
|
|
const replies = await p.locator(".agent-turn-body").allTextContents();
|
|
const lastReply = replies.slice(-1)[0] || "";
|
|
const mentionsProcess = /purchase|laptop|procurement|requisition|process/i.test(lastReply);
|
|
if (mentionsProcess) pass("assistant_lists_processes", lastReply.slice(0, 100));
|
|
else fail("assistant_lists_processes", lastReply.slice(0, 150));
|
|
await p.screenshot({ path: `${OUT}/04-assistant.png` });
|
|
|
|
// 6. Chat opens
|
|
await p.locator(".tab").filter({ hasText: /Chat/i }).first().click();
|
|
await p.waitForTimeout(5000);
|
|
const chatErr = await p.locator(".chat-err").count();
|
|
const chatShell = await p.locator(".scene").first().innerText().catch(() => "");
|
|
if (chatErr > 0) fail("chat_opens_without_error", "chat-err visible");
|
|
else if (chatShell.toLowerCase().includes("conversation") || chatShell.toLowerCase().includes("talk to") || chatShell.toLowerCase().includes("no conversations") || chatShell.toLowerCase().includes("inbox")) pass("chat_opens_without_error", "chat surface visible");
|
|
else fail("chat_opens_without_error", chatShell.slice(0, 200));
|
|
await p.screenshot({ path: `${OUT}/05-chat.png` });
|
|
|
|
// 7. No snapshot mode anywhere
|
|
const hasSnapshot = await p.evaluate(() => document.body.innerText.toLowerCase().includes("snapshot"));
|
|
if (!hasSnapshot) pass("no_snapshot_text_anywhere");
|
|
else fail("no_snapshot_text_anywhere", "found 'snapshot' in body");
|
|
|
|
// 8. LLM gateway: ask the assistant something non-tool
|
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
|
await p.waitForTimeout(3000);
|
|
const llmInput = p.locator(".agent-composer textarea").first();
|
|
if (await llmInput.count() === 0) { fail("llm_input_visible", "no textarea"); }
|
|
else {
|
|
await llmInput.fill("What is the difference between a flow and a view in EA2?");
|
|
await llmInput.press("Enter");
|
|
await p.waitForTimeout(10000);
|
|
}
|
|
const llmReplies = await p.locator(".agent-turn-body").allTextContents();
|
|
const llmLast = llmReplies.slice(-1)[0] || "";
|
|
const llmCallsList = calls.filter(c => c.url.includes("/api/v1/llm/generate"));
|
|
if (llmCallsList.length > 0) {
|
|
const status = llmCallsList.slice(-1)[0].status;
|
|
if (status === 200) pass("llm_gateway_responds", `200 with ${llmLast.length} char reply`);
|
|
else if (status === 503 || status === 500) {
|
|
if (llmLast.toLowerCase().includes("unavailable") || llmLast.toLowerCase().includes("not configured")) pass("llm_gateway_shows_clear_error", llmLast.slice(0, 100));
|
|
else fail("llm_gateway_shows_clear_error", `status=${status} reply=${llmLast.slice(0, 100)}`);
|
|
} else fail("llm_gateway_responds", `unexpected status ${status}`);
|
|
} else fail("llm_gateway_responds", "no calls to /api/v1/llm/generate");
|
|
await p.screenshot({ path: `${OUT}/06-llm-reply.png` });
|
|
|
|
// 9. Network health summary
|
|
const byStatus = {};
|
|
for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1;
|
|
const fivexx = calls.filter(c => c.status >= 500).length;
|
|
const fast502 = calls.filter(c => c.status === 502).length;
|
|
if (fast502 === 0) pass("no_502_anywhere", `5xx total=${fivexx}`);
|
|
else fail("no_502_anywhere", `${fast502} 502s seen`);
|
|
|
|
await b.close();
|
|
|
|
// Write
|
|
writeFileSync(`${OUT}/results.json`, JSON.stringify(results, null, 2));
|
|
writeFileSync(`${OUT}/network.json`, JSON.stringify(calls, null, 2));
|
|
writeFileSync(`${OUT}/console-errors.json`, JSON.stringify(consoleErrs, null, 2));
|
|
|
|
const passed = results.filter(r => r.ok).length;
|
|
console.log(`\n=== ${passed}/${results.length} checks passed ===`);
|
|
console.log(`api calls: ${calls.length} by status: ${JSON.stringify(byStatus)}`);
|
|
process.exit(passed === results.length ? 0 : 1);
|