fix(wizard): retry createDraft on 500/502/503/504 + human error message
POST /api/ea2/flow currently returns 500 after 19s on demo upstream. Retry once with backoff, then surface a clear 'EA2 backend temporarily unable to create new processes' message naming the status. The user gets actionable info instead of a raw 500.
This commit is contained in:
canvas-bot
@@ -0,0 +1,153 @@
|
||||
// Full end-to-end live verification of the user's specific bugs.
|
||||
import { chromium } from "playwright";
|
||||
import { writeFileSync, mkdirSync } from "node:fs";
|
||||
|
||||
const URL = "https://canvas.flow-master.ai/";
|
||||
const OUT = "/tmp/canvas-evidence/oracle-e2e";
|
||||
mkdirSync(OUT, { recursive: true });
|
||||
|
||||
const results = [];
|
||||
function pass(name, detail = "") { results.push({ name, ok: true, detail }); console.log(`✅ PASS ${name}${detail ? " — " + detail : ""}`); }
|
||||
function fail(name, detail = "") { results.push({ name, ok: false, detail }); console.log(`❌ FAIL ${name}${detail ? " — " + detail : ""}`); }
|
||||
|
||||
const calls = [];
|
||||
const consoleErrs = [];
|
||||
|
||||
const b = await chromium.launch({ headless: true });
|
||||
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
||||
const p = await ctx.newPage();
|
||||
p.on("response", (r) => {
|
||||
const u = r.url();
|
||||
if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") });
|
||||
});
|
||||
p.on("console", (m) => { if (m.type() === "error") consoleErrs.push(m.text().slice(0, 200)); });
|
||||
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
||||
|
||||
// 1. Login → mission
|
||||
await p.goto(URL, { waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
||||
await p.waitForSelector(".topbar", { timeout: 30000 });
|
||||
await p.waitForTimeout(2000);
|
||||
const shell = await p.evaluate(() => document.querySelector("[class*='shell-']")?.className);
|
||||
if (shell?.includes("mission")) pass("login_lands_on_mission", shell);
|
||||
else fail("login_lands_on_mission", shell);
|
||||
await p.screenshot({ path: `${OUT}/01-mission.png` });
|
||||
|
||||
// 2. Topbar count
|
||||
const tabCount = await p.locator(".tab").count();
|
||||
const tabs = await p.locator(".tab").allTextContents();
|
||||
if (tabCount === 6) pass("topbar_has_6_tabs", tabs.map(t => t.trim()).join(", "));
|
||||
else fail("topbar_has_6_tabs", `count=${tabCount}, tabs=${tabs.join(",")}`);
|
||||
|
||||
// 3. User menu opens with 5 items
|
||||
await p.locator(".user-avatar").click();
|
||||
await p.waitForTimeout(700);
|
||||
const menuItems = await p.locator(".user-menu-item").allTextContents();
|
||||
if (menuItems.length >= 4) pass("user_menu_has_items", menuItems.map(m => m.trim().slice(0, 30)).join(" | "));
|
||||
else fail("user_menu_has_items", JSON.stringify(menuItems));
|
||||
await p.screenshot({ path: `${OUT}/02-user-menu.png` });
|
||||
await p.keyboard.press("Escape");
|
||||
await p.waitForTimeout(500);
|
||||
|
||||
// 4. Open Studio → fill → start drafting → confirm structure
|
||||
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
||||
await p.waitForTimeout(2500);
|
||||
// Clear any stale draft from localStorage
|
||||
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
|
||||
await p.reload({ waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
||||
await p.waitForSelector("textarea", { timeout: 10000 }).catch(() => {});
|
||||
await p.waitForTimeout(1500);
|
||||
const ta = p.locator("textarea").first();
|
||||
if (await ta.count()) {
|
||||
await ta.fill("Oracle e2e: when a manager requests a laptop, run procurement approval.");
|
||||
const draftBtn = p.locator("button.btn-primary").filter({ hasText: /draft/i }).first();
|
||||
if (await draftBtn.count()) {
|
||||
await draftBtn.click();
|
||||
await p.waitForTimeout(8000);
|
||||
|
||||
const studioCallsBefore = calls.length;
|
||||
const confirmBtn = p.locator("button.btn-primary").filter({ hasText: /Confirm structure/i }).first();
|
||||
if (await confirmBtn.count()) {
|
||||
await confirmBtn.click();
|
||||
await p.waitForTimeout(12000);
|
||||
const studioFails = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 400);
|
||||
const studioOks = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 200 && c.status < 300);
|
||||
if (studioFails.length === 0 && studioOks.length > 0) pass("studio_confirm_structure_no_502", `${studioOks.length} OK applies`);
|
||||
else fail("studio_confirm_structure_no_502", `${studioFails.length} fails, ${studioOks.length} OK`);
|
||||
await p.screenshot({ path: `${OUT}/03-studio-after-confirm.png` });
|
||||
} else fail("studio_confirm_structure_button_visible");
|
||||
} else fail("studio_start_drafting_button");
|
||||
} else fail("studio_intake_visible");
|
||||
|
||||
// 5. Assistant list processes
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForTimeout(2500);
|
||||
const agentInput = p.locator(".agent-input, input[placeholder*='Ask']").first();
|
||||
await agentInput.fill("list processes");
|
||||
await agentInput.press("Enter");
|
||||
await p.waitForTimeout(6000);
|
||||
const replies = await p.locator(".agent-msg-text, .agent-msg-body").allTextContents();
|
||||
const lastReply = replies.slice(-1)[0] || "";
|
||||
const mentionsProcess = /purchase|laptop|procurement|requisition|process/i.test(lastReply);
|
||||
if (mentionsProcess) pass("assistant_lists_processes", lastReply.slice(0, 100));
|
||||
else fail("assistant_lists_processes", lastReply.slice(0, 150));
|
||||
await p.screenshot({ path: `${OUT}/04-assistant.png` });
|
||||
|
||||
// 6. Chat opens
|
||||
await p.locator(".tab").filter({ hasText: /Chat/i }).first().click();
|
||||
await p.waitForTimeout(4000);
|
||||
const chatErr = await p.locator(".chat-err").count();
|
||||
const chatEmpty = await p.locator(".chat-empty").count();
|
||||
const chatShell = await p.locator(".chat-shell, .chat-scene, .scene").first().innerText().catch(() => "");
|
||||
if (chatErr > 0) fail("chat_opens_without_error", "chat-err visible");
|
||||
else if (chatShell.includes("Conversations") || chatShell.includes("Talk to your team") || chatShell.includes("No conversations")) pass("chat_opens_without_error", "chat surface visible");
|
||||
else fail("chat_opens_without_error", chatShell.slice(0, 200));
|
||||
await p.screenshot({ path: `${OUT}/05-chat.png` });
|
||||
|
||||
// 7. No snapshot mode anywhere
|
||||
const hasSnapshot = await p.evaluate(() => document.body.innerText.toLowerCase().includes("snapshot"));
|
||||
if (!hasSnapshot) pass("no_snapshot_text_anywhere");
|
||||
else fail("no_snapshot_text_anywhere", "found 'snapshot' in body");
|
||||
|
||||
// 8. LLM gateway: ask the assistant something non-tool
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForTimeout(1500);
|
||||
const llmInput = p.locator(".agent-input, input[placeholder*='Ask']").first();
|
||||
await llmInput.fill("What is the difference between a flow and a view in EA2?");
|
||||
await llmInput.press("Enter");
|
||||
await p.waitForTimeout(8000);
|
||||
const llmReplies = await p.locator(".agent-msg-text, .agent-msg-body").allTextContents();
|
||||
const llmLast = llmReplies.slice(-1)[0] || "";
|
||||
const llmCallsList = calls.filter(c => c.url.includes("/api/v1/llm/generate"));
|
||||
if (llmCallsList.length > 0) {
|
||||
const status = llmCallsList.slice(-1)[0].status;
|
||||
if (status === 200) pass("llm_gateway_responds", `200 with ${llmLast.length} char reply`);
|
||||
else if (status === 503 || status === 500) {
|
||||
if (llmLast.toLowerCase().includes("unavailable") || llmLast.toLowerCase().includes("not configured")) pass("llm_gateway_shows_clear_error", llmLast.slice(0, 100));
|
||||
else fail("llm_gateway_shows_clear_error", `status=${status} reply=${llmLast.slice(0, 100)}`);
|
||||
} else fail("llm_gateway_responds", `unexpected status ${status}`);
|
||||
} else fail("llm_gateway_responds", "no calls to /api/v1/llm/generate");
|
||||
await p.screenshot({ path: `${OUT}/06-llm-reply.png` });
|
||||
|
||||
// 9. Network health summary
|
||||
const byStatus = {};
|
||||
for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1;
|
||||
const fivexx = calls.filter(c => c.status >= 500).length;
|
||||
const fast502 = calls.filter(c => c.status === 502).length;
|
||||
if (fast502 === 0) pass("no_502_anywhere", `5xx total=${fivexx}`);
|
||||
else fail("no_502_anywhere", `${fast502} 502s seen`);
|
||||
|
||||
await b.close();
|
||||
|
||||
// Write
|
||||
writeFileSync(`${OUT}/results.json`, JSON.stringify(results, null, 2));
|
||||
writeFileSync(`${OUT}/network.json`, JSON.stringify(calls, null, 2));
|
||||
writeFileSync(`${OUT}/console-errors.json`, JSON.stringify(consoleErrs, null, 2));
|
||||
|
||||
const passed = results.filter(r => r.ok).length;
|
||||
console.log(`\n=== ${passed}/${results.length} checks passed ===`);
|
||||
console.log(`api calls: ${calls.length} by status: ${JSON.stringify(byStatus)}`);
|
||||
process.exit(passed === results.length ? 0 : 1);
|
||||
Reference in New Issue
Block a user