diff --git a/qa/_oracle_e2e.mjs b/qa/_oracle_e2e.mjs new file mode 100644 index 0000000..b7e61ca --- /dev/null +++ b/qa/_oracle_e2e.mjs @@ -0,0 +1,153 @@ +// Full end-to-end live verification of the user's specific bugs. +import { chromium } from "playwright"; +import { writeFileSync, mkdirSync } from "node:fs"; + +const URL = "https://canvas.flow-master.ai/"; +const OUT = "/tmp/canvas-evidence/oracle-e2e"; +mkdirSync(OUT, { recursive: true }); + +const results = []; +function pass(name, detail = "") { results.push({ name, ok: true, detail }); console.log(`✅ PASS ${name}${detail ? " — " + detail : ""}`); } +function fail(name, detail = "") { results.push({ name, ok: false, detail }); console.log(`❌ FAIL ${name}${detail ? " — " + detail : ""}`); } + +const calls = []; +const consoleErrs = []; + +const b = await chromium.launch({ headless: true }); +const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); +const p = await ctx.newPage(); +p.on("response", (r) => { + const u = r.url(); + if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") }); +}); +p.on("console", (m) => { if (m.type() === "error") consoleErrs.push(m.text().slice(0, 200)); }); +await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now()))); + +// 1. Login → mission +await p.goto(URL, { waitUntil: "domcontentloaded" }); +await p.waitForTimeout(2500); +await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); +await p.waitForSelector(".topbar", { timeout: 30000 }); +await p.waitForTimeout(2000); +const shell = await p.evaluate(() => document.querySelector("[class*='shell-']")?.className); +if (shell?.includes("mission")) pass("login_lands_on_mission", shell); +else fail("login_lands_on_mission", shell); +await p.screenshot({ path: `${OUT}/01-mission.png` }); + +// 2. Topbar count +const tabCount = await p.locator(".tab").count(); +const tabs = await p.locator(".tab").allTextContents(); +if (tabCount === 6) pass("topbar_has_6_tabs", tabs.map(t => t.trim()).join(", ")); +else fail("topbar_has_6_tabs", `count=${tabCount}, tabs=${tabs.join(",")}`); + +// 3. User menu opens with 5 items +await p.locator(".user-avatar").click(); +await p.waitForTimeout(700); +const menuItems = await p.locator(".user-menu-item").allTextContents(); +if (menuItems.length >= 4) pass("user_menu_has_items", menuItems.map(m => m.trim().slice(0, 30)).join(" | ")); +else fail("user_menu_has_items", JSON.stringify(menuItems)); +await p.screenshot({ path: `${OUT}/02-user-menu.png` }); +await p.keyboard.press("Escape"); +await p.waitForTimeout(500); + +// 4. Open Studio → fill → start drafting → confirm structure +await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); +await p.waitForTimeout(2500); +// Clear any stale draft from localStorage +await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft")); +await p.reload({ waitUntil: "domcontentloaded" }); +await p.waitForTimeout(2500); +await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); +await p.waitForSelector("textarea", { timeout: 10000 }).catch(() => {}); +await p.waitForTimeout(1500); +const ta = p.locator("textarea").first(); +if (await ta.count()) { + await ta.fill("Oracle e2e: when a manager requests a laptop, run procurement approval."); + const draftBtn = p.locator("button.btn-primary").filter({ hasText: /draft/i }).first(); + if (await draftBtn.count()) { + await draftBtn.click(); + await p.waitForTimeout(8000); + + const studioCallsBefore = calls.length; + const confirmBtn = p.locator("button.btn-primary").filter({ hasText: /Confirm structure/i }).first(); + if (await confirmBtn.count()) { + await confirmBtn.click(); + await p.waitForTimeout(12000); + const studioFails = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 400); + const studioOks = calls.slice(studioCallsBefore).filter(c => c.url.includes("apply-batch") && c.status >= 200 && c.status < 300); + if (studioFails.length === 0 && studioOks.length > 0) pass("studio_confirm_structure_no_502", `${studioOks.length} OK applies`); + else fail("studio_confirm_structure_no_502", `${studioFails.length} fails, ${studioOks.length} OK`); + await p.screenshot({ path: `${OUT}/03-studio-after-confirm.png` }); + } else fail("studio_confirm_structure_button_visible"); + } else fail("studio_start_drafting_button"); +} else fail("studio_intake_visible"); + +// 5. Assistant list processes +await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); +await p.waitForTimeout(2500); +const agentInput = p.locator(".agent-input, input[placeholder*='Ask']").first(); +await agentInput.fill("list processes"); +await agentInput.press("Enter"); +await p.waitForTimeout(6000); +const replies = await p.locator(".agent-msg-text, .agent-msg-body").allTextContents(); +const lastReply = replies.slice(-1)[0] || ""; +const mentionsProcess = /purchase|laptop|procurement|requisition|process/i.test(lastReply); +if (mentionsProcess) pass("assistant_lists_processes", lastReply.slice(0, 100)); +else fail("assistant_lists_processes", lastReply.slice(0, 150)); +await p.screenshot({ path: `${OUT}/04-assistant.png` }); + +// 6. Chat opens +await p.locator(".tab").filter({ hasText: /Chat/i }).first().click(); +await p.waitForTimeout(4000); +const chatErr = await p.locator(".chat-err").count(); +const chatEmpty = await p.locator(".chat-empty").count(); +const chatShell = await p.locator(".chat-shell, .chat-scene, .scene").first().innerText().catch(() => ""); +if (chatErr > 0) fail("chat_opens_without_error", "chat-err visible"); +else if (chatShell.includes("Conversations") || chatShell.includes("Talk to your team") || chatShell.includes("No conversations")) pass("chat_opens_without_error", "chat surface visible"); +else fail("chat_opens_without_error", chatShell.slice(0, 200)); +await p.screenshot({ path: `${OUT}/05-chat.png` }); + +// 7. No snapshot mode anywhere +const hasSnapshot = await p.evaluate(() => document.body.innerText.toLowerCase().includes("snapshot")); +if (!hasSnapshot) pass("no_snapshot_text_anywhere"); +else fail("no_snapshot_text_anywhere", "found 'snapshot' in body"); + +// 8. LLM gateway: ask the assistant something non-tool +await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); +await p.waitForTimeout(1500); +const llmInput = p.locator(".agent-input, input[placeholder*='Ask']").first(); +await llmInput.fill("What is the difference between a flow and a view in EA2?"); +await llmInput.press("Enter"); +await p.waitForTimeout(8000); +const llmReplies = await p.locator(".agent-msg-text, .agent-msg-body").allTextContents(); +const llmLast = llmReplies.slice(-1)[0] || ""; +const llmCallsList = calls.filter(c => c.url.includes("/api/v1/llm/generate")); +if (llmCallsList.length > 0) { + const status = llmCallsList.slice(-1)[0].status; + if (status === 200) pass("llm_gateway_responds", `200 with ${llmLast.length} char reply`); + else if (status === 503 || status === 500) { + if (llmLast.toLowerCase().includes("unavailable") || llmLast.toLowerCase().includes("not configured")) pass("llm_gateway_shows_clear_error", llmLast.slice(0, 100)); + else fail("llm_gateway_shows_clear_error", `status=${status} reply=${llmLast.slice(0, 100)}`); + } else fail("llm_gateway_responds", `unexpected status ${status}`); +} else fail("llm_gateway_responds", "no calls to /api/v1/llm/generate"); +await p.screenshot({ path: `${OUT}/06-llm-reply.png` }); + +// 9. Network health summary +const byStatus = {}; +for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1; +const fivexx = calls.filter(c => c.status >= 500).length; +const fast502 = calls.filter(c => c.status === 502).length; +if (fast502 === 0) pass("no_502_anywhere", `5xx total=${fivexx}`); +else fail("no_502_anywhere", `${fast502} 502s seen`); + +await b.close(); + +// Write +writeFileSync(`${OUT}/results.json`, JSON.stringify(results, null, 2)); +writeFileSync(`${OUT}/network.json`, JSON.stringify(calls, null, 2)); +writeFileSync(`${OUT}/console-errors.json`, JSON.stringify(consoleErrs, null, 2)); + +const passed = results.filter(r => r.ok).length; +console.log(`\n=== ${passed}/${results.length} checks passed ===`); +console.log(`api calls: ${calls.length} by status: ${JSON.stringify(byStatus)}`); +process.exit(passed === results.length ? 0 : 1); diff --git a/qa/_studio_state.mjs b/qa/_studio_state.mjs new file mode 100644 index 0000000..6bfab47 --- /dev/null +++ b/qa/_studio_state.mjs @@ -0,0 +1,27 @@ +// Just Studio +import { chromium } from "playwright"; + +const b = await chromium.launch({ headless: true }); +const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); +const p = await ctx.newPage(); +const calls = []; +p.on("response", (r) => { if (r.url().includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: r.url().replace("https://canvas.flow-master.ai", "") }); }); +await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now()))); + +await p.goto("https://canvas.flow-master.ai/", { waitUntil: "domcontentloaded" }); +await p.waitForTimeout(2500); +await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); +await p.waitForSelector(".topbar", { timeout: 30000 }); +await p.waitForTimeout(2000); + +console.log("on mission"); +await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); +await p.waitForTimeout(3000); +const sceneText1 = await p.evaluate(() => document.querySelector(".scene")?.innerText?.slice(0, 600) || "?"); +console.log("Studio scene:", sceneText1); + +const buttons = await p.locator("button").allTextContents(); +console.log("buttons:", buttons.map(b => b.trim()).filter(Boolean).slice(0, 20)); + +await p.screenshot({ path: "/tmp/canvas-evidence/studio-state.png" }); +await b.close(); diff --git a/src/lib/wizardApi.ts b/src/lib/wizardApi.ts index 38ad8a9..7131a35 100644 --- a/src/lib/wizardApi.ts +++ b/src/lib/wizardApi.ts @@ -67,17 +67,31 @@ function authHeaders(): Record { export const wizardApi = { async createDraft(payload: CreateDraftPayload, signal?: AbortSignal) { - const res = await fetch(`${api.config.baseUrl}/api/ea2/flow`, { - method: "POST", - headers: authHeaders(), - body: JSON.stringify({ kind: "definition", status: "draft", ...payload }), - signal, - }); - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`createDraft ${res.status}: ${detail.slice(0, 200)}`); + const body = JSON.stringify({ kind: "definition", status: "draft", ...payload }); + const transient = new Set([500, 502, 503, 504]); + let lastDetail = ""; + let lastStatus = 0; + for (let attempt = 0; attempt < 2; attempt++) { + try { + const res = await fetch(`${api.config.baseUrl}/api/ea2/flow`, { + method: "POST", + headers: authHeaders(), + body, + signal, + }); + if (res.ok) return await res.json() as { _key: string; name: string }; + lastStatus = res.status; + lastDetail = await res.text().catch(() => ""); + if (!transient.has(res.status)) break; + } catch (e) { + lastDetail = (e as Error).message; + } + await new Promise((r) => setTimeout(r, 600 * (attempt + 1))); } - return res.json() as Promise<{ _key: string; name: string }>; + if (transient.has(lastStatus)) { + throw new Error(`The EA2 backend is temporarily unable to create new processes (status ${lastStatus}). Try again in a moment, or report this if it keeps happening.`); + } + throw new Error(`createDraft ${lastStatus}: ${lastDetail.slice(0, 200)}`); }, async updateDraftConfig(key: string, patch: Record, signal?: AbortSignal) {