From fa17445ff03f50e5b98774583d2a4c5ebed3f43a Mon Sep 17 00:00:00 2001 From: canvas-bot Date: Mon, 15 Jun 2026 21:16:11 +0400 Subject: [PATCH] fix(llm): 12s hard deadline on /api/v1/llm/generate Investigation: the wave-3 [llm] telemetry warns weren't firing on live because the fetch had no client-side timeout. When demo's llm-integration-service was slow/stuck, fetch hung indefinitely past the test's 12s wait, so the fallback path never ran and the user saw no agent reply at all. Add a 12s AbortController-backed deadline. On timeout: console.warn '[llm] gateway timed out after Xms (deadline 12000ms)' + return the graceful 'temporarily unavailable' fallback that renders the tool menu in the chat surface. Uses AbortSignal.any when available to preserve any caller-supplied signal. --- qa/_wave3_verify.mjs | 109 +++++++++++++++++++++++++++++++++++++++++++ src/lib/llmClient.ts | 21 ++++++++- 2 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 qa/_wave3_verify.mjs diff --git a/qa/_wave3_verify.mjs b/qa/_wave3_verify.mjs new file mode 100644 index 0000000..9aa5646 --- /dev/null +++ b/qa/_wave3_verify.mjs @@ -0,0 +1,109 @@ +// Verify the four wave-3 watch-out fixes on live. +import { chromium } from "playwright"; + +const URL = "https://canvas.flow-master.ai/"; + +const results = []; +function pass(n, d = "") { results.push({ n, ok: true, d }); console.log(`✅ PASS ${n}${d ? " — " + d : ""}`); } +function fail(n, d = "") { results.push({ n, ok: false, d }); console.log(`❌ FAIL ${n}${d ? " — " + d : ""}`); } + +const b = await chromium.launch({ headless: true }); +const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); +const p = await ctx.newPage(); + +const calls = []; +const consoleMsgs = []; +p.on("response", (r) => { + const u = r.url(); + if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") }); +}); +p.on("console", (m) => consoleMsgs.push({ type: m.type(), text: m.text() })); + +// Skip onboarding tour; KEEP mission primer fresh +await ctx.addInitScript(() => { + localStorage.setItem("fm.canvas.tour.seen", String(Date.now())); +}); + +console.log("\n=== watch-out 1: Wizard config soft-warn telemetry ==="); +await p.goto(URL, { waitUntil: "domcontentloaded" }); +await p.waitForTimeout(2500); +await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); +await p.waitForSelector(".topbar", { timeout: 30000 }); +await p.waitForTimeout(2000); + +// Open Studio + create draft + check for the soft-warn toast (if PUT fails) OR +// successful flow without errors. +await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); +await p.waitForTimeout(2000); +await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft")); +await p.reload({ waitUntil: "domcontentloaded" }); +await p.waitForTimeout(2500); +await p.locator(".tab").filter({ hasText: /Studio/i }).first().click(); +await p.waitForSelector("textarea", { timeout: 10000 }); +await p.locator("textarea").first().fill("Wave3 verification: laptop procurement flow."); +const draftStart = calls.length; +await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click(); +await p.waitForTimeout(6000); +const newCalls = calls.slice(draftStart); +const flowPosts = newCalls.filter(c => c.method === "POST" && c.url === "/api/ea2/flow"); +const flowPuts = newCalls.filter(c => c.method === "PUT" && c.url.startsWith("/api/ea2/flow/")); +console.log(` POST /api/ea2/flow:`, flowPosts.map(c => c.status)); +console.log(` PUT /api/ea2/flow/:`, flowPuts.map(c => c.status)); +const draftOk = flowPosts.some(c => c.status === 201); +if (draftOk) pass("wave3_studio_draft_lands_via_split_post", `POST 201 + PUT ${flowPuts[0]?.status ?? "missing"}`); +else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flowPosts)}`); + +console.log("\n=== watch-out 2: LLM telemetry shows in console ==="); +await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); +await p.waitForTimeout(3000); +const txt = p.locator(".agent-composer textarea").first(); +await txt.fill("Briefly: what is EA2?"); +await txt.press("Enter"); +await p.waitForTimeout(10000); +const llmConsoleEntries = consoleMsgs.filter(m => + m.type === "warning" || m.type === "info" +).filter(m => m.text.includes("[llm]")); +console.log(` console [llm] entries:`, llmConsoleEntries.length); +for (const c of llmConsoleEntries.slice(0, 3)) console.log(` [${c.type}] ${c.text.slice(0, 180)}`); +if (llmConsoleEntries.length > 0) pass("wave3_llm_telemetry_logged", `${llmConsoleEntries.length} [llm] entries`); +else fail("wave3_llm_telemetry_logged", "no [llm] console entries found"); + +console.log("\n=== watch-out 3: Catalog dedupe (no Laptop Procurement x3) ==="); +await txt.fill("list processes"); +await txt.press("Enter"); +await p.waitForTimeout(8000); +const replies = await p.locator(".agent-turn-body").allTextContents(); +const lastList = replies.slice(-1)[0] || ""; +const laptopMatches = (lastList.match(/laptop\s+procurement/gi) || []).length; +console.log(` "Laptop Procurement" occurrences:`, laptopMatches); +console.log(` full reply: ${lastList.slice(0, 400)}`); +if (laptopMatches <= 1) pass("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`); +else fail("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`); + +console.log("\n=== watch-out 4: Mission primer + no 'snapshot' wording ==="); +// Clear primer-dismissed flag in case it's set +await p.evaluate(() => localStorage.removeItem("fm.canvas.mission.primer.dismissed")); +await p.locator(".tab").filter({ hasText: /Mission/i }).first().click(); +await p.waitForTimeout(3500); +const primerVisible = await p.locator(".mc-primer").count(); +const sceneText = (await p.evaluate(() => document.querySelector(".scene")?.innerText || "")).toLowerCase(); +const hasSnapshotInText = sceneText.includes("snapshot"); +console.log(` .mc-primer count:`, primerVisible); +console.log(` 'snapshot' anywhere in Mission text:`, hasSnapshotInText); +if (primerVisible >= 1) pass("wave3_mission_primer_visible"); +else fail("wave3_mission_primer_visible"); +if (!hasSnapshotInText) pass("wave3_mission_no_snapshot_text"); +else fail("wave3_mission_no_snapshot_text"); + +await p.screenshot({ path: "/tmp/canvas-evidence/wave3-mission.png", fullPage: false }); +await b.close(); + +const passed = results.filter(r => r.ok).length; +console.log(`\n=== ${passed}/${results.length} watch-out checks passed ===`); +console.log(`total api calls: ${calls.length}`); +const byStatus = {}; +for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1; +console.log(`by status: ${JSON.stringify(byStatus)}`); +const fivexx = calls.filter(c => c.status >= 500).length; +console.log(`5xx count: ${fivexx}`); +process.exit(passed === results.length ? 0 : 1); diff --git a/src/lib/llmClient.ts b/src/lib/llmClient.ts index c076955..956d6bd 100644 --- a/src/lib/llmClient.ts +++ b/src/lib/llmClient.ts @@ -66,6 +66,8 @@ function extractText(body: unknown): string | null { ); } +const LLM_TIMEOUT_MS = 12_000; + export const llmClient = { async chat(req: LlmRequest, signal?: AbortSignal): Promise { const token = sessionStorage.getItem("fm.mc.token.v1") || ""; @@ -77,6 +79,13 @@ export const llmClient = { temperature: req.temperature ?? 0.3, }; const started = Date.now(); + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(), LLM_TIMEOUT_MS); + const combinedSignal = signal + ? AbortSignal.any + ? AbortSignal.any([signal, deadline.signal]) + : deadline.signal + : deadline.signal; try { const res = await fetch(`${api.config.baseUrl}/api/v1/llm/generate`, { method: "POST", @@ -85,7 +94,7 @@ export const llmClient = { Authorization: `Bearer ${token}`, }, body: JSON.stringify(body), - signal, + signal: combinedSignal, }); const elapsed = Date.now() - started; if (res.status === 503) { @@ -115,8 +124,16 @@ export const llmClient = { console.info(`[llm] gateway ok in ${elapsed}ms · provider=${provider ?? "?"} · ${content.length}ch`); return { ok: true, content: content.trim(), provider }; } catch (e) { - console.warn(`[llm] gateway fetch threw in ${Date.now() - started}ms: ${(e as Error).message}`); + const elapsed = Date.now() - started; + const aborted = (e as Error).name === "AbortError"; + if (aborted && elapsed >= LLM_TIMEOUT_MS - 100) { + console.warn(`[llm] gateway timed out after ${elapsed}ms (deadline ${LLM_TIMEOUT_MS}ms)`); + return { ok: false, configured: false, reason: `LLM gateway timed out after ${Math.round(elapsed / 1000)}s` }; + } + console.warn(`[llm] gateway fetch threw in ${elapsed}ms: ${(e as Error).message}`); return { ok: false, configured: false, reason: `LLM gateway unreachable: ${(e as Error).message}` }; + } finally { + clearTimeout(timer); } }, };