fix(llm): 12s hard deadline on /api/v1/llm/generate
build-and-publish / test (push) Has been cancelled
build-and-publish / image (push) Has been cancelled

Investigation: the wave-3 [llm] telemetry warns weren't firing on live
because the fetch had no client-side timeout. When demo's
llm-integration-service was slow/stuck, fetch hung indefinitely past
the test's 12s wait, so the fallback path never ran and the user saw
no agent reply at all.

Add a 12s AbortController-backed deadline. On timeout: console.warn
'[llm] gateway timed out after Xms (deadline 12000ms)' + return the
graceful 'temporarily unavailable' fallback that renders the tool
menu in the chat surface. Uses AbortSignal.any when available to
preserve any caller-supplied signal.
This commit is contained in:
canvas-bot
2026-06-15 21:16:11 +04:00
parent 9502e36c32
commit fa17445ff0
2 changed files with 128 additions and 2 deletions
+109
View File
@@ -0,0 +1,109 @@
// Verify the four wave-3 watch-out fixes on live.
import { chromium } from "playwright";
const URL = "https://canvas.flow-master.ai/";
const results = [];
function pass(n, d = "") { results.push({ n, ok: true, d }); console.log(`✅ PASS ${n}${d ? " — " + d : ""}`); }
function fail(n, d = "") { results.push({ n, ok: false, d }); console.log(`❌ FAIL ${n}${d ? " — " + d : ""}`); }
const b = await chromium.launch({ headless: true });
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
const p = await ctx.newPage();
const calls = [];
const consoleMsgs = [];
p.on("response", (r) => {
const u = r.url();
if (u.includes("/api/")) calls.push({ status: r.status(), method: r.request().method(), url: u.replace("https://canvas.flow-master.ai", "") });
});
p.on("console", (m) => consoleMsgs.push({ type: m.type(), text: m.text() }));
// Skip onboarding tour; KEEP mission primer fresh
await ctx.addInitScript(() => {
localStorage.setItem("fm.canvas.tour.seen", String(Date.now()));
});
console.log("\n=== watch-out 1: Wizard config soft-warn telemetry ===");
await p.goto(URL, { waitUntil: "domcontentloaded" });
await p.waitForTimeout(2500);
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
await p.waitForSelector(".topbar", { timeout: 30000 });
await p.waitForTimeout(2000);
// Open Studio + create draft + check for the soft-warn toast (if PUT fails) OR
// successful flow without errors.
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
await p.waitForTimeout(2000);
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
await p.reload({ waitUntil: "domcontentloaded" });
await p.waitForTimeout(2500);
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
await p.waitForSelector("textarea", { timeout: 10000 });
await p.locator("textarea").first().fill("Wave3 verification: laptop procurement flow.");
const draftStart = calls.length;
await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click();
await p.waitForTimeout(6000);
const newCalls = calls.slice(draftStart);
const flowPosts = newCalls.filter(c => c.method === "POST" && c.url === "/api/ea2/flow");
const flowPuts = newCalls.filter(c => c.method === "PUT" && c.url.startsWith("/api/ea2/flow/"));
console.log(` POST /api/ea2/flow:`, flowPosts.map(c => c.status));
console.log(` PUT /api/ea2/flow/<key>:`, flowPuts.map(c => c.status));
const draftOk = flowPosts.some(c => c.status === 201);
if (draftOk) pass("wave3_studio_draft_lands_via_split_post", `POST 201 + PUT ${flowPuts[0]?.status ?? "missing"}`);
else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flowPosts)}`);
console.log("\n=== watch-out 2: LLM telemetry shows in console ===");
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
await p.waitForTimeout(3000);
const txt = p.locator(".agent-composer textarea").first();
await txt.fill("Briefly: what is EA2?");
await txt.press("Enter");
await p.waitForTimeout(10000);
const llmConsoleEntries = consoleMsgs.filter(m =>
m.type === "warning" || m.type === "info"
).filter(m => m.text.includes("[llm]"));
console.log(` console [llm] entries:`, llmConsoleEntries.length);
for (const c of llmConsoleEntries.slice(0, 3)) console.log(` [${c.type}] ${c.text.slice(0, 180)}`);
if (llmConsoleEntries.length > 0) pass("wave3_llm_telemetry_logged", `${llmConsoleEntries.length} [llm] entries`);
else fail("wave3_llm_telemetry_logged", "no [llm] console entries found");
console.log("\n=== watch-out 3: Catalog dedupe (no Laptop Procurement x3) ===");
await txt.fill("list processes");
await txt.press("Enter");
await p.waitForTimeout(8000);
const replies = await p.locator(".agent-turn-body").allTextContents();
const lastList = replies.slice(-1)[0] || "";
const laptopMatches = (lastList.match(/laptop\s+procurement/gi) || []).length;
console.log(` "Laptop Procurement" occurrences:`, laptopMatches);
console.log(` full reply: ${lastList.slice(0, 400)}`);
if (laptopMatches <= 1) pass("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`);
else fail("wave3_catalog_no_duplicate_laptop", `count=${laptopMatches}`);
console.log("\n=== watch-out 4: Mission primer + no 'snapshot' wording ===");
// Clear primer-dismissed flag in case it's set
await p.evaluate(() => localStorage.removeItem("fm.canvas.mission.primer.dismissed"));
await p.locator(".tab").filter({ hasText: /Mission/i }).first().click();
await p.waitForTimeout(3500);
const primerVisible = await p.locator(".mc-primer").count();
const sceneText = (await p.evaluate(() => document.querySelector(".scene")?.innerText || "")).toLowerCase();
const hasSnapshotInText = sceneText.includes("snapshot");
console.log(` .mc-primer count:`, primerVisible);
console.log(` 'snapshot' anywhere in Mission text:`, hasSnapshotInText);
if (primerVisible >= 1) pass("wave3_mission_primer_visible");
else fail("wave3_mission_primer_visible");
if (!hasSnapshotInText) pass("wave3_mission_no_snapshot_text");
else fail("wave3_mission_no_snapshot_text");
await p.screenshot({ path: "/tmp/canvas-evidence/wave3-mission.png", fullPage: false });
await b.close();
const passed = results.filter(r => r.ok).length;
console.log(`\n=== ${passed}/${results.length} watch-out checks passed ===`);
console.log(`total api calls: ${calls.length}`);
const byStatus = {};
for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1;
console.log(`by status: ${JSON.stringify(byStatus)}`);
const fivexx = calls.filter(c => c.status >= 500).length;
console.log(`5xx count: ${fivexx}`);
process.exit(passed === results.length ? 0 : 1);