canvas-bot
660de0341b
Oracle wave-4: 'Re-test LLM with a successful upstream response,
not only the 500/503 fallback path. The current evidence proves
graceful degradation but not that LLM is actually on producing
model-backed answers.'
The user said 'Take the LLM integration that we already have on
the other websites and put it on.' demo's llm-integration-service
is 503 (no provider key), but dev.flow-master.ai's identical
gateway returns real OpenAI-backed completions and accepts the
canvas-issued JWT plus has CORS open to canvas.flow-master.ai.
Refactor llmClient:
- Extract callGateway() that returns LlmResponse or 'retry' sentinel
- LLM_GATEWAYS = [primary (canvas's own), fallback-dev (dev.fm.ai)]
- Retry on 500/502/503/504/404 and network errors against the next gateway
- Telemetry prefixes each log with gateway name ('primary' vs 'fallback-dev')
- 12s deadline still applies across the whole cascade
- Only final 'all gateways exhausted' warn if every link in the chain fails
Net result: canvas users get a real model-backed answer on every
non-tool prompt instead of the 'temporarily unavailable' fallback.
112 lines
5.4 KiB
JavaScript
112 lines
5.4 KiB
JavaScript
// Oracle's wave-3 negative path: force the wizard config PUT to fail
|
|
// after the draft POST succeeds, then verify:
|
|
// 1. The soft toast appears with the exact copy
|
|
// 2. The draft remains usable (no hard crash)
|
|
// 3. Confirm/handleStructureSave retries the attach
|
|
import { chromium } from "playwright";
|
|
|
|
const URL = "https://canvas.flow-master.ai/";
|
|
const r = [];
|
|
const pass = (n, d = "") => { r.push({ n, ok: true, d }); console.log(`PASS ${n}${d ? " " + d : ""}`); };
|
|
const fail = (n, d = "") => { r.push({ n, ok: false, d }); console.log(`FAIL ${n}${d ? " " + d : ""}`); };
|
|
|
|
const b = await chromium.launch({ headless: true });
|
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
|
const p = await ctx.newPage();
|
|
const calls = [];
|
|
const consoleMsgs = [];
|
|
p.on("console", m => consoleMsgs.push({ type: m.type(), text: m.text().slice(0, 220) }));
|
|
p.on("response", res => {
|
|
if (res.url().includes("/api/")) calls.push({ status: res.status(), method: res.request().method(), url: res.url().replace("https://canvas.flow-master.ai", "") });
|
|
});
|
|
|
|
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
|
|
|
console.log("\n--- Force the first wizard PUT to fail with 500 ---");
|
|
let injected = 0;
|
|
const targetKeys = new Set();
|
|
await p.route("**/api/ea2/flow/*", async (route, request) => {
|
|
const u = request.url();
|
|
if (request.method() === "PUT" && injected === 0) {
|
|
const m = u.match(/\/api\/ea2\/flow\/([a-f0-9]{16,})$/);
|
|
if (m) {
|
|
targetKeys.add(m[1]);
|
|
injected++;
|
|
console.log(` injecting 500 on first PUT to ${m[1]}`);
|
|
return route.fulfill({ status: 500, contentType: "application/json", body: JSON.stringify({ error: "Injected failure for negative-path test" }) });
|
|
}
|
|
}
|
|
return route.continue();
|
|
});
|
|
|
|
await p.goto(URL, { waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
|
await p.waitForSelector(".topbar", { timeout: 30000 });
|
|
await p.waitForTimeout(2000);
|
|
|
|
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
|
|
await p.reload({ waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
|
await p.waitForSelector("textarea", { timeout: 10000 });
|
|
await p.locator("textarea").first().fill("Negative path: force PUT failure on attach.");
|
|
await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click();
|
|
|
|
// Catch the toast while it is on screen (default toast lifetime is short).
|
|
let toastSeen = "";
|
|
try {
|
|
await p.locator(".toast", { hasText: /didn't attach/i }).first().waitFor({ timeout: 4000 });
|
|
toastSeen = await p.locator(".toast", { hasText: /didn't attach/i }).first().textContent() || "";
|
|
} catch { /* may auto-dismiss; we still have the console.warn */ }
|
|
|
|
await p.waitForTimeout(4000);
|
|
const posts = calls.filter(c => c.method === "POST" && c.url === "/api/ea2/flow");
|
|
const puts = calls.filter(c => c.method === "PUT" && /\/api\/ea2\/flow\/[a-f0-9]+/.test(c.url));
|
|
console.log(` POSTs:`, posts.map(c => c.status));
|
|
console.log(` PUTs:`, puts.map(c => c.status));
|
|
console.log(` toast captured:`, toastSeen || "(auto-dismissed before assertion)");
|
|
|
|
if (posts.some(c => c.status === 201)) pass("neg_draft_post_succeeded");
|
|
else fail("neg_draft_post_succeeded", `posts=${JSON.stringify(posts)}`);
|
|
|
|
if (puts.some(c => c.status === 500)) pass("neg_first_put_intercepted_500");
|
|
else fail("neg_first_put_intercepted_500", `puts=${JSON.stringify(puts)}`);
|
|
|
|
if (/didn't attach.*Confirm/i.test(toastSeen)) pass("neg_soft_toast_with_exact_copy", toastSeen.slice(0, 80));
|
|
else fail("neg_soft_toast_with_exact_copy", `toastSeen="${toastSeen}"`);
|
|
|
|
const warnSeen = consoleMsgs.some(m => m.text.includes("[wizard] config attach failed"));
|
|
if (warnSeen) pass("neg_console_warn_logged");
|
|
else fail("neg_console_warn_logged");
|
|
|
|
// Draft usability: any Studio step button visible OR localStorage has flowKey
|
|
const flowKeyInLs = await p.evaluate(() => {
|
|
try { return !!JSON.parse(localStorage.getItem("fm.mc.wizard.draft") || "{}").flowKey; } catch { return false; }
|
|
});
|
|
if (flowKeyInLs) pass("neg_draft_still_usable_after_failure", "flowKey in localStorage");
|
|
else fail("neg_draft_still_usable_after_failure");
|
|
|
|
console.log("\n--- Click Confirm Structure, verify retry attach fires ---");
|
|
const putsBeforeConfirm = puts.length;
|
|
await p.locator("button.btn-primary").filter({ hasText: /Confirm/i }).first().click();
|
|
await p.waitForTimeout(8000);
|
|
const putsAfter = calls.filter(c => c.method === "PUT" && /\/api\/ea2\/flow\/[a-f0-9]+/.test(c.url));
|
|
const newPuts = putsAfter.slice(putsBeforeConfirm);
|
|
console.log(` PUTs during Confirm:`, newPuts.map(c => `${c.status} ${c.url.slice(-30)}`));
|
|
|
|
const retryPut = newPuts.find(c => Array.from(targetKeys).some(k => c.url.includes(k)));
|
|
if (retryPut && retryPut.status === 200) pass("neg_confirm_retries_attach_successfully", `PUT ${retryPut.status}`);
|
|
else if (retryPut) pass("neg_confirm_retries_attach_fired", `PUT ${retryPut.status} (attempt made)`);
|
|
else fail("neg_confirm_retries_attach", `no PUT to original draft key during Confirm`);
|
|
|
|
const total5xx = calls.filter(c => c.status >= 500).length;
|
|
console.log(`\n5xx count: ${total5xx} (expected 1 from injection)`);
|
|
|
|
await p.screenshot({ path: "/tmp/canvas-evidence/wave3-neg-path.png" });
|
|
await b.close();
|
|
|
|
const passed = r.filter(x => x.ok).length;
|
|
console.log(`\n=== ${passed}/${r.length} negative-path checks passed ===`);
|
|
process.exit(passed === r.length ? 0 : 1);
|