shad/canvas-frontend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: track LLM gateway 5xx separately + lock cascade integrity
build-and-publish / test (push) Has been cancelled
Details
build-and-publish / image (push) Has been cancelled
Details

Browse Source 
Oracle wave-4 watch-outs:
- '5xx count: 1 should not be hidden'
- 'Add one future regression check that fails if both primary and
   fallback LLM paths fail, since the fallback tunnel is now part
   of the live behavior'

E2E now reports llm-gateway 5xx and system 5xx as two distinct
counts. The canvas-primary 503 is acknowledged + counted; only
system 5xx must be zero. New no_system_5xx assertion guards that.

wave4 adds gap1_cascade_did_not_exhaust: fails if telemetry shows
'[llm] all gateways exhausted' on the live assistant path. Locks
in that at least one gateway (primary or fallback-dev) is
returning real model output.
This commit is contained in:
canvas-bot
2026-06-15 22:31:05 +04:00
parent e29667b348
commit 99203267aa
2 changed files with 22 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
qa/_oracle_e2e.mjs
+10 -3
View File
@@ -137,13 +137,20 @@ if (hasFallbackMenu || hasRealReply) pass("llm_user_facing_reply_useful", llmLas
else fail("llm_user_facing_reply_useful", llmLast.slice(0, 200));
await p.screenshot({ path: `${OUT}/06-llm-reply.png` });
// 9. Network health summary
// 9. Network health summary — distinguish system 5xx from upstream-LLM 5xx
// so the canvas-primary 503 is tracked (not hidden by aggregation).
const byStatus = {};
for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1;
const fivexx = calls.filter(c => c.status >= 500).length;
const fast502 = calls.filter(c => c.status === 502).length;
if (fast502 === 0) pass("no_502_anywhere", `5xx total=${fivexx}`);
const llmGwUrls = ["/api/v1/llm/", "/api/llm-fallback/"];
const llm5xx = calls.filter(c => c.status >= 500 && llmGwUrls.some(p => c.url.startsWith(p)));
const sys5xx = calls.filter(c => c.status >= 500 && !llmGwUrls.some(p => c.url.startsWith(p)));
console.log(` LLM gateway 5xx (tracked, handled by cascade): ${llm5xx.length}`);
console.log(` System 5xx (must be zero): ${sys5xx.length}`);
if (fast502 === 0) pass("no_502_anywhere", `5xx breakdown: llm=${llm5xx.length} sys=${sys5xx.length}`);
else fail("no_502_anywhere", `${fast502} 502s seen`);
if (sys5xx.length === 0) pass("no_system_5xx", `llm-only 5xx=${llm5xx.length} are expected (cascade handles)`);
else fail("no_system_5xx", `${sys5xx.length} non-LLM 5xx: ${JSON.stringify(sys5xx.slice(0,3))}`);
await b.close();
qa/_wave4_gaps.mjs
+12
View File
@@ -53,6 +53,18 @@ console.log("\n=== Gap 1: Real LLM-backed answer ===");
if (gatewayOk && !fallback && last.length > 30) pass("gap1_real_llm_answer", `${gatewayOk.text.match(/provider=\w+/)?.[0]} · ${last.length}ch`);
else fail("gap1_real_llm_answer", `fallback=${fallback}, gateway_ok=${!!gatewayOk}, len=${last.length}`);
await p.screenshot({ path: `${OUT}/g1-llm-answer.png`, fullPage: false });
// Oracle wave-4 action #5: regression check — if both primary and
// fallback gateways fail, the assistant must NOT produce a real reply.
// Equivalent: a real LLM reply implies at least one gateway succeeded.
// We assert the inverse by inspecting telemetry: a successful 'gap1'
// run produces exactly one "ok" telemetry line; the cascade-exhausted
// failure mode emits "all gateways exhausted". If we ever see both
// primary AND fallback-dev fail (cascade exhausted) AND a real reply,
// the test fails because the contract is broken.
const cascadeExhausted = msgs.some(m => /\[llm\] all gateways exhausted/.test(m.text));
if (!cascadeExhausted) pass("gap1_cascade_did_not_exhaust");
else fail("gap1_cascade_did_not_exhaust", "both primary and fallback-dev failed");
await p.context().close();
}