diff --git a/qa/_oracle_e2e.mjs b/qa/_oracle_e2e.mjs index 4c25564..13b5ffe 100644 --- a/qa/_oracle_e2e.mjs +++ b/qa/_oracle_e2e.mjs @@ -137,13 +137,20 @@ if (hasFallbackMenu || hasRealReply) pass("llm_user_facing_reply_useful", llmLas else fail("llm_user_facing_reply_useful", llmLast.slice(0, 200)); await p.screenshot({ path: `${OUT}/06-llm-reply.png` }); -// 9. Network health summary +// 9. Network health summary — distinguish system 5xx from upstream-LLM 5xx +// so the canvas-primary 503 is tracked (not hidden by aggregation). const byStatus = {}; for (const c of calls) byStatus[c.status] = (byStatus[c.status] || 0) + 1; -const fivexx = calls.filter(c => c.status >= 500).length; const fast502 = calls.filter(c => c.status === 502).length; -if (fast502 === 0) pass("no_502_anywhere", `5xx total=${fivexx}`); +const llmGwUrls = ["/api/v1/llm/", "/api/llm-fallback/"]; +const llm5xx = calls.filter(c => c.status >= 500 && llmGwUrls.some(p => c.url.startsWith(p))); +const sys5xx = calls.filter(c => c.status >= 500 && !llmGwUrls.some(p => c.url.startsWith(p))); +console.log(` LLM gateway 5xx (tracked, handled by cascade): ${llm5xx.length}`); +console.log(` System 5xx (must be zero): ${sys5xx.length}`); +if (fast502 === 0) pass("no_502_anywhere", `5xx breakdown: llm=${llm5xx.length} sys=${sys5xx.length}`); else fail("no_502_anywhere", `${fast502} 502s seen`); +if (sys5xx.length === 0) pass("no_system_5xx", `llm-only 5xx=${llm5xx.length} are expected (cascade handles)`); +else fail("no_system_5xx", `${sys5xx.length} non-LLM 5xx: ${JSON.stringify(sys5xx.slice(0,3))}`); await b.close(); diff --git a/qa/_wave4_gaps.mjs b/qa/_wave4_gaps.mjs index 11f1c76..f0ac134 100644 --- a/qa/_wave4_gaps.mjs +++ b/qa/_wave4_gaps.mjs @@ -53,6 +53,18 @@ console.log("\n=== Gap 1: Real LLM-backed answer ==="); if (gatewayOk && !fallback && last.length > 30) pass("gap1_real_llm_answer", `${gatewayOk.text.match(/provider=\w+/)?.[0]} · ${last.length}ch`); else fail("gap1_real_llm_answer", `fallback=${fallback}, gateway_ok=${!!gatewayOk}, len=${last.length}`); await p.screenshot({ path: `${OUT}/g1-llm-answer.png`, fullPage: false }); + + // Oracle wave-4 action #5: regression check — if both primary and + // fallback gateways fail, the assistant must NOT produce a real reply. + // Equivalent: a real LLM reply implies at least one gateway succeeded. + // We assert the inverse by inspecting telemetry: a successful 'gap1' + // run produces exactly one "ok" telemetry line; the cascade-exhausted + // failure mode emits "all gateways exhausted". If we ever see both + // primary AND fallback-dev fail (cascade exhausted) AND a real reply, + // the test fails because the contract is broken. + const cascadeExhausted = msgs.some(m => /\[llm\] all gateways exhausted/.test(m.text)); + if (!cascadeExhausted) pass("gap1_cascade_did_not_exhaust"); + else fail("gap1_cascade_did_not_exhaust", "both primary and fallback-dev failed"); await p.context().close(); }