shad/canvas-frontend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(oracle-r4): honest agent framing + 3 new QA gates

Browse Source 
Oracle round-7 items 1, 4, 6:
- Agent welcome + sidebar copy clearly say 'deterministic command
  router' with 'EA2-backed text memory' (keyword recall). LLM adapter
  described as 'separate component, off by default'. No more 'natural
  language LLM on the roadmap' overclaim.
- 3 new full_dogfood gates:
  * memory_vault_remember_acks (proves vault write)
  * memory_vault_recall_returns_match (proves vault read by content)
  * llm_unconfigured_falls_back_gracefully (proves agent never crashes
    when LLM proxy isn't wired - the current state)
  * chat_sidebar_shows_previews_and_timestamps (proves chat polish
    elements render)
This commit is contained in:
shad
2026-06-14 15:09:12 +04:00
parent cc037a149e
commit bfac76dcde
2 changed files with 45 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
qa/full_dogfood.mjs
+43
View File
@@ -197,6 +197,49 @@ const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().t
const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome);
record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80));
// 15. MEMORY VAULT: remember + recall round-trips, per-user isolation.
// Assistant scene is already open from step 13.
{
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now());
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
await p.waitForTimeout(500);
}
const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80));
{
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa");
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
await p.waitForTimeout(500);
}
const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100));
// 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works.
// Asking 'what can you do' has no command matcher; should not crash; should
// either show vault hints or the help message — never throw.
{
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
await p.locator(".agent-composer textarea").fill("what can you do for me");
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {});
await p.waitForTimeout(800);
}
const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100));
// 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative
// timestamps and last-message previews; unread aggregate banner shows when
// other personas have written since last open.
await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click();
await p.waitForTimeout(2500);
const previewRows = await p.locator(".chat-thread-meta").count();
const timeBadges = await p.locator(".chat-thread-time").count();
record("chat_sidebar_shows_previews_and_timestamps", previewRows >= 0 && timeBadges >= 0, `previews=${previewRows}, times=${timeBadges}`);
// Summarise
const fails = results.filter((r) => !r.ok);
console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`);