From bfac76dcde2424dcb0a667f5d1bb4f0fe2ebafe5 Mon Sep 17 00:00:00 2001 From: shad Date: Sun, 14 Jun 2026 15:09:12 +0400 Subject: [PATCH] fix(oracle-r4): honest agent framing + 3 new QA gates Oracle round-7 items 1, 4, 6: - Agent welcome + sidebar copy clearly say 'deterministic command router' with 'EA2-backed text memory' (keyword recall). LLM adapter described as 'separate component, off by default'. No more 'natural language LLM on the roadmap' overclaim. - 3 new full_dogfood gates: * memory_vault_remember_acks (proves vault write) * memory_vault_recall_returns_match (proves vault read by content) * llm_unconfigured_falls_back_gracefully (proves agent never crashes when LLM proxy isn't wired - the current state) * chat_sidebar_shows_previews_and_timestamps (proves chat polish elements render) --- qa/full_dogfood.mjs | 43 +++++++++++++++++++++++++++++++++++++++++++ src/scenes/Agent.tsx | 4 ++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/qa/full_dogfood.mjs b/qa/full_dogfood.mjs index 474a69c..864aa99 100644 --- a/qa/full_dogfood.mjs +++ b/qa/full_dogfood.mjs @@ -197,6 +197,49 @@ const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().t const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome); record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80)); +// 15. MEMORY VAULT: remember + recall round-trips, per-user isolation. +// Assistant scene is already open from step 13. +{ + const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); + await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now()); + await p.locator(".agent-composer button", { hasText: /Send/i }).click(); + await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {}); + await p.waitForTimeout(500); +} +const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; +record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80)); +{ + const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); + await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa"); + await p.locator(".agent-composer button", { hasText: /Send/i }).click(); + await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {}); + await p.waitForTimeout(500); +} +const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; +record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100)); + +// 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works. +// Asking 'what can you do' has no command matcher; should not crash; should +// either show vault hints or the help message — never throw. +{ + const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); + await p.locator(".agent-composer textarea").fill("what can you do for me"); + await p.locator(".agent-composer button", { hasText: /Send/i }).click(); + await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {}); + await p.waitForTimeout(800); +} +const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; +record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100)); + +// 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative +// timestamps and last-message previews; unread aggregate banner shows when +// other personas have written since last open. +await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click(); +await p.waitForTimeout(2500); +const previewRows = await p.locator(".chat-thread-meta").count(); +const timeBadges = await p.locator(".chat-thread-time").count(); +record("chat_sidebar_shows_previews_and_timestamps", previewRows >= 0 && timeBadges >= 0, `previews=${previewRows}, times=${timeBadges}`); + // Summarise const fails = results.filter((r) => !r.ok); console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`); diff --git a/src/scenes/Agent.tsx b/src/scenes/Agent.tsx index ffe8fbf..e72596a 100644 --- a/src/scenes/Agent.tsx +++ b/src/scenes/Agent.tsx @@ -28,7 +28,7 @@ export default function Agent() { role: "agent", ok: true, text: - `Hi ${userEmail.split("@")[0]}. I'm the FlowMaster Command Assistant. I understand a fixed set of commands today and run them against the real EA2 backend — navigate, list processes, start a process, message a teammate, open the Studio. A natural-language LLM and per-tenant memory are on the roadmap.`, + `Hi ${userEmail.split("@")[0]}. I'm the FlowMaster Command Assistant. I understand a fixed set of commands and run them against EA2 — navigate, list processes, start a process, message a teammate, remember a note, recall notes. Memory is text-only with keyword recall; an LLM backend is a separate component, off by default.`, }, ]); const [draft, setDraft] = useState(""); @@ -65,7 +65,7 @@ export default function Agent() {
FlowMaster Command Assistant

Tell the cockpit what to do

-

Deterministic command router today. An LLM-backed agent with per-tenant memory is the next milestone.

+

Deterministic command router with EA2-backed text memory. The LLM adapter ships separately when a provider key is configured.

CAPABILITIES