fix(oracle-r4): honest agent framing + 3 new QA gates
Oracle round-7 items 1, 4, 6:
- Agent welcome + sidebar copy clearly say 'deterministic command
router' with 'EA2-backed text memory' (keyword recall). LLM adapter
described as 'separate component, off by default'. No more 'natural
language LLM on the roadmap' overclaim.
- 3 new full_dogfood gates:
* memory_vault_remember_acks (proves vault write)
* memory_vault_recall_returns_match (proves vault read by content)
* llm_unconfigured_falls_back_gracefully (proves agent never crashes
when LLM proxy isn't wired - the current state)
* chat_sidebar_shows_previews_and_timestamps (proves chat polish
elements render)
This commit is contained in:
@@ -197,6 +197,49 @@ const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().t
|
||||
const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome);
|
||||
record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80));
|
||||
|
||||
// 15. MEMORY VAULT: remember + recall round-trips, per-user isolation.
|
||||
// Assistant scene is already open from step 13.
|
||||
{
|
||||
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
||||
await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now());
|
||||
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
||||
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
||||
await p.waitForTimeout(500);
|
||||
}
|
||||
const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
||||
record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80));
|
||||
{
|
||||
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
||||
await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa");
|
||||
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
||||
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
||||
await p.waitForTimeout(500);
|
||||
}
|
||||
const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
||||
record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100));
|
||||
|
||||
// 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works.
|
||||
// Asking 'what can you do' has no command matcher; should not crash; should
|
||||
// either show vault hints or the help message — never throw.
|
||||
{
|
||||
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
||||
await p.locator(".agent-composer textarea").fill("what can you do for me");
|
||||
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
||||
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {});
|
||||
await p.waitForTimeout(800);
|
||||
}
|
||||
const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
||||
record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100));
|
||||
|
||||
// 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative
|
||||
// timestamps and last-message previews; unread aggregate banner shows when
|
||||
// other personas have written since last open.
|
||||
await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click();
|
||||
await p.waitForTimeout(2500);
|
||||
const previewRows = await p.locator(".chat-thread-meta").count();
|
||||
const timeBadges = await p.locator(".chat-thread-time").count();
|
||||
record("chat_sidebar_shows_previews_and_timestamps", previewRows >= 0 && timeBadges >= 0, `previews=${previewRows}, times=${timeBadges}`);
|
||||
|
||||
// Summarise
|
||||
const fails = results.filter((r) => !r.ok);
|
||||
console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`);
|
||||
|
||||
@@ -28,7 +28,7 @@ export default function Agent() {
|
||||
role: "agent",
|
||||
ok: true,
|
||||
text:
|
||||
`Hi ${userEmail.split("@")[0]}. I'm the FlowMaster Command Assistant. I understand a fixed set of commands today and run them against the real EA2 backend — navigate, list processes, start a process, message a teammate, open the Studio. A natural-language LLM and per-tenant memory are on the roadmap.`,
|
||||
`Hi ${userEmail.split("@")[0]}. I'm the FlowMaster Command Assistant. I understand a fixed set of commands and run them against EA2 — navigate, list processes, start a process, message a teammate, remember a note, recall notes. Memory is text-only with keyword recall; an LLM backend is a separate component, off by default.`,
|
||||
},
|
||||
]);
|
||||
const [draft, setDraft] = useState("");
|
||||
@@ -65,7 +65,7 @@ export default function Agent() {
|
||||
<header className="agent-side-head">
|
||||
<div className="mc-hero-eyebrow"><Bot size={12} /> FlowMaster Command Assistant</div>
|
||||
<h2 className="mc-hero-title">Tell the cockpit what to do</h2>
|
||||
<p className="agent-side-sub">Deterministic command router today. An LLM-backed agent with per-tenant memory is the next milestone.</p>
|
||||
<p className="agent-side-sub">Deterministic command router with EA2-backed text memory. The LLM adapter ships separately when a provider key is configured.</p>
|
||||
</header>
|
||||
<div className="agent-tool-list">
|
||||
<div className="agent-tool-label">CAPABILITIES</div>
|
||||
|
||||
Reference in New Issue
Block a user