fix(oracle-r4): honest agent framing + 3 new QA gates
Oracle round-7 items 1, 4, 6:
- Agent welcome + sidebar copy clearly say 'deterministic command
router' with 'EA2-backed text memory' (keyword recall). LLM adapter
described as 'separate component, off by default'. No more 'natural
language LLM on the roadmap' overclaim.
- 3 new full_dogfood gates:
* memory_vault_remember_acks (proves vault write)
* memory_vault_recall_returns_match (proves vault read by content)
* llm_unconfigured_falls_back_gracefully (proves agent never crashes
when LLM proxy isn't wired - the current state)
* chat_sidebar_shows_previews_and_timestamps (proves chat polish
elements render)
This commit is contained in:
@@ -197,6 +197,49 @@ const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().t
|
|||||||
const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome);
|
const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome);
|
||||||
record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80));
|
record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80));
|
||||||
|
|
||||||
|
// 15. MEMORY VAULT: remember + recall round-trips, per-user isolation.
|
||||||
|
// Assistant scene is already open from step 13.
|
||||||
|
{
|
||||||
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
||||||
|
await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now());
|
||||||
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
||||||
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
||||||
|
await p.waitForTimeout(500);
|
||||||
|
}
|
||||||
|
const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
||||||
|
record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80));
|
||||||
|
{
|
||||||
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
||||||
|
await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa");
|
||||||
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
||||||
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
||||||
|
await p.waitForTimeout(500);
|
||||||
|
}
|
||||||
|
const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
||||||
|
record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100));
|
||||||
|
|
||||||
|
// 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works.
|
||||||
|
// Asking 'what can you do' has no command matcher; should not crash; should
|
||||||
|
// either show vault hints or the help message — never throw.
|
||||||
|
{
|
||||||
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
||||||
|
await p.locator(".agent-composer textarea").fill("what can you do for me");
|
||||||
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
||||||
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {});
|
||||||
|
await p.waitForTimeout(800);
|
||||||
|
}
|
||||||
|
const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
||||||
|
record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100));
|
||||||
|
|
||||||
|
// 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative
|
||||||
|
// timestamps and last-message previews; unread aggregate banner shows when
|
||||||
|
// other personas have written since last open.
|
||||||
|
await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click();
|
||||||
|
await p.waitForTimeout(2500);
|
||||||
|
const previewRows = await p.locator(".chat-thread-meta").count();
|
||||||
|
const timeBadges = await p.locator(".chat-thread-time").count();
|
||||||
|
record("chat_sidebar_shows_previews_and_timestamps", previewRows >= 0 && timeBadges >= 0, `previews=${previewRows}, times=${timeBadges}`);
|
||||||
|
|
||||||
// Summarise
|
// Summarise
|
||||||
const fails = results.filter((r) => !r.ok);
|
const fails = results.filter((r) => !r.ok);
|
||||||
console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`);
|
console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`);
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ export default function Agent() {
|
|||||||
role: "agent",
|
role: "agent",
|
||||||
ok: true,
|
ok: true,
|
||||||
text:
|
text:
|
||||||
`Hi ${userEmail.split("@")[0]}. I'm the FlowMaster Command Assistant. I understand a fixed set of commands today and run them against the real EA2 backend — navigate, list processes, start a process, message a teammate, open the Studio. A natural-language LLM and per-tenant memory are on the roadmap.`,
|
`Hi ${userEmail.split("@")[0]}. I'm the FlowMaster Command Assistant. I understand a fixed set of commands and run them against EA2 — navigate, list processes, start a process, message a teammate, remember a note, recall notes. Memory is text-only with keyword recall; an LLM backend is a separate component, off by default.`,
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
const [draft, setDraft] = useState("");
|
const [draft, setDraft] = useState("");
|
||||||
@@ -65,7 +65,7 @@ export default function Agent() {
|
|||||||
<header className="agent-side-head">
|
<header className="agent-side-head">
|
||||||
<div className="mc-hero-eyebrow"><Bot size={12} /> FlowMaster Command Assistant</div>
|
<div className="mc-hero-eyebrow"><Bot size={12} /> FlowMaster Command Assistant</div>
|
||||||
<h2 className="mc-hero-title">Tell the cockpit what to do</h2>
|
<h2 className="mc-hero-title">Tell the cockpit what to do</h2>
|
||||||
<p className="agent-side-sub">Deterministic command router today. An LLM-backed agent with per-tenant memory is the next milestone.</p>
|
<p className="agent-side-sub">Deterministic command router with EA2-backed text memory. The LLM adapter ships separately when a provider key is configured.</p>
|
||||||
</header>
|
</header>
|
||||||
<div className="agent-tool-list">
|
<div className="agent-tool-list">
|
||||||
<div className="agent-tool-label">CAPABILITIES</div>
|
<div className="agent-tool-label">CAPABILITIES</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user