fix(agent): send_chat matcher requires single-word recipient
Bug: 'tell me a haiku about flowcharts' matched send_chat with target='me a haiku' / body='flowcharts', producing the bogus "I don't know who 'me a haiku' is" reply and silently swallowing all non-tool prompts that started with tell/ask. The LLM fallback never ran, so [llm] telemetry from wave 3 never fired in tests. Fix: tighten the matcher to require a single-word [A-Za-z]+ name between the verb and that/about/:. Drop the substring search in the directory; require an exact match. Now 'tell me a haiku about X' falls through to llmFallback as intended.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
// Isolated LLM telemetry check — uses a non-tool prompt + waits for the
|
||||
// 12s deadline so the timeout-or-error branch fires deterministically.
|
||||
import { chromium } from "playwright";
|
||||
|
||||
const b = await chromium.launch({ headless: true });
|
||||
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
||||
const p = await ctx.newPage();
|
||||
const msgs = [];
|
||||
p.on("console", (m) => msgs.push({ type: m.type(), text: m.text() }));
|
||||
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
||||
|
||||
await p.goto("https://canvas.flow-master.ai/", { waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
||||
await p.waitForSelector(".topbar", { timeout: 30000 });
|
||||
await p.waitForTimeout(2000);
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||
await p.waitForTimeout(2000);
|
||||
|
||||
// Three different non-tool prompts to make sure we hit the LLM fallback
|
||||
const prompts = [
|
||||
"tell me a haiku",
|
||||
"summarize what you can do for me",
|
||||
"is the sky blue today",
|
||||
];
|
||||
for (const q of prompts) {
|
||||
const ta = p.locator(".agent-composer textarea").first();
|
||||
await ta.fill(q);
|
||||
await ta.press("Enter");
|
||||
await p.waitForTimeout(14000); // > 12s LLM deadline
|
||||
}
|
||||
|
||||
const llm = msgs.filter((m) => m.text.includes("[llm]"));
|
||||
console.log("[llm] entries:", llm.length);
|
||||
for (const e of llm) console.log(` [${e.type}] ${e.text.slice(0, 200)}`);
|
||||
|
||||
const replies = await p.locator(".agent-turn-body").allTextContents();
|
||||
console.log("\nlast 3 replies:");
|
||||
for (const r of replies.slice(-3)) console.log(" -", r.slice(0, 160));
|
||||
|
||||
await b.close();
|
||||
@@ -55,11 +55,13 @@ else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flo
|
||||
|
||||
console.log("\n=== watch-out 2: LLM telemetry shows in console ===");
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForTimeout(3000);
|
||||
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||
await p.waitForTimeout(2000);
|
||||
const txt = p.locator(".agent-composer textarea").first();
|
||||
await txt.fill("Briefly: what is EA2?");
|
||||
// Genuinely non-tool prompt so routeAgentInput falls through to llmFallback.
|
||||
await txt.fill("tell me a haiku about flowcharts");
|
||||
await txt.press("Enter");
|
||||
await p.waitForTimeout(10000);
|
||||
await p.waitForTimeout(14000);
|
||||
const llmConsoleEntries = consoleMsgs.filter(m =>
|
||||
m.type === "warning" || m.type === "info"
|
||||
).filter(m => m.text.includes("[llm]"));
|
||||
|
||||
Reference in New Issue
Block a user