fix(agent): send_chat matcher requires single-word recipient
Bug: 'tell me a haiku about flowcharts' matched send_chat with target='me a haiku' / body='flowcharts', producing the bogus "I don't know who 'me a haiku' is" reply and silently swallowing all non-tool prompts that started with tell/ask. The LLM fallback never ran, so [llm] telemetry from wave 3 never fired in tests. Fix: tighten the matcher to require a single-word [A-Za-z]+ name between the verb and that/about/:. Drop the substring search in the directory; require an exact match. Now 'tell me a haiku about X' falls through to llmFallback as intended.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
// Isolated LLM telemetry check — uses a non-tool prompt + waits for the
|
||||
// 12s deadline so the timeout-or-error branch fires deterministically.
|
||||
import { chromium } from "playwright";
|
||||
|
||||
const b = await chromium.launch({ headless: true });
|
||||
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
||||
const p = await ctx.newPage();
|
||||
const msgs = [];
|
||||
p.on("console", (m) => msgs.push({ type: m.type(), text: m.text() }));
|
||||
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
||||
|
||||
await p.goto("https://canvas.flow-master.ai/", { waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
||||
await p.waitForSelector(".topbar", { timeout: 30000 });
|
||||
await p.waitForTimeout(2000);
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||
await p.waitForTimeout(2000);
|
||||
|
||||
// Three different non-tool prompts to make sure we hit the LLM fallback
|
||||
const prompts = [
|
||||
"tell me a haiku",
|
||||
"summarize what you can do for me",
|
||||
"is the sky blue today",
|
||||
];
|
||||
for (const q of prompts) {
|
||||
const ta = p.locator(".agent-composer textarea").first();
|
||||
await ta.fill(q);
|
||||
await ta.press("Enter");
|
||||
await p.waitForTimeout(14000); // > 12s LLM deadline
|
||||
}
|
||||
|
||||
const llm = msgs.filter((m) => m.text.includes("[llm]"));
|
||||
console.log("[llm] entries:", llm.length);
|
||||
for (const e of llm) console.log(` [${e.type}] ${e.text.slice(0, 200)}`);
|
||||
|
||||
const replies = await p.locator(".agent-turn-body").allTextContents();
|
||||
console.log("\nlast 3 replies:");
|
||||
for (const r of replies.slice(-3)) console.log(" -", r.slice(0, 160));
|
||||
|
||||
await b.close();
|
||||
@@ -55,11 +55,13 @@ else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flo
|
||||
|
||||
console.log("\n=== watch-out 2: LLM telemetry shows in console ===");
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForTimeout(3000);
|
||||
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||
await p.waitForTimeout(2000);
|
||||
const txt = p.locator(".agent-composer textarea").first();
|
||||
await txt.fill("Briefly: what is EA2?");
|
||||
// Genuinely non-tool prompt so routeAgentInput falls through to llmFallback.
|
||||
await txt.fill("tell me a haiku about flowcharts");
|
||||
await txt.press("Enter");
|
||||
await p.waitForTimeout(10000);
|
||||
await p.waitForTimeout(14000);
|
||||
const llmConsoleEntries = consoleMsgs.filter(m =>
|
||||
m.type === "warning" || m.type === "info"
|
||||
).filter(m => m.text.includes("[llm]"));
|
||||
|
||||
@@ -103,9 +103,11 @@ const TOOLS: ToolDef[] = [
|
||||
{
|
||||
name: "send_chat",
|
||||
description: "Send a chat message to a teammate",
|
||||
matcher: /^(message|tell|text|ask)\s+(.+?)\s+(that|about|:)\s+(.+)$/i,
|
||||
// Recipient must be a single word (a name). Stops "tell me a haiku about X"
|
||||
// from matching as message-target="me a haiku" / body="X".
|
||||
matcher: /^(message|tell|text|ask)\s+([A-Za-z][A-Za-z'-]*)\s+(that|about|:)\s+(.+)$/i,
|
||||
async run(input, ctx) {
|
||||
const m = input.match(/^(message|tell|text|ask)\s+(.+?)\s+(that|about|:)\s+(.+)$/i);
|
||||
const m = input.match(/^(message|tell|text|ask)\s+([A-Za-z][A-Za-z'-]*)\s+(that|about|:)\s+(.+)$/i);
|
||||
if (!m) return { ok: false, display: "Try: message Mariana that the laptop quote is approved" };
|
||||
const target = m[2].toLowerCase();
|
||||
const body = m[4];
|
||||
@@ -114,7 +116,7 @@ const TOOLS: ToolDef[] = [
|
||||
aisha: "hr-head@flow-master.ai", hr: "hr-head@flow-master.ai",
|
||||
rohan: "it-head@flow-master.ai", it: "it-head@flow-master.ai",
|
||||
};
|
||||
const recipient = Object.entries(directory).find(([k]) => target.includes(k))?.[1];
|
||||
const recipient = directory[target];
|
||||
if (!recipient) return { ok: false, display: `I don't know who "${target}" is. Try Mariana, Aisha, or Rohan.` };
|
||||
const threads = await chatApi.listThreads(ctx.userEmail);
|
||||
const existing = threads.find((t) => t.participants.includes(recipient) && t.participants.includes(ctx.userEmail));
|
||||
|
||||
Reference in New Issue
Block a user