fix(agent): send_chat matcher requires single-word recipient
Bug: 'tell me a haiku about flowcharts' matched send_chat with target='me a haiku' / body='flowcharts', producing the bogus "I don't know who 'me a haiku' is" reply and silently swallowing all non-tool prompts that started with tell/ask. The LLM fallback never ran, so [llm] telemetry from wave 3 never fired in tests. Fix: tighten the matcher to require a single-word [A-Za-z]+ name between the verb and that/about/:. Drop the substring search in the directory; require an exact match. Now 'tell me a haiku about X' falls through to llmFallback as intended.
This commit is contained in:
@@ -0,0 +1,42 @@
|
|||||||
|
// Isolated LLM telemetry check — uses a non-tool prompt + waits for the
|
||||||
|
// 12s deadline so the timeout-or-error branch fires deterministically.
|
||||||
|
import { chromium } from "playwright";
|
||||||
|
|
||||||
|
const b = await chromium.launch({ headless: true });
|
||||||
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
||||||
|
const p = await ctx.newPage();
|
||||||
|
const msgs = [];
|
||||||
|
p.on("console", (m) => msgs.push({ type: m.type(), text: m.text() }));
|
||||||
|
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
||||||
|
|
||||||
|
await p.goto("https://canvas.flow-master.ai/", { waitUntil: "domcontentloaded" });
|
||||||
|
await p.waitForTimeout(2500);
|
||||||
|
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
||||||
|
await p.waitForSelector(".topbar", { timeout: 30000 });
|
||||||
|
await p.waitForTimeout(2000);
|
||||||
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||||
|
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||||
|
await p.waitForTimeout(2000);
|
||||||
|
|
||||||
|
// Three different non-tool prompts to make sure we hit the LLM fallback
|
||||||
|
const prompts = [
|
||||||
|
"tell me a haiku",
|
||||||
|
"summarize what you can do for me",
|
||||||
|
"is the sky blue today",
|
||||||
|
];
|
||||||
|
for (const q of prompts) {
|
||||||
|
const ta = p.locator(".agent-composer textarea").first();
|
||||||
|
await ta.fill(q);
|
||||||
|
await ta.press("Enter");
|
||||||
|
await p.waitForTimeout(14000); // > 12s LLM deadline
|
||||||
|
}
|
||||||
|
|
||||||
|
const llm = msgs.filter((m) => m.text.includes("[llm]"));
|
||||||
|
console.log("[llm] entries:", llm.length);
|
||||||
|
for (const e of llm) console.log(` [${e.type}] ${e.text.slice(0, 200)}`);
|
||||||
|
|
||||||
|
const replies = await p.locator(".agent-turn-body").allTextContents();
|
||||||
|
console.log("\nlast 3 replies:");
|
||||||
|
for (const r of replies.slice(-3)) console.log(" -", r.slice(0, 160));
|
||||||
|
|
||||||
|
await b.close();
|
||||||
@@ -55,11 +55,13 @@ else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flo
|
|||||||
|
|
||||||
console.log("\n=== watch-out 2: LLM telemetry shows in console ===");
|
console.log("\n=== watch-out 2: LLM telemetry shows in console ===");
|
||||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||||
await p.waitForTimeout(3000);
|
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||||
|
await p.waitForTimeout(2000);
|
||||||
const txt = p.locator(".agent-composer textarea").first();
|
const txt = p.locator(".agent-composer textarea").first();
|
||||||
await txt.fill("Briefly: what is EA2?");
|
// Genuinely non-tool prompt so routeAgentInput falls through to llmFallback.
|
||||||
|
await txt.fill("tell me a haiku about flowcharts");
|
||||||
await txt.press("Enter");
|
await txt.press("Enter");
|
||||||
await p.waitForTimeout(10000);
|
await p.waitForTimeout(14000);
|
||||||
const llmConsoleEntries = consoleMsgs.filter(m =>
|
const llmConsoleEntries = consoleMsgs.filter(m =>
|
||||||
m.type === "warning" || m.type === "info"
|
m.type === "warning" || m.type === "info"
|
||||||
).filter(m => m.text.includes("[llm]"));
|
).filter(m => m.text.includes("[llm]"));
|
||||||
|
|||||||
@@ -103,9 +103,11 @@ const TOOLS: ToolDef[] = [
|
|||||||
{
|
{
|
||||||
name: "send_chat",
|
name: "send_chat",
|
||||||
description: "Send a chat message to a teammate",
|
description: "Send a chat message to a teammate",
|
||||||
matcher: /^(message|tell|text|ask)\s+(.+?)\s+(that|about|:)\s+(.+)$/i,
|
// Recipient must be a single word (a name). Stops "tell me a haiku about X"
|
||||||
|
// from matching as message-target="me a haiku" / body="X".
|
||||||
|
matcher: /^(message|tell|text|ask)\s+([A-Za-z][A-Za-z'-]*)\s+(that|about|:)\s+(.+)$/i,
|
||||||
async run(input, ctx) {
|
async run(input, ctx) {
|
||||||
const m = input.match(/^(message|tell|text|ask)\s+(.+?)\s+(that|about|:)\s+(.+)$/i);
|
const m = input.match(/^(message|tell|text|ask)\s+([A-Za-z][A-Za-z'-]*)\s+(that|about|:)\s+(.+)$/i);
|
||||||
if (!m) return { ok: false, display: "Try: message Mariana that the laptop quote is approved" };
|
if (!m) return { ok: false, display: "Try: message Mariana that the laptop quote is approved" };
|
||||||
const target = m[2].toLowerCase();
|
const target = m[2].toLowerCase();
|
||||||
const body = m[4];
|
const body = m[4];
|
||||||
@@ -114,7 +116,7 @@ const TOOLS: ToolDef[] = [
|
|||||||
aisha: "hr-head@flow-master.ai", hr: "hr-head@flow-master.ai",
|
aisha: "hr-head@flow-master.ai", hr: "hr-head@flow-master.ai",
|
||||||
rohan: "it-head@flow-master.ai", it: "it-head@flow-master.ai",
|
rohan: "it-head@flow-master.ai", it: "it-head@flow-master.ai",
|
||||||
};
|
};
|
||||||
const recipient = Object.entries(directory).find(([k]) => target.includes(k))?.[1];
|
const recipient = directory[target];
|
||||||
if (!recipient) return { ok: false, display: `I don't know who "${target}" is. Try Mariana, Aisha, or Rohan.` };
|
if (!recipient) return { ok: false, display: `I don't know who "${target}" is. Try Mariana, Aisha, or Rohan.` };
|
||||||
const threads = await chatApi.listThreads(ctx.userEmail);
|
const threads = await chatApi.listThreads(ctx.userEmail);
|
||||||
const existing = threads.find((t) => t.participants.includes(recipient) && t.participants.includes(ctx.userEmail));
|
const existing = threads.find((t) => t.participants.includes(recipient) && t.participants.includes(ctx.userEmail));
|
||||||
|
|||||||
Reference in New Issue
Block a user