diff --git a/qa/_llm_telemetry_check.mjs b/qa/_llm_telemetry_check.mjs new file mode 100644 index 0000000..1c63e1f --- /dev/null +++ b/qa/_llm_telemetry_check.mjs @@ -0,0 +1,42 @@ +// Isolated LLM telemetry check — uses a non-tool prompt + waits for the +// 12s deadline so the timeout-or-error branch fires deterministically. +import { chromium } from "playwright"; + +const b = await chromium.launch({ headless: true }); +const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); +const p = await ctx.newPage(); +const msgs = []; +p.on("console", (m) => msgs.push({ type: m.type(), text: m.text() })); +await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now()))); + +await p.goto("https://canvas.flow-master.ai/", { waitUntil: "domcontentloaded" }); +await p.waitForTimeout(2500); +await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click(); +await p.waitForSelector(".topbar", { timeout: 30000 }); +await p.waitForTimeout(2000); +await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); +await p.waitForSelector(".agent-composer textarea", { timeout: 15000 }); +await p.waitForTimeout(2000); + +// Three different non-tool prompts to make sure we hit the LLM fallback +const prompts = [ + "tell me a haiku", + "summarize what you can do for me", + "is the sky blue today", +]; +for (const q of prompts) { + const ta = p.locator(".agent-composer textarea").first(); + await ta.fill(q); + await ta.press("Enter"); + await p.waitForTimeout(14000); // > 12s LLM deadline +} + +const llm = msgs.filter((m) => m.text.includes("[llm]")); +console.log("[llm] entries:", llm.length); +for (const e of llm) console.log(` [${e.type}] ${e.text.slice(0, 200)}`); + +const replies = await p.locator(".agent-turn-body").allTextContents(); +console.log("\nlast 3 replies:"); +for (const r of replies.slice(-3)) console.log(" -", r.slice(0, 160)); + +await b.close(); diff --git a/qa/_wave3_verify.mjs b/qa/_wave3_verify.mjs index 9aa5646..22b60d3 100644 --- a/qa/_wave3_verify.mjs +++ b/qa/_wave3_verify.mjs @@ -55,11 +55,13 @@ else fail("wave3_studio_draft_lands_via_split_post", `posts=${JSON.stringify(flo console.log("\n=== watch-out 2: LLM telemetry shows in console ==="); await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click(); -await p.waitForTimeout(3000); +await p.waitForSelector(".agent-composer textarea", { timeout: 15000 }); +await p.waitForTimeout(2000); const txt = p.locator(".agent-composer textarea").first(); -await txt.fill("Briefly: what is EA2?"); +// Genuinely non-tool prompt so routeAgentInput falls through to llmFallback. +await txt.fill("tell me a haiku about flowcharts"); await txt.press("Enter"); -await p.waitForTimeout(10000); +await p.waitForTimeout(14000); const llmConsoleEntries = consoleMsgs.filter(m => m.type === "warning" || m.type === "info" ).filter(m => m.text.includes("[llm]")); diff --git a/src/lib/agentTools.ts b/src/lib/agentTools.ts index 3597604..c374914 100644 --- a/src/lib/agentTools.ts +++ b/src/lib/agentTools.ts @@ -103,9 +103,11 @@ const TOOLS: ToolDef[] = [ { name: "send_chat", description: "Send a chat message to a teammate", - matcher: /^(message|tell|text|ask)\s+(.+?)\s+(that|about|:)\s+(.+)$/i, + // Recipient must be a single word (a name). Stops "tell me a haiku about X" + // from matching as message-target="me a haiku" / body="X". + matcher: /^(message|tell|text|ask)\s+([A-Za-z][A-Za-z'-]*)\s+(that|about|:)\s+(.+)$/i, async run(input, ctx) { - const m = input.match(/^(message|tell|text|ask)\s+(.+?)\s+(that|about|:)\s+(.+)$/i); + const m = input.match(/^(message|tell|text|ask)\s+([A-Za-z][A-Za-z'-]*)\s+(that|about|:)\s+(.+)$/i); if (!m) return { ok: false, display: "Try: message Mariana that the laptop quote is approved" }; const target = m[2].toLowerCase(); const body = m[4]; @@ -114,7 +116,7 @@ const TOOLS: ToolDef[] = [ aisha: "hr-head@flow-master.ai", hr: "hr-head@flow-master.ai", rohan: "it-head@flow-master.ai", it: "it-head@flow-master.ai", }; - const recipient = Object.entries(directory).find(([k]) => target.includes(k))?.[1]; + const recipient = directory[target]; if (!recipient) return { ok: false, display: `I don't know who "${target}" is. Try Mariana, Aisha, or Rohan.` }; const threads = await chatApi.listThreads(ctx.userEmail); const existing = threads.find((t) => t.participants.includes(recipient) && t.participants.includes(ctx.userEmail));