canvas-bot
3ebfaf81ae
Bug: 'tell me a haiku about flowcharts' matched send_chat with target='me a haiku' / body='flowcharts', producing the bogus "I don't know who 'me a haiku' is" reply and silently swallowing all non-tool prompts that started with tell/ask. The LLM fallback never ran, so [llm] telemetry from wave 3 never fired in tests. Fix: tighten the matcher to require a single-word [A-Za-z]+ name between the verb and that/about/:. Drop the substring search in the directory; require an exact match. Now 'tell me a haiku about X' falls through to llmFallback as intended.
43 lines
1.7 KiB
JavaScript
43 lines
1.7 KiB
JavaScript
// Isolated LLM telemetry check — uses a non-tool prompt + waits for the
|
|
// 12s deadline so the timeout-or-error branch fires deterministically.
|
|
import { chromium } from "playwright";
|
|
|
|
const b = await chromium.launch({ headless: true });
|
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
|
const p = await ctx.newPage();
|
|
const msgs = [];
|
|
p.on("console", (m) => msgs.push({ type: m.type(), text: m.text() }));
|
|
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
|
|
|
await p.goto("https://canvas.flow-master.ai/", { waitUntil: "domcontentloaded" });
|
|
await p.waitForTimeout(2500);
|
|
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
|
await p.waitForSelector(".topbar", { timeout: 30000 });
|
|
await p.waitForTimeout(2000);
|
|
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
|
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
|
await p.waitForTimeout(2000);
|
|
|
|
// Three different non-tool prompts to make sure we hit the LLM fallback
|
|
const prompts = [
|
|
"tell me a haiku",
|
|
"summarize what you can do for me",
|
|
"is the sky blue today",
|
|
];
|
|
for (const q of prompts) {
|
|
const ta = p.locator(".agent-composer textarea").first();
|
|
await ta.fill(q);
|
|
await ta.press("Enter");
|
|
await p.waitForTimeout(14000); // > 12s LLM deadline
|
|
}
|
|
|
|
const llm = msgs.filter((m) => m.text.includes("[llm]"));
|
|
console.log("[llm] entries:", llm.length);
|
|
for (const e of llm) console.log(` [${e.type}] ${e.text.slice(0, 200)}`);
|
|
|
|
const replies = await p.locator(".agent-turn-body").allTextContents();
|
|
console.log("\nlast 3 replies:");
|
|
for (const r of replies.slice(-3)) console.log(" -", r.slice(0, 160));
|
|
|
|
await b.close();
|