fix(llm): tunnel fallback through nginx for same-origin (no CORS)
Browser trace showed fallback-dev cascade failing with 'Failed to fetch' on the live site. dev.flow-master.ai OPTIONS preflight returns HTTP 400 from foreign origins and the POST response lacks Access-Control-Allow-Origin, so the browser blocks direct cross-origin calls. Add nginx location /api/llm-fallback/ that rewrites to /api/v1/llm/ and proxies to dev.flow-master.ai (same DNS resolver pattern as the demo proxy). Frontend now hits both gateways same-origin via canvas.flow-master.ai, no CORS preflight needed. The canvas-issued JWT is forwarded as-is and dev accepts it (verified by curl).
This commit is contained in:
canvas-bot
@@ -0,0 +1,163 @@
|
||||
// Wave 4: Oracle's gaps — real LLM answer, multi-persona, perf timings, visual proof.
|
||||
import { chromium } from "playwright";
|
||||
import { writeFileSync, mkdirSync } from "fs";
|
||||
|
||||
const URL = "https://canvas.flow-master.ai/";
|
||||
const OUT = "/tmp/canvas-evidence/wave4";
|
||||
mkdirSync(OUT, { recursive: true });
|
||||
|
||||
const r = [];
|
||||
const pass = (n, d = "") => { r.push({ n, ok: true, d }); console.log(`PASS ${n}${d ? " " + d : ""}`); };
|
||||
const fail = (n, d = "") => { r.push({ n, ok: false, d }); console.log(`FAIL ${n}${d ? " " + d : ""}`); };
|
||||
|
||||
const b = await chromium.launch({ headless: true });
|
||||
|
||||
async function login(personaName) {
|
||||
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
||||
const p = await ctx.newPage();
|
||||
const msgs = [];
|
||||
const calls = [];
|
||||
p.on("console", m => msgs.push({ type: m.type(), text: m.text().slice(0, 240) }));
|
||||
p.on("response", res => {
|
||||
if (res.url().includes("/api/")) calls.push({ status: res.status(), method: res.request().method(), url: res.url().replace(/https?:\/\/[^/]+/, "") });
|
||||
});
|
||||
await ctx.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
||||
const t0 = Date.now();
|
||||
await p.goto(URL, { waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".persona-chip").filter({ hasText: new RegExp(personaName) }).click();
|
||||
await p.waitForSelector(".topbar", { timeout: 30000 });
|
||||
return { ctx, p, msgs, calls, loginMs: Date.now() - t0 };
|
||||
}
|
||||
|
||||
// ============= GAP 1: Real LLM-backed answer on canvas =============
|
||||
console.log("\n=== Gap 1: Real LLM-backed answer ===");
|
||||
{
|
||||
const { p, msgs, calls } = await login("Mariana");
|
||||
await p.waitForTimeout(2000);
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||
await p.waitForTimeout(2000);
|
||||
const t0 = Date.now();
|
||||
await p.locator(".agent-composer textarea").first().fill("In one sentence, what does FlowMaster do?");
|
||||
await p.locator(".agent-composer textarea").first().press("Enter");
|
||||
await p.waitForTimeout(15000);
|
||||
const elapsed = Date.now() - t0;
|
||||
const replies = await p.locator(".agent-turn-body").allTextContents();
|
||||
const last = replies.slice(-1)[0] || "";
|
||||
console.log(` elapsed: ${elapsed}ms`);
|
||||
console.log(` last reply: ${last.slice(0, 300)}`);
|
||||
const fallback = /trouble|temporarily|fallback|can do for you immediately/i.test(last);
|
||||
const gatewayOk = msgs.find(m => /\[llm\] (primary|fallback-dev) ok in \d+ms · provider=/.test(m.text));
|
||||
if (gatewayOk) console.log(` telemetry: ${gatewayOk.text.slice(0, 200)}`);
|
||||
if (gatewayOk && !fallback && last.length > 30) pass("gap1_real_llm_answer", `${gatewayOk.text.match(/provider=\w+/)?.[0]} · ${last.length}ch`);
|
||||
else fail("gap1_real_llm_answer", `fallback=${fallback}, gateway_ok=${!!gatewayOk}, len=${last.length}`);
|
||||
await p.screenshot({ path: `${OUT}/g1-llm-answer.png`, fullPage: false });
|
||||
await p.context().close();
|
||||
}
|
||||
|
||||
// ============= GAP 2: Multi-persona coverage =============
|
||||
console.log("\n=== Gap 2: Multi-persona coverage (Aisha + Rohan) ===");
|
||||
for (const persona of ["Aisha", "Rohan"]) {
|
||||
console.log(` ${persona}:`);
|
||||
const { p, calls } = await login(persona);
|
||||
await p.waitForTimeout(2500);
|
||||
const tabs = await p.locator(".tab").allTextContents();
|
||||
const has6 = ["Mission", "Approvals", "Runs", "Studio", "Chat", "Assistant"].every(t => tabs.some(x => x.includes(t)));
|
||||
if (has6) pass(`gap2_${persona.toLowerCase()}_topbar_6_tabs`);
|
||||
else fail(`gap2_${persona.toLowerCase()}_topbar_6_tabs`, `tabs=${tabs.join("|")}`);
|
||||
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
||||
await p.waitForTimeout(2000);
|
||||
await p.evaluate(() => localStorage.removeItem("fm.mc.wizard.draft"));
|
||||
await p.reload({ waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".tab").filter({ hasText: /Studio/i }).first().click();
|
||||
await p.waitForSelector("textarea", { timeout: 10000 });
|
||||
await p.locator("textarea").first().fill(`${persona} test draft.`);
|
||||
await p.locator("button.btn-primary").filter({ hasText: /draft/i }).first().click();
|
||||
await p.waitForTimeout(7000);
|
||||
const fivexx = calls.filter(c => c.status >= 500 && !c.url.includes("/llm/")).length;
|
||||
const post201 = calls.some(c => c.method === "POST" && c.url === "/api/ea2/flow" && c.status === 201);
|
||||
if (post201 && fivexx === 0) pass(`gap2_${persona.toLowerCase()}_studio_no_5xx`);
|
||||
else fail(`gap2_${persona.toLowerCase()}_studio_no_5xx`, `post201=${post201}, fivexx=${fivexx}`);
|
||||
await p.locator(".tab").filter({ hasText: /Chat/i }).first().click();
|
||||
await p.waitForTimeout(2500);
|
||||
const chatVisible = await p.locator(".chat-surface, .chat-list, .threadlist, .chat-empty, .chat-err").count();
|
||||
if (chatVisible > 0) pass(`gap2_${persona.toLowerCase()}_chat_surface_visible`);
|
||||
else fail(`gap2_${persona.toLowerCase()}_chat_surface_visible`);
|
||||
await p.screenshot({ path: `${OUT}/g2-${persona.toLowerCase()}-topbar.png`, clip: { x: 0, y: 0, width: 1440, height: 64 } });
|
||||
await p.context().close();
|
||||
}
|
||||
|
||||
// ============= GAP 3: Perf timings (Mission + EA2 sync + Assistant) =============
|
||||
console.log("\n=== Gap 3: Performance timings ===");
|
||||
{
|
||||
const { p, calls, loginMs } = await login("Mariana");
|
||||
const t0 = Date.now();
|
||||
await p.waitForSelector(".mc, .mc-strip, .mc-hero", { timeout: 15000 });
|
||||
await p.waitForTimeout(3000);
|
||||
const missionInitialMs = Date.now() - t0;
|
||||
const ea2Calls = calls.filter(c => c.url.startsWith("/api/ea2/"));
|
||||
console.log(` login: ${loginMs}ms`);
|
||||
console.log(` Mission first paint: ${missionInitialMs}ms`);
|
||||
console.log(` EA2 calls during initial sync: ${ea2Calls.length} (statuses: ${[...new Set(ea2Calls.map(c => c.status))].join(",")})`);
|
||||
if (missionInitialMs < 12000) pass("gap3_mission_initial_paint_under_12s", `${missionInitialMs}ms`);
|
||||
else fail("gap3_mission_initial_paint_under_12s", `${missionInitialMs}ms`);
|
||||
await p.locator(".tab").filter({ hasText: /Assistant/i }).first().click();
|
||||
await p.waitForSelector(".agent-composer textarea", { timeout: 15000 });
|
||||
await p.waitForTimeout(1500);
|
||||
const t1 = Date.now();
|
||||
await p.locator(".agent-composer textarea").first().fill("list processes");
|
||||
await p.locator(".agent-composer textarea").first().press("Enter");
|
||||
await p.waitForFunction(() => document.querySelectorAll(".agent-turn-body").length >= 3, { timeout: 20000 });
|
||||
const assistantListMs = Date.now() - t1;
|
||||
console.log(` Assistant 'list processes' end-to-end: ${assistantListMs}ms`);
|
||||
if (assistantListMs < 10000) pass("gap3_assistant_list_under_10s", `${assistantListMs}ms`);
|
||||
else fail("gap3_assistant_list_under_10s", `${assistantListMs}ms`);
|
||||
writeFileSync(`${OUT}/perf-timings.json`, JSON.stringify({ loginMs, missionInitialMs, assistantListMs, ea2Calls: ea2Calls.length }, null, 2));
|
||||
await p.context().close();
|
||||
}
|
||||
|
||||
// ============= GAP 4: Topbar visual + responsive + focus =============
|
||||
console.log("\n=== Gap 4: Topbar visual + responsive + focus ===");
|
||||
for (const width of [1920, 1440, 1024, 768]) {
|
||||
const ctx2 = await b.newContext({ viewport: { width, height: 900 } });
|
||||
const p = await ctx2.newPage();
|
||||
await ctx2.addInitScript(() => localStorage.setItem("fm.canvas.tour.seen", String(Date.now())));
|
||||
await p.goto(URL, { waitUntil: "domcontentloaded" });
|
||||
await p.waitForTimeout(2500);
|
||||
await p.locator(".persona-chip").filter({ hasText: /Mariana/ }).click();
|
||||
await p.waitForSelector(".topbar", { timeout: 30000 });
|
||||
await p.waitForTimeout(1500);
|
||||
const topbarBox = await p.locator(".topbar").boundingBox();
|
||||
const tabsOverflow = await p.evaluate(() => {
|
||||
const tb = document.querySelector(".topbar");
|
||||
return tb ? { scrollWidth: tb.scrollWidth, clientWidth: tb.clientWidth, overflowed: tb.scrollWidth > tb.clientWidth } : null;
|
||||
});
|
||||
console.log(` ${width}px: topbar h=${topbarBox?.height}px, overflowed=${tabsOverflow?.overflowed}`);
|
||||
if (tabsOverflow && !tabsOverflow.overflowed) pass(`gap4_topbar_no_overflow_at_${width}`);
|
||||
else fail(`gap4_topbar_no_overflow_at_${width}`, JSON.stringify(tabsOverflow));
|
||||
await p.screenshot({ path: `${OUT}/g4-topbar-${width}.png`, clip: { x: 0, y: 0, width, height: 80 } });
|
||||
await ctx2.close();
|
||||
}
|
||||
{
|
||||
const { p } = await login("Mariana");
|
||||
await p.waitForTimeout(2000);
|
||||
await p.keyboard.press("Tab");
|
||||
await p.keyboard.press("Tab");
|
||||
await p.keyboard.press("Tab");
|
||||
await p.waitForTimeout(300);
|
||||
const focused = await p.evaluate(() => document.activeElement?.tagName);
|
||||
console.log(` Tab focus reached: ${focused}`);
|
||||
if (focused === "BUTTON" || focused === "A") pass("gap4_topbar_keyboard_reachable", focused);
|
||||
else fail("gap4_topbar_keyboard_reachable", focused || "null");
|
||||
await p.screenshot({ path: `${OUT}/g4-keyboard-focus.png`, clip: { x: 0, y: 0, width: 1440, height: 80 } });
|
||||
await p.context().close();
|
||||
}
|
||||
|
||||
await b.close();
|
||||
|
||||
const passed = r.filter(x => x.ok).length;
|
||||
console.log(`\n=== ${passed}/${r.length} wave-4 gap checks passed ===`);
|
||||
writeFileSync(`${OUT}/results.json`, JSON.stringify(r, null, 2));
|
||||
process.exit(passed === r.length ? 0 : 1);
|
||||
Reference in New Issue
Block a user