// Single end-to-end QA gate. Exercises every new surface as a real user would, // against the live canvas.flow-master.ai URL with real EA2 traffic. Exits non-zero // on any failure so CI can wrap it. import { chromium } from "playwright"; const URL = "https://canvas.flow-master.ai/"; const args = ["--host-resolver-rules=MAP canvas.flow-master.ai 65.21.71.186", "--ignore-certificate-errors"]; const results = []; function record(name, ok, detail = "") { results.push({ name, ok, detail }); console.log(`${ok ? "PASS" : "FAIL"} ${name}${detail ? " — " + detail : ""}`); } const b = await chromium.launch({ headless: true, args }); const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } }); const p = await ctx.newPage(); // 1. Landing renders & login page is gated await p.goto(URL, { waitUntil: "networkidle" }); await p.waitForTimeout(1000); const onLogin = await p.locator(".login-page").count(); record("auth_guard_unauthed_redirects_to_login", onLogin === 1); const chips = await p.locator(".persona-chip").allTextContents(); record("login_shows_three_personas", chips.length === 3, chips.join(" | ")); const ssoBtn = await p.locator(".sso-btn").count(); record("login_has_microsoft_sso_button", ssoBtn === 1); // 2. Persona dev-login works await p.locator(".persona-chip", { hasText: /Mariana/ }).click(); await p.locator(".dev-login-btn").click(); await p.waitForTimeout(2500); const onLanding = await p.locator(".hero-actions").count(); record("ceo_persona_dev_login_lands_on_landing", onLanding === 1); // 3. Landing chips include every new surface const hubChips = await p.locator(".hub-chip").allTextContents(); const expected = ["Procurement Hub", "People Hub", "IT Hub", "Attendance Map", "Command Assistant", "Team Chat", "What is FlowMaster?"]; const missing = expected.filter((e) => !hubChips.some((h) => h.includes(e))); record("landing_exposes_all_hubs_and_extras", missing.length === 0, missing.length ? `missing: ${missing.join(", ")}` : "all 7 present"); // 4. Explainer renders 8 cards await p.locator(".hub-chip", { hasText: /What is FlowMaster/ }).click(); await p.waitForTimeout(1000); const explainerCards = await p.locator(".explainer-card").count(); record("explainer_renders_eight_cards", explainerCards === 8, `cards=${explainerCards}`); // 5. Geo-attendance: tiles + markers + stats await p.locator(".tab", { hasText: /Home/ }).first().click(); await p.waitForTimeout(800); await p.locator(".hub-chip", { hasText: /Attendance Map/ }).click(); await p.waitForTimeout(3000); const tiles = await p.locator(".leaflet-tile").count(); const markers = await p.locator(".leaflet-marker-icon").count(); record("geo_attendance_renders_tiles", tiles > 10, `tiles=${tiles}`); record("geo_attendance_renders_eight_markers", markers === 8, `markers=${markers}`); // 6. Command Assistant: tool list + list_processes round-trip await p.locator(".tab", { hasText: /Home/ }).first().click(); await p.waitForTimeout(800); await p.locator(".hub-chip", { hasText: /Command Assistant/ }).click(); await p.waitForTimeout(1200); const toolNames = await p.locator(".agent-tool-name").allTextContents(); record("assistant_shows_at_least_five_tools", toolNames.length >= 5, toolNames.join(", ")); const beforeCount = await p.locator(".agent-turn-agent .agent-turn-body").count(); await p.locator(".agent-composer textarea").fill("list processes"); await p.locator(".agent-composer button", { hasText: /Send/i }).click(); await p.waitForFunction( (n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, beforeCount, { timeout: 8000 } ).catch(() => {}); await p.waitForTimeout(500); const lastReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; record("assistant_list_processes_returns_real_ea2_data", lastReply.includes("•") || lastReply.toLowerCase().includes("no published"), lastReply.slice(0, 80)); // 7. Procurement hub returns real catalogue await p.locator(".tab", { hasText: /Home/ }).first().click(); await p.waitForTimeout(800); await p.locator(".hub-chip", { hasText: /Procurement Hub/ }).click(); await p.waitForTimeout(2200); const hubFlows = await p.locator(".hub-flow-title").count(); record("procurement_hub_lists_real_flows", hubFlows > 0, `flows=${hubFlows}`); // 8. Chat thread + message round-trip in one session (already covered by audit_chat, // here we just verify the Chat scene loads + sidebar shows existing threads) await p.locator(".tab", { hasText: /Chat/ }).first().click(); await p.waitForTimeout(2000); const existingThreads = await p.locator(".chat-thread-row").count(); record("chat_scene_loads_with_thread_sidebar", existingThreads >= 0, `threads=${existingThreads}`); // 9. Theme toggle flips the data-theme attribute const themeBtn = p.locator(".theme-toggle").first(); const before = await p.evaluate(() => document.documentElement.dataset.theme || "light"); await themeBtn.click(); await p.waitForTimeout(300); const after = await p.evaluate(() => document.documentElement.dataset.theme || "light"); record("theme_toggle_flips_theme", before !== after, `${before} -> ${after}`); // 10. Security headers const headers = (await p.request.get(URL)).headers(); const securityChecks = { "strict-transport-security": (v) => /max-age=\d+/.test(v), "x-frame-options": (v) => /deny|sameorigin/i.test(v), "x-content-type-options": (v) => /nosniff/i.test(v), "referrer-policy": (v) => v.length > 0, "content-security-policy": (v) => v.length > 0, "permissions-policy": (v) => v.length > 0, }; for (const [h, check] of Object.entries(securityChecks)) { const v = headers[h] || ""; record(`security_header_${h.replace(/-/g, "_")}`, !!v && check(v), v ? v.slice(0, 60) : "missing"); } // 11. NEGATIVE assertions: no dev artefacts anywhere a buyer would look. // Oracle remediation gap #4 — these must stay green after every change. async function pageBodyText() { return (await p.locator("body").innerText()).toLowerCase(); } async function neg(name, scene, banned) { await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {}); await p.waitForTimeout(400); if (scene === "landing") { // already there } else if (scene === "agent-list") { await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click(); await p.waitForTimeout(800); const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); await p.locator(".agent-composer textarea").fill("list processes"); await p.locator(".agent-composer button", { hasText: /Send/i }).click(); await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {}); await p.waitForTimeout(500); } else { await p.locator(".hub-chip", { hasText: new RegExp(scene, "i") }).click(); await p.waitForTimeout(1500); } const body = await pageBodyText(); const hits = banned.filter((b) => body.includes(b.toLowerCase())); record(`negative_${name}_no_dev_artefacts`, hits.length === 0, hits.length ? `LEAKED: ${hits.join(", ")}` : "clean"); } const BANNED = ["rv test flow", "rv_flow", "atlas f1", "atlas-f1-fresh", "mcp sdx smoke", "codex test", "sidekick", "orphan rv_", " orphan "]; await neg("landing", "landing", BANNED); await neg("procurement_hub", "Procurement Hub", BANNED); await neg("people_hub", "People Hub", BANNED); await neg("it_hub", "IT Hub", BANNED); await neg("agent_list_processes", "agent-list", BANNED); // 12. NEGATIVE: no raw 32-char hex IDs in user-facing toasts/UI. // Acceptable in dev console / debug only. await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {}); await p.waitForTimeout(300); const visibleText = await pageBodyText(); const hexLeaks = visibleText.match(/[a-f0-9]{32}/g) || []; record("negative_no_raw_32hex_ids_visible", hexLeaks.length === 0, hexLeaks.length ? `LEAKED ${hexLeaks.length}: ${hexLeaks[0]}` : "clean"); // 13. STARTABILITY: every process the Assistant lists must be a real startable // definition (not a wizard draft that lacks view/data attachments). // Oracle round-6 watch-out: a non-startable flow in the user-visible list is // a product-quality bug even if it's named like a business process. await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {}); await p.waitForTimeout(400); await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click(); await p.waitForTimeout(800); { const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); await p.locator(".agent-composer textarea").fill("list processes"); await p.locator(".agent-composer button", { hasText: /Send/i }).click(); await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {}); await p.waitForTimeout(500); } const listReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; const flowNames = listReply.split("\n").filter((l) => l.startsWith("•")).map((l) => l.replace(/^•\s*/, "").trim()); const tokenResp = await p.request.post("https://canvas.flow-master.ai/api/v1/auth/dev-login", { data: { email: "ceo-head@flow-master.ai" }, }); const token = (await tokenResp.json()).access_token; const catalogue = await p.request.get("https://canvas.flow-master.ai/api/ea2/flow/processes?limit=500", { headers: { Authorization: `Bearer ${token}` }, }); const items = ((await catalogue.json())?.items || []); const byName = new Map(items.filter((it) => it.display_name).map((it) => [it.display_name, it._key])); const unstartable = []; for (const name of flowNames) { const key = byName.get(name); if (!key) continue; const r = await p.request.post("https://canvas.flow-master.ai/api/runtime/transactions", { headers: { Authorization: `Bearer ${token}` }, data: { process_definition_id: key, business_subject: `qa-startability-${Date.now()}` }, }); if (r.status() >= 400) unstartable.push(`${name} (HTTP ${r.status()})`); } record("assistant_list_processes_all_startable", unstartable.length === 0, unstartable.length ? `unstartable: ${unstartable.join(", ")}` : `${flowNames.length} flows all startable`); // 14. NEGATIVE: agent welcome must not claim to be an "AI agent". The Assistant // scene is already open from step 13, so read the first turn body directly. const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().textContent()) || ""; const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome); record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80)); // 14b. NEGATIVE: no user-visible 'Pi' branding anywhere (deterministic router // should never read as the pi coding agent product). Sweep landing + // explainer + agent + chat scenes for the word boundary. async function scenePiLeak(scene) { if (scene === "landing") { await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {}); } else if (scene === "explainer") { await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {}); await p.waitForTimeout(400); await p.locator(".hub-chip", { hasText: /What is FlowMaster/ }).click(); } else if (scene === "agent") { await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {}); await p.waitForTimeout(400); await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click(); } await p.waitForTimeout(800); const body = await p.locator("body").innerText(); return /\bPi\b/.test(body); } const leaks = []; for (const sc of ["landing", "explainer", "agent"]) { if (await scenePiLeak(sc)) leaks.push(sc); } record("no_pi_branding_in_user_facing_scenes", leaks.length === 0, leaks.length ? `LEAKED in: ${leaks.join(", ")}` : "clean"); // 15. MEMORY VAULT: remember + recall round-trips, per-user isolation. // Assistant scene is already open from step 13. { const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now()); await p.locator(".agent-composer button", { hasText: /Send/i }).click(); await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {}); await p.waitForTimeout(500); } const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80)); { const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa"); await p.locator(".agent-composer button", { hasText: /Send/i }).click(); await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {}); await p.waitForTimeout(500); } const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100)); // 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works. // Asking 'what can you do' has no command matcher; should not crash; should // either show vault hints or the help message — never throw. { const before = await p.locator(".agent-turn-agent .agent-turn-body").count(); await p.locator(".agent-composer textarea").fill("what can you do for me"); await p.locator(".agent-composer button", { hasText: /Send/i }).click(); await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {}); await p.waitForTimeout(800); } const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || ""; record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100)); // 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative // timestamps and last-message previews; unread aggregate banner shows when // other personas have written since last open. await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click(); await p.waitForTimeout(2500); await p.waitForTimeout(2000); const threadRowCount = await p.locator(".chat-thread-row").count(); const previewRows = await p.locator(".chat-thread-meta").count(); const timeBadges = await p.locator(".chat-thread-time").count(); const sidebarHonest = threadRowCount === 0 || (previewRows === threadRowCount && timeBadges >= 1); record( "chat_sidebar_shows_previews_and_timestamps", sidebarHonest, `threads=${threadRowCount}, previews=${previewRows}, times=${timeBadges}` ); // Summarise const fails = results.filter((r) => !r.ok); console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`); if (fails.length) { console.log("FAILED:"); fails.forEach((f) => console.log(` - ${f.name}${f.detail ? ": " + f.detail : ""}`)); } await b.close(); process.exit(fails.length === 0 ? 0 : 1);