shad
7a08b51eec
When the QA runs against a fresh CEO session (no pre-existing chat threads), the new-thread create + inbox-edge + message send chain is 4-8 EA2 round-trips. Prior waits (8s thread + 2.5s send + 12s condition) were occasionally too tight. Bumped to 15s + 4s + 20s. 36/36 dogfood stable across the slow-path.
346 lines
18 KiB
JavaScript
346 lines
18 KiB
JavaScript
// Single end-to-end QA gate. Exercises every new surface as a real user would,
|
|
// against the live canvas.flow-master.ai URL with real EA2 traffic. Exits non-zero
|
|
// on any failure so CI can wrap it.
|
|
import { chromium } from "playwright";
|
|
|
|
const URL = "https://canvas.flow-master.ai/";
|
|
const args = ["--host-resolver-rules=MAP canvas.flow-master.ai 65.21.71.186", "--ignore-certificate-errors"];
|
|
|
|
const results = [];
|
|
function record(name, ok, detail = "") {
|
|
results.push({ name, ok, detail });
|
|
console.log(`${ok ? "PASS" : "FAIL"} ${name}${detail ? " — " + detail : ""}`);
|
|
}
|
|
|
|
const b = await chromium.launch({ headless: true, args });
|
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
|
const p = await ctx.newPage();
|
|
|
|
// 1. Landing renders & login page is gated
|
|
await p.goto(URL, { waitUntil: "networkidle" });
|
|
await p.waitForTimeout(1000);
|
|
const onLogin = await p.locator(".login-page").count();
|
|
record("auth_guard_unauthed_redirects_to_login", onLogin === 1);
|
|
const chips = await p.locator(".persona-chip").allTextContents();
|
|
record("login_shows_three_personas", chips.length === 3, chips.join(" | "));
|
|
const ssoBtn = await p.locator(".sso-btn").count();
|
|
record("login_has_microsoft_sso_button", ssoBtn === 1);
|
|
|
|
// 2. Persona dev-login works. Wait for the landing hero to actually hydrate
|
|
// rather than a fixed sleep so a slow EA2 round-trip doesn't flake.
|
|
await p.locator(".persona-chip", { hasText: /Mariana/ }).click();
|
|
await p.locator(".dev-login-btn").click();
|
|
await p.waitForSelector(".hero-actions", { timeout: 15000 }).catch(() => {});
|
|
await p.waitForFunction(() => document.querySelectorAll(".hub-chip").length >= 7, undefined, { timeout: 15000 }).catch(() => {});
|
|
const onLanding = await p.locator(".hero-actions").count();
|
|
record("ceo_persona_dev_login_lands_on_landing", onLanding === 1);
|
|
|
|
// 3. Landing chips include every new surface
|
|
const hubChips = await p.locator(".hub-chip").allTextContents();
|
|
const expected = ["Procurement Hub", "People Hub", "IT Hub", "Attendance Map", "Command Assistant", "Team Chat", "What is FlowMaster?"];
|
|
const missing = expected.filter((e) => !hubChips.some((h) => h.includes(e)));
|
|
record("landing_exposes_all_hubs_and_extras", missing.length === 0, missing.length ? `missing: ${missing.join(", ")}` : "all 7 present");
|
|
|
|
// 4. Explainer renders 8 cards
|
|
await p.locator(".hub-chip", { hasText: /What is FlowMaster/ }).click();
|
|
await p.waitForTimeout(1000);
|
|
const explainerCards = await p.locator(".explainer-card").count();
|
|
record("explainer_renders_eight_cards", explainerCards === 8, `cards=${explainerCards}`);
|
|
|
|
// 5. Geo-attendance: tiles + markers + stats. Wait up to 20s for EA2 fetch
|
|
// + leaflet hydration; the attendance API materialises 8 docs lazily.
|
|
await p.locator(".tab", { hasText: /Home/ }).first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.locator(".hub-chip", { hasText: /Attendance Map/ }).click();
|
|
await p.waitForFunction(() => document.querySelectorAll(".leaflet-marker-icon").length >= 8, undefined, { timeout: 20000 }).catch(() => {});
|
|
const tiles = await p.locator(".leaflet-tile").count();
|
|
const markers = await p.locator(".leaflet-marker-icon").count();
|
|
record("geo_attendance_renders_tiles", tiles > 10, `tiles=${tiles}`);
|
|
record("geo_attendance_renders_eight_markers", markers === 8, `markers=${markers}`);
|
|
|
|
// 6. Command Assistant: tool list + list_processes round-trip
|
|
await p.locator(".tab", { hasText: /Home/ }).first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/ }).click();
|
|
await p.waitForTimeout(1200);
|
|
const toolNames = await p.locator(".agent-tool-name").allTextContents();
|
|
record("assistant_shows_at_least_five_tools", toolNames.length >= 5, toolNames.join(", "));
|
|
const beforeCount = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("list processes");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction(
|
|
(n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n,
|
|
beforeCount,
|
|
{ timeout: 8000 }
|
|
).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
const lastReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("assistant_list_processes_returns_real_ea2_data", lastReply.includes("•") || lastReply.toLowerCase().includes("no published"), lastReply.slice(0, 80));
|
|
|
|
// 7. Procurement hub returns real catalogue
|
|
await p.locator(".tab", { hasText: /Home/ }).first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.locator(".hub-chip", { hasText: /Procurement Hub/ }).click();
|
|
await p.waitForTimeout(2200);
|
|
const hubFlows = await p.locator(".hub-flow-title").count();
|
|
record("procurement_hub_lists_real_flows", hubFlows > 0, `flows=${hubFlows}`);
|
|
|
|
// 8. Chat thread + message round-trip in one session (already covered by audit_chat,
|
|
// here we just verify the Chat scene loads + sidebar shows existing threads)
|
|
await p.locator(".tab", { hasText: /Chat/ }).first().click();
|
|
await p.waitForTimeout(2000);
|
|
const existingThreads = await p.locator(".chat-thread-row").count();
|
|
record("chat_scene_loads_with_thread_sidebar", existingThreads >= 0, `threads=${existingThreads}`);
|
|
|
|
// 9. Theme toggle flips the data-theme attribute
|
|
const themeBtn = p.locator(".theme-toggle").first();
|
|
const before = await p.evaluate(() => document.documentElement.dataset.theme || "light");
|
|
await themeBtn.click();
|
|
await p.waitForTimeout(300);
|
|
const after = await p.evaluate(() => document.documentElement.dataset.theme || "light");
|
|
record("theme_toggle_flips_theme", before !== after, `${before} -> ${after}`);
|
|
|
|
// 10. Security headers
|
|
const headers = (await p.request.get(URL)).headers();
|
|
const securityChecks = {
|
|
"strict-transport-security": (v) => /max-age=\d+/.test(v),
|
|
"x-frame-options": (v) => /deny|sameorigin/i.test(v),
|
|
"x-content-type-options": (v) => /nosniff/i.test(v),
|
|
"referrer-policy": (v) => v.length > 0,
|
|
"content-security-policy": (v) => v.length > 0,
|
|
"permissions-policy": (v) => v.length > 0,
|
|
};
|
|
for (const [h, check] of Object.entries(securityChecks)) {
|
|
const v = headers[h] || "";
|
|
record(`security_header_${h.replace(/-/g, "_")}`, !!v && check(v), v ? v.slice(0, 60) : "missing");
|
|
}
|
|
|
|
// 11. NEGATIVE assertions: no dev artefacts anywhere a buyer would look.
|
|
// Oracle remediation gap #4 — these must stay green after every change.
|
|
async function pageBodyText() {
|
|
return (await p.locator("body").innerText()).toLowerCase();
|
|
}
|
|
async function neg(name, scene, banned) {
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
if (scene === "landing") {
|
|
// already there
|
|
} else if (scene === "agent-list") {
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click();
|
|
await p.waitForTimeout(800);
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("list processes");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
} else {
|
|
await p.locator(".hub-chip", { hasText: new RegExp(scene, "i") }).click();
|
|
await p.waitForTimeout(1500);
|
|
}
|
|
const body = await pageBodyText();
|
|
const hits = banned.filter((b) => body.includes(b.toLowerCase()));
|
|
record(`negative_${name}_no_dev_artefacts`, hits.length === 0, hits.length ? `LEAKED: ${hits.join(", ")}` : "clean");
|
|
}
|
|
const BANNED = ["rv test flow", "rv_flow", "atlas f1", "atlas-f1-fresh", "mcp sdx smoke", "codex test", "sidekick", "orphan rv_", " orphan "];
|
|
await neg("landing", "landing", BANNED);
|
|
await neg("procurement_hub", "Procurement Hub", BANNED);
|
|
await neg("people_hub", "People Hub", BANNED);
|
|
await neg("it_hub", "IT Hub", BANNED);
|
|
await neg("agent_list_processes", "agent-list", BANNED);
|
|
|
|
// 12. NEGATIVE: no raw 32-char hex IDs in user-facing toasts/UI.
|
|
// Acceptable in dev console / debug only.
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(300);
|
|
const visibleText = await pageBodyText();
|
|
const hexLeaks = visibleText.match(/[a-f0-9]{32}/g) || [];
|
|
record("negative_no_raw_32hex_ids_visible", hexLeaks.length === 0, hexLeaks.length ? `LEAKED ${hexLeaks.length}: ${hexLeaks[0]}` : "clean");
|
|
|
|
// 13. STARTABILITY: every process the Assistant lists must be a real startable
|
|
// definition (not a wizard draft that lacks view/data attachments).
|
|
// Oracle round-6 watch-out: a non-startable flow in the user-visible list is
|
|
// a product-quality bug even if it's named like a business process.
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click();
|
|
await p.waitForTimeout(800);
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("list processes");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
}
|
|
const listReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
const flowNames = listReply.split("\n").filter((l) => l.startsWith("•")).map((l) => l.replace(/^•\s*/, "").trim());
|
|
const tokenResp = await p.request.post("https://canvas.flow-master.ai/api/v1/auth/dev-login", {
|
|
data: { email: "ceo-head@flow-master.ai" },
|
|
});
|
|
const token = (await tokenResp.json()).access_token;
|
|
const catalogue = await p.request.get("https://canvas.flow-master.ai/api/ea2/flow/processes?limit=500", {
|
|
headers: { Authorization: `Bearer ${token}` },
|
|
});
|
|
const items = ((await catalogue.json())?.items || []);
|
|
const byName = new Map(items.filter((it) => it.display_name).map((it) => [it.display_name, it._key]));
|
|
const unstartable = [];
|
|
for (const name of flowNames) {
|
|
const key = byName.get(name);
|
|
if (!key) continue;
|
|
const r = await p.request.post("https://canvas.flow-master.ai/api/runtime/transactions", {
|
|
headers: { Authorization: `Bearer ${token}` },
|
|
data: { process_definition_id: key, business_subject: `qa-startability-${Date.now()}` },
|
|
});
|
|
if (r.status() >= 400) unstartable.push(`${name} (HTTP ${r.status()})`);
|
|
}
|
|
record("assistant_list_processes_all_startable", unstartable.length === 0, unstartable.length ? `unstartable: ${unstartable.join(", ")}` : `${flowNames.length} flows all startable`);
|
|
|
|
// 14. NEGATIVE: agent welcome must not claim to be an "AI agent". The Assistant
|
|
// scene is already open from step 13, so read the first turn body directly.
|
|
const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().textContent()) || "";
|
|
const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome);
|
|
record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80));
|
|
|
|
// 14b. NEGATIVE: no user-visible 'Pi' branding anywhere (deterministic router
|
|
// should never read as the pi coding agent product). Sweep landing +
|
|
// explainer + agent + chat scenes for the word boundary.
|
|
async function scenePiLeak(scene) {
|
|
if (scene === "landing") {
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
} else if (scene === "explainer") {
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
await p.locator(".hub-chip", { hasText: /What is FlowMaster/ }).click();
|
|
} else if (scene === "agent") {
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click();
|
|
}
|
|
await p.waitForTimeout(800);
|
|
const body = await p.locator("body").innerText();
|
|
return /\bPi\b/.test(body);
|
|
}
|
|
const leaks = [];
|
|
for (const sc of ["landing", "explainer", "agent"]) {
|
|
if (await scenePiLeak(sc)) leaks.push(sc);
|
|
}
|
|
record("no_pi_branding_in_user_facing_scenes", leaks.length === 0, leaks.length ? `LEAKED in: ${leaks.join(", ")}` : "clean");
|
|
|
|
// 15. MEMORY VAULT: remember + recall round-trips, per-user isolation.
|
|
// Assistant scene is already open from step 13.
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now());
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
}
|
|
const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80));
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
}
|
|
const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100));
|
|
|
|
// 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works.
|
|
// Asking 'what can you do' has no command matcher; should not crash; should
|
|
// either show vault hints or the help message — never throw.
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("what can you do for me");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {});
|
|
await p.waitForTimeout(800);
|
|
}
|
|
const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100));
|
|
|
|
// 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative
|
|
// timestamps and last-message previews; unread aggregate banner shows when
|
|
// other personas have written since last open.
|
|
await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click();
|
|
await p.waitForTimeout(2500);
|
|
const initialThreadCount = await p.locator(".chat-thread-row").count();
|
|
if (initialThreadCount === 0) {
|
|
const sel = p.locator(".chat-new-row select").first();
|
|
if (await sel.count()) {
|
|
await sel.selectOption("hr-head@flow-master.ai").catch(() => {});
|
|
await p.locator(".chat-new-row button", { hasText: /Open/ }).click().catch(() => {});
|
|
await p.waitForSelector(".chat-thread-row", { timeout: 15000 }).catch(() => {});
|
|
await p.waitForTimeout(1500);
|
|
}
|
|
const ta = p.locator(".chat-composer textarea").first();
|
|
if (await ta.count()) {
|
|
await ta.fill(`qa seed message ${Date.now()}`);
|
|
await p.locator(".chat-composer button", { hasText: /Send/i }).click().catch(() => {});
|
|
await p.waitForTimeout(4000);
|
|
}
|
|
}
|
|
await p.waitForFunction(() => document.querySelectorAll(".chat-thread-row").length >= 1, undefined, { timeout: 20000 }).catch(() => {});
|
|
await p.waitForTimeout(2500);
|
|
const threadRowCount = await p.locator(".chat-thread-row").count();
|
|
const previewRows = await p.locator(".chat-thread-meta").count();
|
|
const timeBadges = await p.locator(".chat-thread-time").count();
|
|
// Strict: at least one thread, every thread has a preview, and the timestamp
|
|
// count equals the thread count (one per row).
|
|
const sidebarHonest = threadRowCount >= 1 && previewRows === threadRowCount && timeBadges === threadRowCount;
|
|
record(
|
|
"chat_sidebar_shows_previews_and_timestamps",
|
|
sidebarHonest,
|
|
`threads=${threadRowCount}, previews=${previewRows}, times=${timeBadges}`
|
|
);
|
|
|
|
// 18. APPROVALS scene loads with tenant-scoped queue + clickable rows.
|
|
await p.goto("https://canvas.flow-master.ai/", { waitUntil: "networkidle" }).catch(() => {});
|
|
await p.waitForTimeout(600);
|
|
await p.locator(".hub-chip", { hasText: /Approvals queue/ }).click().catch(() => {});
|
|
await p.waitForSelector(".approvals-row", { timeout: 15000 }).catch(() => {});
|
|
await p.waitForTimeout(1000);
|
|
const approvalsRows = await p.locator(".approvals-row").count();
|
|
record("approvals_scene_renders_queue", approvalsRows > 0, `rows=${approvalsRows}`);
|
|
if (approvalsRows > 0) {
|
|
await p.locator(".approvals-row").first().click();
|
|
await p.waitForSelector(".approvals-card", { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(600);
|
|
const cardTitle = (await p.locator(".approvals-card-title").first().textContent()) || "";
|
|
const actionBtns = await p.locator(".approvals-actions .btn").count();
|
|
record(
|
|
"approvals_detail_shows_active_step_with_actions",
|
|
cardTitle.trim().length > 0 && actionBtns > 0,
|
|
`step="${cardTitle.trim().slice(0, 40)}" actions=${actionBtns}`
|
|
);
|
|
}
|
|
|
|
// 19. DOCUMENTS scene loads with EA2-sourced data definitions.
|
|
await p.goto("https://canvas.flow-master.ai/", { waitUntil: "networkidle" }).catch(() => {});
|
|
await p.waitForTimeout(600);
|
|
await p.locator(".hub-chip", { hasText: /^Documents$/ }).click().catch(() => {});
|
|
await p.waitForSelector(".docs-row", { timeout: 20000 }).catch(() => {});
|
|
await p.waitForTimeout(1500);
|
|
const docsRows = await p.locator(".docs-row").count();
|
|
record("documents_scene_renders_data_definitions", docsRows > 0, `rows=${docsRows}`);
|
|
|
|
// 20. LLM proxy is reachable through canvas nginx; returns 503 without provider.
|
|
const llmRes = await p.request.post("https://canvas.flow-master.ai/internal/canvas-llm/chat", {
|
|
data: { messages: [{ role: "user", content: "ping" }] },
|
|
});
|
|
record(
|
|
"llm_proxy_chain_returns_503_without_provider_key",
|
|
llmRes.status() === 503,
|
|
`HTTP ${llmRes.status()}`
|
|
);
|
|
|
|
// Summarise
|
|
const fails = results.filter((r) => !r.ok);
|
|
console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`);
|
|
if (fails.length) {
|
|
console.log("FAILED:");
|
|
fails.forEach((f) => console.log(` - ${f.name}${f.detail ? ": " + f.detail : ""}`));
|
|
}
|
|
await b.close();
|
|
process.exit(fails.length === 0 ? 0 : 1);
|