shad
05be9ca8dc
Three Oracle round-7 caveats: 1. Wizard view replacement now deletes old presentation edges before creating new field-based ones. Previously handleDataSave added new edges additively, so runtime could still hit the generic Notes view. 2. Chat sidebar QA assertion was vacuously true (previewRows >= 0 && timeBadges >= 0). Now: when threads exist, preview count must equal thread count AND at least one relative timestamp must render. 3. Agent composer placeholder rebranded from 'Ask Pi...' to 'Ask the assistant...'. Closes residual Pi/LLM framing leak.
259 lines
14 KiB
JavaScript
259 lines
14 KiB
JavaScript
// Single end-to-end QA gate. Exercises every new surface as a real user would,
|
|
// against the live canvas.flow-master.ai URL with real EA2 traffic. Exits non-zero
|
|
// on any failure so CI can wrap it.
|
|
import { chromium } from "playwright";
|
|
|
|
const URL = "https://canvas.flow-master.ai/";
|
|
const args = ["--host-resolver-rules=MAP canvas.flow-master.ai 65.21.71.186", "--ignore-certificate-errors"];
|
|
|
|
const results = [];
|
|
function record(name, ok, detail = "") {
|
|
results.push({ name, ok, detail });
|
|
console.log(`${ok ? "PASS" : "FAIL"} ${name}${detail ? " — " + detail : ""}`);
|
|
}
|
|
|
|
const b = await chromium.launch({ headless: true, args });
|
|
const ctx = await b.newContext({ viewport: { width: 1440, height: 900 } });
|
|
const p = await ctx.newPage();
|
|
|
|
// 1. Landing renders & login page is gated
|
|
await p.goto(URL, { waitUntil: "networkidle" });
|
|
await p.waitForTimeout(1000);
|
|
const onLogin = await p.locator(".login-page").count();
|
|
record("auth_guard_unauthed_redirects_to_login", onLogin === 1);
|
|
const chips = await p.locator(".persona-chip").allTextContents();
|
|
record("login_shows_three_personas", chips.length === 3, chips.join(" | "));
|
|
const ssoBtn = await p.locator(".sso-btn").count();
|
|
record("login_has_microsoft_sso_button", ssoBtn === 1);
|
|
|
|
// 2. Persona dev-login works
|
|
await p.locator(".persona-chip", { hasText: /Mariana/ }).click();
|
|
await p.locator(".dev-login-btn").click();
|
|
await p.waitForTimeout(2500);
|
|
const onLanding = await p.locator(".hero-actions").count();
|
|
record("ceo_persona_dev_login_lands_on_landing", onLanding === 1);
|
|
|
|
// 3. Landing chips include every new surface
|
|
const hubChips = await p.locator(".hub-chip").allTextContents();
|
|
const expected = ["Procurement Hub", "People Hub", "IT Hub", "Attendance Map", "Command Assistant", "Team Chat", "What is FlowMaster?"];
|
|
const missing = expected.filter((e) => !hubChips.some((h) => h.includes(e)));
|
|
record("landing_exposes_all_hubs_and_extras", missing.length === 0, missing.length ? `missing: ${missing.join(", ")}` : "all 7 present");
|
|
|
|
// 4. Explainer renders 8 cards
|
|
await p.locator(".hub-chip", { hasText: /What is FlowMaster/ }).click();
|
|
await p.waitForTimeout(1000);
|
|
const explainerCards = await p.locator(".explainer-card").count();
|
|
record("explainer_renders_eight_cards", explainerCards === 8, `cards=${explainerCards}`);
|
|
|
|
// 5. Geo-attendance: tiles + markers + stats
|
|
await p.locator(".tab", { hasText: /Home/ }).first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.locator(".hub-chip", { hasText: /Attendance Map/ }).click();
|
|
await p.waitForTimeout(3000);
|
|
const tiles = await p.locator(".leaflet-tile").count();
|
|
const markers = await p.locator(".leaflet-marker-icon").count();
|
|
record("geo_attendance_renders_tiles", tiles > 10, `tiles=${tiles}`);
|
|
record("geo_attendance_renders_eight_markers", markers === 8, `markers=${markers}`);
|
|
|
|
// 6. Command Assistant: tool list + list_processes round-trip
|
|
await p.locator(".tab", { hasText: /Home/ }).first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/ }).click();
|
|
await p.waitForTimeout(1200);
|
|
const toolNames = await p.locator(".agent-tool-name").allTextContents();
|
|
record("assistant_shows_at_least_five_tools", toolNames.length >= 5, toolNames.join(", "));
|
|
const beforeCount = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("list processes");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction(
|
|
(n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n,
|
|
beforeCount,
|
|
{ timeout: 8000 }
|
|
).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
const lastReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("assistant_list_processes_returns_real_ea2_data", lastReply.includes("•") || lastReply.toLowerCase().includes("no published"), lastReply.slice(0, 80));
|
|
|
|
// 7. Procurement hub returns real catalogue
|
|
await p.locator(".tab", { hasText: /Home/ }).first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.locator(".hub-chip", { hasText: /Procurement Hub/ }).click();
|
|
await p.waitForTimeout(2200);
|
|
const hubFlows = await p.locator(".hub-flow-title").count();
|
|
record("procurement_hub_lists_real_flows", hubFlows > 0, `flows=${hubFlows}`);
|
|
|
|
// 8. Chat thread + message round-trip in one session (already covered by audit_chat,
|
|
// here we just verify the Chat scene loads + sidebar shows existing threads)
|
|
await p.locator(".tab", { hasText: /Chat/ }).first().click();
|
|
await p.waitForTimeout(2000);
|
|
const existingThreads = await p.locator(".chat-thread-row").count();
|
|
record("chat_scene_loads_with_thread_sidebar", existingThreads >= 0, `threads=${existingThreads}`);
|
|
|
|
// 9. Theme toggle flips the data-theme attribute
|
|
const themeBtn = p.locator(".theme-toggle").first();
|
|
const before = await p.evaluate(() => document.documentElement.dataset.theme || "light");
|
|
await themeBtn.click();
|
|
await p.waitForTimeout(300);
|
|
const after = await p.evaluate(() => document.documentElement.dataset.theme || "light");
|
|
record("theme_toggle_flips_theme", before !== after, `${before} -> ${after}`);
|
|
|
|
// 10. Security headers
|
|
const headers = (await p.request.get(URL)).headers();
|
|
const securityChecks = {
|
|
"strict-transport-security": (v) => /max-age=\d+/.test(v),
|
|
"x-frame-options": (v) => /deny|sameorigin/i.test(v),
|
|
"x-content-type-options": (v) => /nosniff/i.test(v),
|
|
"referrer-policy": (v) => v.length > 0,
|
|
"content-security-policy": (v) => v.length > 0,
|
|
"permissions-policy": (v) => v.length > 0,
|
|
};
|
|
for (const [h, check] of Object.entries(securityChecks)) {
|
|
const v = headers[h] || "";
|
|
record(`security_header_${h.replace(/-/g, "_")}`, !!v && check(v), v ? v.slice(0, 60) : "missing");
|
|
}
|
|
|
|
// 11. NEGATIVE assertions: no dev artefacts anywhere a buyer would look.
|
|
// Oracle remediation gap #4 — these must stay green after every change.
|
|
async function pageBodyText() {
|
|
return (await p.locator("body").innerText()).toLowerCase();
|
|
}
|
|
async function neg(name, scene, banned) {
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
if (scene === "landing") {
|
|
// already there
|
|
} else if (scene === "agent-list") {
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click();
|
|
await p.waitForTimeout(800);
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("list processes");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
} else {
|
|
await p.locator(".hub-chip", { hasText: new RegExp(scene, "i") }).click();
|
|
await p.waitForTimeout(1500);
|
|
}
|
|
const body = await pageBodyText();
|
|
const hits = banned.filter((b) => body.includes(b.toLowerCase()));
|
|
record(`negative_${name}_no_dev_artefacts`, hits.length === 0, hits.length ? `LEAKED: ${hits.join(", ")}` : "clean");
|
|
}
|
|
const BANNED = ["rv test flow", "rv_flow", "atlas f1", "atlas-f1-fresh", "mcp sdx smoke", "codex test", "sidekick", "orphan rv_", " orphan "];
|
|
await neg("landing", "landing", BANNED);
|
|
await neg("procurement_hub", "Procurement Hub", BANNED);
|
|
await neg("people_hub", "People Hub", BANNED);
|
|
await neg("it_hub", "IT Hub", BANNED);
|
|
await neg("agent_list_processes", "agent-list", BANNED);
|
|
|
|
// 12. NEGATIVE: no raw 32-char hex IDs in user-facing toasts/UI.
|
|
// Acceptable in dev console / debug only.
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(300);
|
|
const visibleText = await pageBodyText();
|
|
const hexLeaks = visibleText.match(/[a-f0-9]{32}/g) || [];
|
|
record("negative_no_raw_32hex_ids_visible", hexLeaks.length === 0, hexLeaks.length ? `LEAKED ${hexLeaks.length}: ${hexLeaks[0]}` : "clean");
|
|
|
|
// 13. STARTABILITY: every process the Assistant lists must be a real startable
|
|
// definition (not a wizard draft that lacks view/data attachments).
|
|
// Oracle round-6 watch-out: a non-startable flow in the user-visible list is
|
|
// a product-quality bug even if it's named like a business process.
|
|
await p.locator(".tab", { hasText: /Home/i }).first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
await p.locator(".hub-chip", { hasText: /Command Assistant/i }).click();
|
|
await p.waitForTimeout(800);
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("list processes");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
}
|
|
const listReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
const flowNames = listReply.split("\n").filter((l) => l.startsWith("•")).map((l) => l.replace(/^•\s*/, "").trim());
|
|
const tokenResp = await p.request.post("https://canvas.flow-master.ai/api/v1/auth/dev-login", {
|
|
data: { email: "ceo-head@flow-master.ai" },
|
|
});
|
|
const token = (await tokenResp.json()).access_token;
|
|
const catalogue = await p.request.get("https://canvas.flow-master.ai/api/ea2/flow/processes?limit=500", {
|
|
headers: { Authorization: `Bearer ${token}` },
|
|
});
|
|
const items = ((await catalogue.json())?.items || []);
|
|
const byName = new Map(items.filter((it) => it.display_name).map((it) => [it.display_name, it._key]));
|
|
const unstartable = [];
|
|
for (const name of flowNames) {
|
|
const key = byName.get(name);
|
|
if (!key) continue;
|
|
const r = await p.request.post("https://canvas.flow-master.ai/api/runtime/transactions", {
|
|
headers: { Authorization: `Bearer ${token}` },
|
|
data: { process_definition_id: key, business_subject: `qa-startability-${Date.now()}` },
|
|
});
|
|
if (r.status() >= 400) unstartable.push(`${name} (HTTP ${r.status()})`);
|
|
}
|
|
record("assistant_list_processes_all_startable", unstartable.length === 0, unstartable.length ? `unstartable: ${unstartable.join(", ")}` : `${flowNames.length} flows all startable`);
|
|
|
|
// 14. NEGATIVE: agent welcome must not claim to be an "AI agent". The Assistant
|
|
// scene is already open from step 13, so read the first turn body directly.
|
|
const welcome = (await p.locator(".agent-turn-agent .agent-turn-body").first().textContent()) || "";
|
|
const honestyOk = !/\bai agent\b/i.test(welcome) || /command assistant/i.test(welcome);
|
|
record("agent_welcome_does_not_overclaim_ai", honestyOk, welcome.slice(0, 80));
|
|
|
|
// 15. MEMORY VAULT: remember + recall round-trips, per-user isolation.
|
|
// Assistant scene is already open from step 13.
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("remember that the marker for r4 qa is " + Date.now());
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
}
|
|
const rememberReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("memory_vault_remember_acks", /I'll remember/i.test(rememberReply), rememberReply.slice(0, 80));
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("recall about marker for r4 qa");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 8000 }).catch(() => {});
|
|
await p.waitForTimeout(500);
|
|
}
|
|
const recallReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("memory_vault_recall_returns_match", /marker for r4 qa/i.test(recallReply), recallReply.slice(0, 100));
|
|
|
|
// 16. LLM FALLBACK: when proxy unconfigured, deterministic command path still works.
|
|
// Asking 'what can you do' has no command matcher; should not crash; should
|
|
// either show vault hints or the help message — never throw.
|
|
{
|
|
const before = await p.locator(".agent-turn-agent .agent-turn-body").count();
|
|
await p.locator(".agent-composer textarea").fill("what can you do for me");
|
|
await p.locator(".agent-composer button", { hasText: /Send/i }).click();
|
|
await p.waitForFunction((n) => document.querySelectorAll(".agent-turn-agent .agent-turn-body").length > n, before, { timeout: 12000 }).catch(() => {});
|
|
await p.waitForTimeout(800);
|
|
}
|
|
const unmatchedReply = (await p.locator(".agent-turn-agent .agent-turn-body").last().textContent()) || "";
|
|
record("llm_unconfigured_falls_back_gracefully", unmatchedReply.length > 0 && !/Error/i.test(unmatchedReply), unmatchedReply.slice(0, 100));
|
|
|
|
// 17. CHAT polish: open Chat tab; if any thread exists, sidebar shows relative
|
|
// timestamps and last-message previews; unread aggregate banner shows when
|
|
// other personas have written since last open.
|
|
await p.locator(".tab", { hasText: /^\s*Chat\s*$/ }).click();
|
|
await p.waitForTimeout(2500);
|
|
await p.waitForTimeout(2000);
|
|
const threadRowCount = await p.locator(".chat-thread-row").count();
|
|
const previewRows = await p.locator(".chat-thread-meta").count();
|
|
const timeBadges = await p.locator(".chat-thread-time").count();
|
|
const sidebarHonest = threadRowCount === 0 || (previewRows === threadRowCount && timeBadges >= 1);
|
|
record(
|
|
"chat_sidebar_shows_previews_and_timestamps",
|
|
sidebarHonest,
|
|
`threads=${threadRowCount}, previews=${previewRows}, times=${timeBadges}`
|
|
);
|
|
|
|
// Summarise
|
|
const fails = results.filter((r) => !r.ok);
|
|
console.log(`\n=== ${results.length - fails.length}/${results.length} passed ===`);
|
|
if (fails.length) {
|
|
console.log("FAILED:");
|
|
fails.forEach((f) => console.log(` - ${f.name}${f.detail ? ": " + f.detail : ""}`));
|
|
}
|
|
await b.close();
|
|
process.exit(fails.length === 0 ? 0 : 1);
|