249 lines
11 KiB
JavaScript
249 lines
11 KiB
JavaScript
// Real human-style QA against https://canvas.flow-master.ai.
|
|
// Drives the live site through every flow a person would touch and
|
|
// captures: console errors, network errors, real action round-trips,
|
|
// identity switching, dark/light, mobile width, screenshot evidence.
|
|
//
|
|
// Bypasses local DNS cache by mapping to the LB IP directly.
|
|
import { chromium, devices } from "playwright";
|
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
|
const URL = "https://canvas.flow-master.ai/";
|
|
const FORCE_IP = "65.21.71.186";
|
|
const OUT = "qa/screenshots/human";
|
|
mkdirSync(OUT, { recursive: true });
|
|
|
|
const findings = [];
|
|
const note = (kind, where, msg) => {
|
|
const f = { kind, where, msg };
|
|
findings.push(f);
|
|
console.log(`[${kind}] ${where} :: ${msg}`);
|
|
};
|
|
|
|
async function attach(page, label) {
|
|
page.on("pageerror", (e) => note("pageerror", label, e.message.slice(0, 200)));
|
|
page.on("console", (m) => {
|
|
if (m.type() === "error") note("console.error", label, m.text().slice(0, 200));
|
|
if (m.type() === "warning") note("console.warning", label, m.text().slice(0, 200));
|
|
});
|
|
page.on("requestfailed", (r) => note("requestfailed", label, `${r.url()} :: ${r.failure()?.errorText}`));
|
|
page.on("response", (r) => {
|
|
if (r.status() >= 400) note("http", label, `${r.status()} ${r.url()}`);
|
|
});
|
|
}
|
|
|
|
const browser = await chromium.launch({
|
|
headless: true,
|
|
args: [
|
|
`--host-resolver-rules=MAP canvas.flow-master.ai ${FORCE_IP}`,
|
|
"--ignore-certificate-errors",
|
|
],
|
|
});
|
|
|
|
// =========================================================================
|
|
// FLOW 1 — desktop landing → mission → real action round-trip
|
|
// =========================================================================
|
|
{
|
|
console.log("\n=== FLOW 1: desktop full walkthrough ===");
|
|
const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, ignoreHTTPSErrors: true });
|
|
const p = await ctx.newPage();
|
|
await attach(p, "F1");
|
|
const tApiCalls = [];
|
|
p.on("request", (req) => { if (req.url().includes("/api/")) tApiCalls.push({ url: req.url(), at: Date.now() }); });
|
|
|
|
await p.goto(URL, { waitUntil: "networkidle" });
|
|
await p.screenshot({ path: `${OUT}/f1-01-landing.png` });
|
|
|
|
// R1.S6: scan the page for the word 'demo'
|
|
const visibleText = await p.evaluate(() => document.body.innerText);
|
|
const demoHits = (visibleText.match(/\bdemo\b/gi) || []).length;
|
|
if (demoHits > 0) note("de-demo", "landing", `'demo' appears ${demoHits} times in visible text`);
|
|
|
|
// F1.a click first scenario card
|
|
await p.locator(".sc-card").first().click();
|
|
await p.waitForSelector(".mc"); await p.waitForTimeout(800);
|
|
await p.screenshot({ path: `${OUT}/f1-02-mission.png` });
|
|
|
|
// F1.b switch to LIVE mode
|
|
await p.locator(".mode-toggle").first().click();
|
|
const liveOk = await p.waitForFunction(
|
|
() => Array.from(document.querySelectorAll(".mode-pill")).some((el) => el.textContent?.trim() === "LIVE"),
|
|
{ timeout: 15000 },
|
|
).then(() => true).catch(() => false);
|
|
if (!liveOk) note("flow", "mission", "LIVE toggle did not flip");
|
|
await p.waitForTimeout(2500);
|
|
await p.screenshot({ path: `${OUT}/f1-03-live.png` });
|
|
|
|
// F1.c open console drawer
|
|
await p.locator(".link-btn", { hasText: /console/i }).first().click();
|
|
await p.waitForSelector(".console"); await p.waitForTimeout(400);
|
|
const callsInConsole = await p.locator(".call").count();
|
|
await p.screenshot({ path: `${OUT}/f1-04-console.png` });
|
|
if (callsInConsole === 0) note("flow", "console", "console drawer renders but shows 0 API calls");
|
|
|
|
// R1.S5: GET-STORM test — sit idle on mission for 60s, count /api calls
|
|
console.log("idle 60s to measure GET storm…");
|
|
const idleStart = Date.now();
|
|
const beforeIdle = tApiCalls.length;
|
|
await p.waitForTimeout(60000);
|
|
const afterIdle = tApiCalls.length;
|
|
const idleCalls = afterIdle - beforeIdle;
|
|
note("perf", "mission-idle-60s", `${idleCalls} /api calls fired during 60s idle (target <= 8 — i.e. one poll-cycle worth)`);
|
|
|
|
// F1.d sign in via Settings quick-user (current build path)
|
|
await p.locator(".tab", { hasText: /settings/i }).click();
|
|
await p.waitForSelector(".settings"); await p.waitForTimeout(300);
|
|
await p.screenshot({ path: `${OUT}/f1-05-settings.png` });
|
|
const quickUserBtns = p.locator(".quick-users .link-btn");
|
|
const quickUserCount = await quickUserBtns.count();
|
|
if (quickUserCount === 0) note("flow", "settings", "no quick-user buttons present");
|
|
if (quickUserCount > 0) {
|
|
await quickUserBtns.first().click();
|
|
await p.waitForTimeout(2500);
|
|
await p.screenshot({ path: `${OUT}/f1-06-signed-in.png` });
|
|
}
|
|
|
|
// F1.e try to start a real instance from Mission → LeftRail
|
|
await p.locator(".tab", { hasText: /mission/i }).click();
|
|
await p.waitForSelector(".mc"); await p.waitForTimeout(600);
|
|
const startBtn = p.locator(".start-btn");
|
|
const startBtnExists = await startBtn.count();
|
|
if (startBtnExists > 0) {
|
|
const beforeStart = tApiCalls.filter((c) => c.url.includes("/api/runtime/transactions") && !c.url.includes("/actions/")).length;
|
|
await startBtn.first().click();
|
|
await p.waitForTimeout(3000);
|
|
const afterStart = tApiCalls.filter((c) => c.url.includes("/api/runtime/transactions") && !c.url.includes("/actions/")).length;
|
|
if (afterStart === beforeStart) {
|
|
note("flow", "start-instance", "click did not fire POST /api/runtime/transactions");
|
|
} else {
|
|
note("ok", "start-instance", `POST /api/runtime/transactions fired ${afterStart - beforeStart} time(s)`);
|
|
}
|
|
await p.screenshot({ path: `${OUT}/f1-07-after-start.png` });
|
|
}
|
|
|
|
// F1.f try to fire Submit on the inspector overview
|
|
await p.locator(".qcard").first().click().catch(() => {});
|
|
await p.waitForTimeout(400);
|
|
await p.locator(".itab", { hasText: /overview/i }).click().catch(() => {});
|
|
await p.waitForTimeout(200);
|
|
const inspectorBtns = await p.locator(".i-actions .btn").count();
|
|
if (inspectorBtns > 0) {
|
|
await p.locator(".i-actions .btn").first().click();
|
|
await p.waitForTimeout(2000);
|
|
await p.screenshot({ path: `${OUT}/f1-08-after-submit.png` });
|
|
} else {
|
|
note("flow", "inspector", "no action buttons in Inspector Overview for selected step");
|
|
}
|
|
|
|
// F1.g try the Process Studio publish path
|
|
await p.locator(".tab", { hasText: /studio/i }).click();
|
|
await p.waitForSelector(".studio"); await p.waitForTimeout(500);
|
|
await p.screenshot({ path: `${OUT}/f1-09-studio.png` });
|
|
const publishBtn = p.locator(".studio-head .btn", { hasText: /publish/i });
|
|
if (await publishBtn.count() > 0) {
|
|
const beforePublish = tApiCalls.filter((c) => c.url.includes("/api/ea2/flow")).length;
|
|
await publishBtn.click();
|
|
await p.waitForTimeout(3000);
|
|
const afterPublish = tApiCalls.filter((c) => c.url.includes("/api/ea2/flow")).length;
|
|
if (afterPublish === beforePublish) {
|
|
note("flow", "studio-publish", "Publish click did not fire POST /api/ea2/flow");
|
|
} else {
|
|
note("ok", "studio-publish", `POST /api/ea2/flow fired ${afterPublish - beforePublish} time(s)`);
|
|
}
|
|
await p.screenshot({ path: `${OUT}/f1-10-after-publish.png` });
|
|
}
|
|
|
|
// F1.h check Run History
|
|
await p.locator(".tab", { hasText: /runs/i }).click();
|
|
await p.waitForSelector(".rh"); await p.waitForTimeout(300);
|
|
const rhRows = await p.locator(".rh-row").count();
|
|
if (rhRows === 0) note("flow", "run-history", "RunHistory shows 0 rows even in live mode");
|
|
await p.screenshot({ path: `${OUT}/f1-11-runs.png` });
|
|
|
|
// F1.i ⌘K palette
|
|
await p.keyboard.press("Meta+k");
|
|
await p.waitForSelector(".cmd").catch(() => note("flow", "cmd-palette", "⌘K did not open palette"));
|
|
await p.waitForTimeout(200);
|
|
await p.screenshot({ path: `${OUT}/f1-12-palette.png` });
|
|
await p.keyboard.press("Escape");
|
|
await ctx.close();
|
|
}
|
|
|
|
// =========================================================================
|
|
// FLOW 2 — mobile viewport (regional store manager on a phone)
|
|
// =========================================================================
|
|
{
|
|
console.log("\n=== FLOW 2: mobile (iPhone 14 Pro) ===");
|
|
const ctx = await browser.newContext({ ...devices["iPhone 14 Pro"], ignoreHTTPSErrors: true });
|
|
const p = await ctx.newPage();
|
|
await attach(p, "F2-mobile");
|
|
await p.goto(URL, { waitUntil: "networkidle" });
|
|
await p.screenshot({ path: `${OUT}/f2-01-mobile-landing.png` });
|
|
const horizontalScroll = await p.evaluate(() => document.documentElement.scrollWidth > window.innerWidth + 2);
|
|
if (horizontalScroll) note("mobile", "landing", "horizontal scroll present on mobile — layout overflow");
|
|
const cards = await p.locator(".sc-card").count();
|
|
if (cards === 0) note("mobile", "landing", "0 scenario cards visible on mobile");
|
|
if (cards > 0) {
|
|
await p.locator(".sc-card").first().click();
|
|
await p.waitForTimeout(800);
|
|
await p.screenshot({ path: `${OUT}/f2-02-mobile-mission.png` });
|
|
const mcVisible = await p.locator(".mc-body").isVisible();
|
|
if (!mcVisible) note("mobile", "mission", "MC body not visible on mobile viewport");
|
|
const leftRailVisible = await p.locator(".left-rail").isVisible();
|
|
if (!leftRailVisible) note("mobile", "mission", "left rail invisible at mobile width (acceptable but worth noting)");
|
|
}
|
|
await ctx.close();
|
|
}
|
|
|
|
// =========================================================================
|
|
// FLOW 3 — reload + deep state check (do we lose mode/scenario across reload?)
|
|
// =========================================================================
|
|
{
|
|
console.log("\n=== FLOW 3: persistence across reload ===");
|
|
const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, ignoreHTTPSErrors: true });
|
|
const p = await ctx.newPage();
|
|
await attach(p, "F3");
|
|
await p.goto(URL, { waitUntil: "networkidle" });
|
|
await p.locator(".sc-card").nth(3).click();
|
|
await p.waitForSelector(".mc"); await p.waitForTimeout(500);
|
|
const titleBefore = await p.locator(".mc-hero-title").innerText();
|
|
await p.reload({ waitUntil: "networkidle" });
|
|
await p.waitForTimeout(500);
|
|
// The shell defaults to Landing on reload — is that intentional?
|
|
const landingBack = await p.locator(".landing").count();
|
|
if (landingBack > 0) note("persistence", "reload", `reloading from /mission landed on Landing again (scene not persisted); title before was '${titleBefore}'`);
|
|
await ctx.close();
|
|
}
|
|
|
|
// =========================================================================
|
|
// FLOW 4 — security headers (curl-equivalent via fetch)
|
|
// =========================================================================
|
|
{
|
|
console.log("\n=== FLOW 4: security headers ===");
|
|
const ctx = await browser.newContext({ ignoreHTTPSErrors: true });
|
|
const p = await ctx.newPage();
|
|
const resp = await p.goto(URL);
|
|
const h = resp?.headers() ?? {};
|
|
const required = [
|
|
"strict-transport-security",
|
|
"x-content-type-options",
|
|
"x-frame-options",
|
|
"referrer-policy",
|
|
"content-security-policy",
|
|
];
|
|
for (const k of required) {
|
|
if (!h[k]) note("security", "headers", `missing ${k}`);
|
|
else note("ok", "headers", `${k}: ${h[k].slice(0, 80)}`);
|
|
}
|
|
await ctx.close();
|
|
}
|
|
|
|
await browser.close();
|
|
|
|
// Write the findings dump
|
|
writeFileSync(`${OUT}/findings.json`, JSON.stringify(findings, null, 2));
|
|
const byKind = findings.reduce((acc, f) => { acc[f.kind] = (acc[f.kind] || 0) + 1; return acc; }, {});
|
|
console.log("\n=== SUMMARY ===");
|
|
console.log(JSON.stringify(byKind, null, 2));
|
|
console.log(`\n→ ${findings.length} findings written to ${OUT}/findings.json`);
|
|
console.log(`→ screenshots in ${OUT}/`);
|