import { existsSync } from "node:fs"; import { readFile, readdir, realpath } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { LayoutRole } from "../../packages/definitions/index.ts"; import { boardNameSchema } from "../../packages/validation/src/board.ts"; import { findFixtureGeometryErrors } from "./geometry.mts"; import { resolveCheckoutCandidateSha } from "./provenance.mts"; import { collectHumanWidgetStatusErrors, normalizeReportMetadata } from "./report-integrity.mts"; import { assertSafeReportPath, readSafeReportFile, validateResolvedArtifactPath } from "./report-path-integrity.mts"; import { validateRuntimeExecutionContract } from "./runtime.mts"; import { assertSafeRunRoot, isPathWithin, validateFixtureUrl } from "./safety.mts"; type Status = "passed" | "failed" | "blocked" | "not-reached"; type Severity = "P0" | "P1" | "P2" | "P3"; interface Finding { severity: Severity; title: string; caseIds?: string[]; evidence?: string[]; } interface Packet { id: string; wave: string; prRefs: number[]; personas: string[]; boards: string[]; profiles: string[]; viewports: string[]; zooms: number[]; inputs: string[]; widgetKinds?: string[]; cases: string[]; } interface Manifest { statuses: Status[]; pullRequests: { number: number }[]; viewports: { id: string }[]; zooms: number[]; inputs: string[]; profiles: { id: string }[]; personas: string[]; boards: { id: string; handle: string }[]; wavePacketCounts: Record; caseDimensions: Record; packets: Packet[]; } interface ReportMetadata { packetId: string; status: Status; caseStatuses: Record; execution: { candidateSha: string | null; url: string | null; actualPort: number | null; runtimeProfile: string | null; runtimeFlags: string[]; persona: string | null; sessionId: string | null; timestamp: string | null; viewport: string | null; input: string | null; zoom: number | null; }; findings: Finding[]; artifacts: string[]; widgetChecks: { widgetKind: string; viewport: string; sizeRequirement: string; status: Status; }[]; performance: { measurements: { name: string; value: number | null; unit: string; threshold: string; status: Status; evidence: string[]; }[]; limitations: string[]; }; independentReproductions: { findingFingerprint: string; agentId: string; outcome: "reproduced" | "not-reproduced" | "blocked" | "not-reached"; evidence: string[]; notes: string; }[]; notes: string; } interface Ledger { packetStatuses: Record }>; } interface RuntimeManifest { schemaVersion: number; runId: string; status: "running" | "stopped"; candidateSha: string; slot: number; profile: string; repoRoot: string; runRoot: string; slotDir: string; dbPath: string; fixtureManifestPath: string; nextDistDir: string; url: string; fixtureUrl: string; ports: { app: number; fixture: number; redis: number }; flags: { demoMode: boolean; demoReadOnly: boolean; unsafeMockIntegration: boolean }; bundler: string; watcher: { watchpackPollingIntervalMs: number }; processes: { app: { pid: number; logPath: string }; fixture: { pid: number; logPath: string }; }; } interface FixtureManifest { schemaVersion: number; candidateSha: string; profile: string; personas: { id: string; handle: string; name: string; loginUsername: string }[]; boards: { id: string; handle: string; name: string; label: string; layouts: string[]; flags: { shared: boolean } }[]; layouts: { id: string; boardId: string; columnCount: number }[]; expectedBoardPermissions: Record; counts: Record; flags: { demoMode: boolean; demoReadOnly: boolean; unsafeMockIntegration: boolean; allWidgetKindsCovered: boolean; fixtureOrigin: string; }; } interface VerifyOptions { runRoot?: string; manifestPaths: string[]; } const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const qaRoot = join(repoRoot, "qa/release-v2"); const reportsRoot = join(qaRoot, "reports"); const campaignCandidateSha = resolveCheckoutCandidateSha(repoRoot); const exhaustiveSizeWidgetKinds = new Set([ "downloads", "calendar", "mediaServer", "mediaRequests-requestList", "mediaRequests-requestStats", "systemResources", "systemDisks", "dockerContainers", "networkControllerSummary", "networkControllerStatus", "beszelSystemStats", "uptimeKuma", "bookmarks", "notebook", "iframe", "customApi", "assistant", ]); const expectedWidgetGroups: Record = { "widgets-01": ["clock", "weather", "airQuality", "countdown", "timer"], "widgets-02": ["app", "iframe", "video", "minecraftServerStatus", "stockPrice"], "widgets-03": ["notebook", "anchorNote", "bookmarks", "rssFeed", "timetable"], "widgets-04": ["downloads", "dockerContainers", "indexerManager", "dnsHoleSummary", "dnsHoleControls"], "widgets-05": [ "smartHome-entityState", "smartHome-executeAutomation", "healthMonitoring", "systemResources", "systemDisks", ], "widgets-06": ["firewall", "notifications", "networkControllerSummary", "networkControllerStatus", "uptimeKuma"], "widgets-07": ["beszelSystemTable", "beszelSystemGrid", "beszelAlerts", "beszelSystemStats", "wud"], "widgets-08": ["ups", "vpn", "speedtestTracker", "traefik", "umami"], "widgets-09": ["calendar", "mediaServer", "mediaRequests-requestList", "mediaRequests-requestStats", "mediaMissing"], "widgets-10": ["mediaReleases", "mediaTranscoding", "immich-serverStats", "immich-albumCarousel", "audioStats"], "widgets-11": ["paperlessNgx", "patchmon", "bazarr", "tracearr", "releases"], "widgets-12": ["coolify", "archiveTeamWarrior", "customApi", "assistant"], }; const expectedProfileFlags: Record = { "main-writable": ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], "main-readonly": ["DEMO_MODE=true", "DEMO_READ_ONLY=true", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], "onboarding-fresh": ["DEMO_MODE=false", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], degraded: ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], }; const errors: string[] = []; const parseOptions = (): VerifyOptions => { const args = process.argv.slice(2).filter((argument) => argument !== "--"); const options: VerifyOptions = { manifestPaths: [] }; for (let index = 0; index < args.length; index += 1) { const argument = args[index]; const value = args[index + 1]; if (argument === "--run-root" && value) { options.runRoot = resolve(value); index += 1; continue; } if ((argument === "--manifest" || argument === "--runtime-manifest") && value) { options.manifestPaths.push(resolve(value)); index += 1; continue; } throw new Error(`Unknown or incomplete argument: ${argument ?? ""}`); } return options; }; const readJson = async (path: string): Promise => JSON.parse(await readFile(path, "utf8")) as T; const sameSet = (actual: T[], expected: T[]): boolean => actual.length === expected.length && actual.every((value) => expected.includes(value)); const requiredSizeInstruction = (widgetKind: string) => { if (exhaustiveSizeWidgetKinds.has(widgetKind)) return "every width 1-24 × every height 1-6"; return "minimum / canonical / wide / tall / maximum / overflow / behavior-changing thresholds"; }; const duplicateValues = (values: T[]): T[] => values.filter((value, index) => values.indexOf(value) !== index); const expandedCases = (manifest: Manifest, packet: Packet): { id: string; expected: string }[] => { const dimensions = manifest.caseDimensions[packet.wave] ?? []; return packet.cases.flatMap((baseId) => dimensions.map((dimension) => ({ id: `${baseId}-${dimension.id}`, expected: dimension.expected })), ); }; const expectedRollup = (statuses: Status[]): Status => { if (statuses.includes("failed")) return "failed"; if (statuses.includes("blocked")) return "blocked"; if (statuses.includes("not-reached")) return "not-reached"; return "passed"; }; const parseReportMetadata = (content: string, packetId: string): unknown | undefined => { const match = content.match(//); if (!match?.[1]) { errors.push(`${packetId}: report metadata block is missing`); return undefined; } try { return JSON.parse(match[1]) as unknown; } catch (error) { errors.push(`${packetId}: report metadata is invalid JSON (${String(error)})`); return undefined; } }; const assertReferences = (packet: Packet, field: keyof Packet, values: T[], allowed: T[]): void => { for (const value of values) { if (!allowed.includes(value)) errors.push(`${packet.id}: unknown ${String(field)} value ${String(value)}`); } }; const booleanProfileFlags = (profile: string) => ({ demoMode: profile !== "onboarding-fresh", demoReadOnly: profile === "main-readonly", unsafeMockIntegration: true, }); const validateFixtureEndpoint = async (fixtureUrl: string, label: string) => { try { const response = await fetch(`${fixtureUrl}/health`, { signal: AbortSignal.timeout(5_000) }); if (!response.ok) { errors.push(`${label}: fixture health returned HTTP ${response.status}`); return; } const body = (await response.json()) as { status?: string; fixture?: string }; if (body.status !== "ok" || body.fixture !== "release-v2-qa") { errors.push(`${label}: fixture health payload is not the release-v2 QA fixture`); } } catch { errors.push(`${label}: fixture health endpoint is unreachable`); } }; const validateFixtureManifest = async ( fixture: FixtureManifest, fixturePath: string, sourceWidgetKinds: string[], checkEndpoint: boolean, expectedSharedBoardNames: Map, ) => { const label = `fixture manifest ${fixturePath}`; if (fixture.schemaVersion !== 1) errors.push(`${label}: unsupported schema version`); if (fixture.candidateSha !== campaignCandidateSha) errors.push(`${label}: candidate does not match the campaign`); if (!expectedProfileFlags[fixture.profile]) errors.push(`${label}: unknown profile ${fixture.profile}`); const expectedFlags = booleanProfileFlags(fixture.profile); for (const [flag, expected] of Object.entries(expectedFlags)) { if (fixture.flags?.[flag as keyof typeof expectedFlags] !== expected) { errors.push(`${label}: ${flag} does not match profile ${fixture.profile}`); } } if (!Array.isArray(fixture.personas) || fixture.counts?.personas !== fixture.personas.length) { errors.push(`${label}: persona count does not match the persona manifest`); } if (!Array.isArray(fixture.boards) || fixture.counts?.boards !== fixture.boards.length) { errors.push(`${label}: board count does not match the board manifest`); } if (!Array.isArray(fixture.layouts) || fixture.counts?.layouts !== fixture.layouts.length) { errors.push(`${label}: layout count does not match the layout manifest`); } if (duplicateValues((fixture.personas ?? []).map((persona) => persona.id)).length > 0) { errors.push(`${label}: duplicate persona IDs`); } if (duplicateValues((fixture.personas ?? []).map((persona) => persona.loginUsername)).length > 0) { errors.push(`${label}: duplicate persona login usernames`); } if (duplicateValues((fixture.boards ?? []).map((board) => board.id)).length > 0) { errors.push(`${label}: duplicate board IDs`); } if (duplicateValues((fixture.boards ?? []).map((board) => board.name)).length > 0) { errors.push(`${label}: duplicate board names`); } if (duplicateValues((fixture.layouts ?? []).map((layout) => layout.id)).length > 0) { errors.push(`${label}: duplicate layout IDs`); } const boardIds = new Set((fixture.boards ?? []).map((board) => board.id)); const layoutIds = new Set((fixture.layouts ?? []).map((layout) => layout.id)); for (const layout of fixture.layouts ?? []) { if (!boardIds.has(layout.boardId)) errors.push(`${label}: layout ${layout.id} references an unknown board`); if (!Number.isInteger(layout.columnCount) || layout.columnCount < 1 || layout.columnCount > 24) { errors.push(`${label}: layout ${layout.id} has an invalid column count`); } } for (const board of fixture.boards ?? []) { if (!boardNameSchema.safeParse(board.name).success) { errors.push(`${label}: board ${board.id} has an invalid board name ${String(board.name)}`); } if (!board.label?.trim()) errors.push(`${label}: board ${board.id} is missing its human-readable label`); if (board.flags.shared && board.name !== expectedSharedBoardNames.get(board.handle)) { errors.push(`${label}: shared board ${board.handle} name differs from the coverage manifest`); } if (board.layouts.some((layoutId) => !layoutIds.has(layoutId))) { errors.push(`${label}: board ${board.id} references an unknown layout`); } } const populated = fixture.profile !== "onboarding-fresh"; if (populated) { if (fixture.personas.length !== 14) errors.push(`${label}: expected 14 seeded personas`); for (const persona of fixture.personas) { if (!persona.name) errors.push(`${label}: persona ${persona.id} is missing its human-readable name`); if (persona.loginUsername !== persona.handle || persona.loginUsername !== persona.loginUsername.toLowerCase()) { errors.push(`${label}: persona ${persona.id} has an invalid login username`); } } if (fixture.boards.length !== 35) errors.push(`${label}: expected 35 seeded boards`); if (fixture.counts.widgetKinds !== sourceWidgetKinds.length || !fixture.flags.allWidgetKindsCovered) { errors.push(`${label}: widget-kind coverage does not match the canonical source`); } if (fixture.counts.apps !== 8 || fixture.counts.icons !== 8 || fixture.counts.customWidgets !== 3) { errors.push(`${label}: fixture object counts are incomplete`); } const expectedPermissions = { "rowan-owner": "full", "eden-editor": "modify", "vivian-viewer": "view", "nolan-outsider": "none", }; if (JSON.stringify(fixture.expectedBoardPermissions) !== JSON.stringify(expectedPermissions)) { errors.push(`${label}: expected board permissions are incorrect`); } } else { const emptyFixtureCounts = [ "personas", "boards", "homeBoards", "sharedBoards", "layouts", "items", "widgetKinds", "apps", "icons", "customWidgets", ]; const hasPopulatedFixtureCount = emptyFixtureCounts.some((countName) => fixture.counts[countName] !== 0); if ( fixture.personas.length !== 0 || fixture.boards.length !== 0 || fixture.layouts.length !== 0 || hasPopulatedFixtureCount || fixture.flags.allWidgetKindsCovered ) { errors.push(`${label}: onboarding-fresh must not contain populated QA fixtures`); } } let fixtureOrigin: URL | undefined; try { fixtureOrigin = new URL(validateFixtureUrl(fixture.flags.fixtureOrigin)); } catch { errors.push(`${label}: fixture origin must be credential-free loopback HTTP`); } if (checkEndpoint && fixtureOrigin) await validateFixtureEndpoint(fixtureOrigin.origin, label); }; const validateSqliteState = async (runtime: RuntimeManifest, fixture: FixtureManifest, sourceWidgetKinds: string[]) => { const label = `runtime slot ${runtime.slot}`; if (!existsSync(runtime.dbPath)) { errors.push(`${label}: SQLite database is missing`); return; } const sqliteModule = await import("better-sqlite3"); type DatabaseLike = { prepare: (sql: string) => { get: (...parameters: unknown[]) => unknown; all: (...parameters: unknown[]) => unknown[]; }; close: () => void; }; const Database = sqliteModule.default as unknown as new ( filename: string, options: { readonly: boolean; fileMustExist: boolean }, ) => DatabaseLike; const database = new Database(runtime.dbPath, { readonly: true, fileMustExist: true }); const count = (sql: string, ...parameters: unknown[]) => (database.prepare(sql).get(...parameters) as { count: number }).count; try { const qaUserCount = count(`select count(*) as count from "user" where id like 'qa-v2-user-%'`); const qaBoardCount = count(`select count(*) as count from board where id like 'qa-v2-board-%'`); const qaLayoutCount = count(`select count(*) as count from layout where board_id like 'qa-v2-board-%'`); const qaItemCount = count(`select count(*) as count from item where board_id like 'qa-v2-board-%'`); if (qaUserCount !== fixture.counts.personas) errors.push(`${label}: SQLite persona count differs from manifest`); if (qaBoardCount !== fixture.counts.boards) errors.push(`${label}: SQLite board count differs from manifest`); if (qaLayoutCount !== fixture.counts.layouts) errors.push(`${label}: SQLite layout count differs from manifest`); if (qaItemCount !== fixture.counts.items) errors.push(`${label}: SQLite item count differs from manifest`); const populated = fixture.profile !== "onboarding-fresh"; if (!populated) { const qaFixtureCounts = [ count(`select count(*) as count from "group" where id like 'qa-v2-%'`), count(`select count(*) as count from app where id like 'qa-v2-%'`), count(`select count(*) as count from icon where id like 'qa-v2-%'`), count(`select count(*) as count from integration where id like 'qa-v2-%'`), count(`select count(*) as count from custom_widget_v2_definition where id like 'qa-v2-%'`), ]; if (qaFixtureCounts.some((fixtureCount) => fixtureCount !== 0)) { errors.push(`${label}: onboarding-fresh contains QA-scoped fixture records`); } if (count(`select count(*) as count from "user"`) !== 0) { errors.push(`${label}: onboarding-fresh must not contain users`); } const onboardingRows = database.prepare(`select step, previous_step as previousStep from onboarding`).all() as { step: string; previousStep: string | null; }[]; if ( onboardingRows.length !== 1 || onboardingRows[0]?.step !== "start" || onboardingRows[0]?.previousStep !== null ) { errors.push(`${label}: onboarding-fresh is not at the initial onboarding step`); } } const qaUsers = database.prepare(`select id, name from "user" where id like 'qa-v2-user-%'`).all() as { id: string; name: string; }[]; const loginUsernameById = new Map(fixture.personas.map((persona) => [persona.id, persona.loginUsername])); for (const user of qaUsers) { if (user.name !== loginUsernameById.get(user.id)) { errors.push(`${label}: SQLite persona ${user.id} login username differs from manifest`); } } const qaBoards = database.prepare(`select id, name from board where id like 'qa-v2-board-%'`).all() as { id: string; name: string; }[]; const boardNameById = new Map(fixture.boards.map((board) => [board.id, board.name])); if (duplicateValues(qaBoards.map((board) => board.name)).length > 0) { errors.push(`${label}: SQLite contains duplicate QA board names`); } for (const board of qaBoards) { if (!boardNameSchema.safeParse(board.name).success) { errors.push(`${label}: SQLite board ${board.id} has an invalid board name ${board.name}`); } if (board.name !== boardNameById.get(board.id)) { errors.push(`${label}: SQLite board ${board.id} name differs from manifest`); } } const missingPlacements = count( `select count(*) as count from item i join layout l on l.board_id = i.board_id left join item_layout il on il.item_id = i.id and il.layout_id = l.id where i.board_id like 'qa-v2-board-%' and il.item_id is null`, ); if (missingPlacements !== 0) errors.push(`${label}: ${missingPlacements} item/layout placements are missing`); const geometryLayouts = database .prepare( `select id, board_id as boardId, column_count as columnCount, left_gutter_column_count as leftGutterColumnCount, right_gutter_column_count as rightGutterColumnCount, role from layout where board_id like 'qa-v2-board-%'`, ) .all() as { id: string; boardId: string; columnCount: number; leftGutterColumnCount: number; rightGutterColumnCount: number; role: LayoutRole; }[]; const geometrySections = database .prepare( `select id, board_id as boardId, kind, x_offset as xOffset from section where board_id like 'qa-v2-board-%'`, ) .all() as { id: string; boardId: string; kind: "empty" | "container"; xOffset: number | null; }[]; const geometrySectionLayouts = database .prepare( `select sl.section_id as sectionId, sl.layout_id as layoutId, sl.parent_section_id as parentSectionId, sl.x_offset as xOffset, sl.y_offset as yOffset, sl.width, sl.height from section_layout sl join layout l on l.id = sl.layout_id where l.board_id like 'qa-v2-board-%'`, ) .all() as { sectionId: string; layoutId: string; parentSectionId: string | null; xOffset: number; yOffset: number; width: number; height: number; }[]; const geometryItemLayouts = database .prepare( `select il.item_id as itemId, i.board_id as boardId, il.section_id as sectionId, il.layout_id as layoutId, il.x_offset as xOffset, il.y_offset as yOffset, il.width, il.height from item_layout il join item i on i.id = il.item_id where i.board_id like 'qa-v2-board-%'`, ) .all() as { itemId: string; boardId: string; sectionId: string; layoutId: string; xOffset: number; yOffset: number; width: number; height: number; }[]; const geometryErrors = findFixtureGeometryErrors({ layouts: geometryLayouts, sections: geometrySections, sectionLayouts: geometrySectionLayouts, itemLayouts: geometryItemLayouts, }); if (geometryErrors.length > 0) { errors.push(`${label}: invalid fixture geometry (${geometryErrors.join("; ")})`); } if (populated) { const actualWidgetKinds = database .prepare(`select distinct kind from item where board_id like 'qa-v2-board-widgets-%' order by kind`) .all() .map((row) => (row as { kind: string }).kind); if (!sameSet(actualWidgetKinds, sourceWidgetKinds)) { errors.push(`${label}: widget gallery does not contain every canonical widget kind exactly once`); } for (const board of fixture.boards.filter((candidate) => candidate.flags.shared)) { for (const [handle, permission] of Object.entries(fixture.expectedBoardPermissions)) { const permissionCount = count( `select count(*) as count from boardUserPermission where board_id = ? and user_id = ? and permission = ?`, board.id, `qa-v2-user-${handle}`, permission, ); if (permission === "none") { const anyPermissionCount = count( `select count(*) as count from boardUserPermission where board_id = ? and user_id = ?`, board.id, `qa-v2-user-${handle}`, ); if (anyPermissionCount !== 0) errors.push(`${label}: ${handle} unexpectedly has access to ${board.handle}`); continue; } if (permissionCount !== 1) errors.push(`${label}: ${handle} lacks ${permission} access to ${board.handle}`); } } } } finally { database.close(); } }; const validateRuntimeManifest = async ( runtime: RuntimeManifest, manifestPath: string, sourceWidgetKinds: string[], expectedSharedBoardNames: Map, ) => { const label = `runtime manifest ${manifestPath}`; if (runtime.schemaVersion !== 1) errors.push(`${label}: unsupported schema version`); if (runtime.candidateSha !== campaignCandidateSha) errors.push(`${label}: candidate does not match the campaign`); if (![1, 2, 3].includes(runtime.slot)) errors.push(`${label}: slot must be 1, 2, or 3`); if (!expectedProfileFlags[runtime.profile]) errors.push(`${label}: unknown profile ${runtime.profile}`); const expectedFlags = booleanProfileFlags(runtime.profile); if (JSON.stringify(runtime.flags) !== JSON.stringify(expectedFlags)) { errors.push(`${label}: runtime flags do not match profile ${runtime.profile}`); } for (const executionError of validateRuntimeExecutionContract(runtime)) { errors.push(`${label}: ${executionError}`); } if (resolve(runtime.repoRoot) !== repoRoot) errors.push(`${label}: repoRoot does not match this checkout`); try { await assertSafeRunRoot(runtime.runRoot); } catch { errors.push(`${label}: runRoot is not a safe release-v2 QA temporary root`); } const expectedSlotDir = join(runtime.runRoot, "slots", String(runtime.slot)); const expectedManifestPath = join(expectedSlotDir, "runtime-manifest.json"); const expectedDbPath = join(expectedSlotDir, "db.sqlite"); const expectedFixtureManifestPath = join(expectedSlotDir, "fixture-manifest.json"); const expectedNextDistDir = join(repoRoot, "apps/nextjs/.next-qa", `slot-${runtime.slot}`); if (resolve(runtime.slotDir) !== resolve(expectedSlotDir)) errors.push(`${label}: slotDir is outside the selected slot`); if (resolve(manifestPath) !== resolve(expectedManifestPath)) errors.push(`${label}: runtime manifest is outside the selected slot`); if (resolve(runtime.dbPath) !== resolve(expectedDbPath)) errors.push(`${label}: dbPath is outside the selected slot`); if (resolve(runtime.fixtureManifestPath) !== resolve(expectedFixtureManifestPath)) { errors.push(`${label}: fixture manifest path is outside the selected slot`); } if (resolve(runtime.nextDistDir) !== resolve(expectedNextDistDir)) { errors.push(`${label}: Next dist directory is not isolated to the selected slot`); } for (const processRecord of Object.values(runtime.processes ?? {})) { if (!isPathWithin(expectedSlotDir, resolve(processRecord.logPath))) { errors.push(`${label}: process log path escapes the selected slot`); } } try { if ((await realpath(runtime.runRoot)) !== resolve(runtime.runRoot)) errors.push(`${label}: runRoot must not be a symlink`); if ((await realpath(runtime.slotDir)) !== resolve(runtime.slotDir)) errors.push(`${label}: slotDir must not be a symlink`); } catch { errors.push(`${label}: runRoot or slotDir is missing`); } let appUrl: URL | undefined; let fixtureUrl: URL | undefined; try { appUrl = new URL(runtime.url); fixtureUrl = new URL(runtime.fixtureUrl); } catch { errors.push(`${label}: runtime URLs are invalid`); } for (const [name, url] of [ ["app", appUrl], ["fixture", fixtureUrl], ] as const) { if (!url) continue; try { validateFixtureUrl(url.href); } catch { errors.push(`${label}: ${name} URL must be credential-free loopback HTTP`); } } if (appUrl && Number(appUrl.port) !== runtime.ports.app) errors.push(`${label}: app URL and port differ`); if (fixtureUrl && Number(fixtureUrl.port) !== runtime.ports.fixture) errors.push(`${label}: fixture URL and port differ`); if (!existsSync(runtime.fixtureManifestPath)) { errors.push(`${label}: fixture manifest is missing`); return; } const fixture = await readJson(runtime.fixtureManifestPath); if (fixture.profile !== runtime.profile) errors.push(`${label}: runtime and fixture profiles differ`); if (fixture.candidateSha !== runtime.candidateSha) errors.push(`${label}: runtime and fixture candidates differ`); await validateFixtureManifest( fixture, runtime.fixtureManifestPath, sourceWidgetKinds, runtime.status === "running", expectedSharedBoardNames, ); await validateSqliteState(runtime, fixture, sourceWidgetKinds); if (runtime.status === "running" && appUrl) { try { const response = await fetch(`${appUrl.origin}/api/health/ready`, { signal: AbortSignal.timeout(5_000) }); if (!response.ok) errors.push(`${label}: application readiness returned HTTP ${response.status}`); } catch { errors.push(`${label}: application readiness endpoint is unreachable`); } } }; const validateRequestedRuntime = async ( options: VerifyOptions, sourceWidgetKinds: string[], expectedSharedBoardNames: Map, ) => { const manifestPaths = [...options.manifestPaths]; if (options.runRoot) { try { options.runRoot = await assertSafeRunRoot(options.runRoot); } catch { errors.push(`run-root is not a safe release-v2 QA temporary root: ${options.runRoot}`); } const markerPath = join(options.runRoot, ".homarr-release-v2-qa-root.json"); if (!existsSync(markerPath)) { errors.push(`run-root marker is missing: ${markerPath}`); } else { const marker = await readJson<{ schemaVersion?: number; repoRoot?: string; runRoot?: string }>(markerPath); if (marker.schemaVersion !== 1 || resolve(marker.repoRoot ?? "") !== repoRoot) { errors.push(`run-root marker does not belong to this checkout: ${markerPath}`); } if (marker.runRoot && resolve(marker.runRoot) !== options.runRoot) { errors.push(`run-root marker records a different root: ${markerPath}`); } } const slotsRoot = join(options.runRoot, "slots"); if (existsSync(slotsRoot)) { const slotEntries = await readdir(slotsRoot, { withFileTypes: true }); for (const entry of slotEntries) { if (!entry.isDirectory() || !["1", "2", "3"].includes(entry.name)) { errors.push(`run-root contains an unexpected slot entry: ${entry.name}`); continue; } const runtimePath = join(slotsRoot, entry.name, "runtime-manifest.json"); if (!existsSync(runtimePath)) { errors.push(`slot ${entry.name} contains incomplete state without a runtime manifest`); continue; } manifestPaths.push(runtimePath); } } } const uniqueManifestPaths = new Set(manifestPaths); if (options.runRoot && uniqueManifestPaths.size === 0) { errors.push(`run-root contains no runtime manifests: ${options.runRoot}`); } for (const manifestPath of uniqueManifestPaths) { const value = await readJson(manifestPath); if ("runId" in value) { await validateRuntimeManifest(value, manifestPath, sourceWidgetKinds, expectedSharedBoardNames); continue; } await validateFixtureManifest(value, manifestPath, sourceWidgetKinds, true, expectedSharedBoardNames); } }; const main = async (): Promise => { const options = parseOptions(); const manifest = await readJson(join(qaRoot, "coverage-manifest.json")); const ledgerPath = join(qaRoot, "ledger.json"); await assertSafeReportPath(qaRoot, ledgerPath, "release-v2 QA ledger path"); await assertSafeReportPath(qaRoot, join(qaRoot, "master-report.md"), "release-v2 QA master report path"); const ledger = JSON.parse(await readSafeReportFile(qaRoot, ledgerPath, "release-v2 QA ledger path")) as Ledger; const allowedStatuses: Status[] = ["passed", "failed", "blocked", "not-reached"]; const expectedCounts = { preflight: 3, board: 9, widgets: 12, "core-v2": 8, "whole-product": 9, performance: 4 }; if (!sameSet(manifest.statuses, allowedStatuses)) errors.push("manifest statuses do not match the required status vocabulary"); if (manifest.packets.length !== 45) errors.push(`expected 45 packets, found ${manifest.packets.length}`); if (JSON.stringify(manifest.wavePacketCounts) !== JSON.stringify(expectedCounts)) errors.push("wavePacketCounts is not 3/9/12/8/9/4"); const packetIds = manifest.packets.map((packet) => packet.id); for (const duplicate of new Set(duplicateValues(packetIds))) errors.push(`duplicate packet id ${duplicate}`); for (const wave of Object.keys(expectedCounts)) { const dimensions = manifest.caseDimensions[wave] ?? []; if (dimensions.length < 3) errors.push(`${wave}: expected at least three explicit case dimensions`); if (duplicateValues(dimensions.map((dimension) => dimension.id)).length > 0) errors.push(`${wave}: duplicate case dimension IDs`); if (dimensions.some((dimension) => dimension.expected.trim().length === 0)) errors.push(`${wave}: empty case expectation`); } const allCases = manifest.packets.flatMap((packet) => expandedCases(manifest, packet).map(({ id }) => id)); for (const duplicate of new Set(duplicateValues(allCases))) errors.push(`duplicate case id ${duplicate}`); const prNumbers = manifest.pullRequests.map((pullRequest) => pullRequest.number); const profileIds = manifest.profiles.map((profile) => profile.id); const viewportIds = manifest.viewports.map((viewport) => viewport.id); const boardIds = manifest.boards.map((board) => board.id); const boardHandles = manifest.boards.map((board) => board.handle); for (const duplicate of new Set(duplicateValues(boardIds))) errors.push(`duplicate board name ${duplicate}`); for (const duplicate of new Set(duplicateValues(boardHandles))) errors.push(`duplicate board handle ${duplicate}`); for (const board of manifest.boards) { if (!boardNameSchema.safeParse(board.id).success) errors.push(`invalid coverage board name ${board.id}`); } const expectedSharedBoardNames = new Map(manifest.boards.map((board) => [board.handle, board.id])); for (const [wave, expected] of Object.entries(expectedCounts)) { const actual = manifest.packets.filter((packet) => packet.wave === wave).length; if (actual !== expected) errors.push(`${wave}: expected ${expected} packets, found ${actual}`); } for (const packet of manifest.packets) { if (packet.cases.length === 0 || expandedCases(manifest, packet).length < 3) errors.push(`${packet.id}: insufficient cases assigned`); assertReferences(packet, "prRefs", packet.prRefs, prNumbers); assertReferences(packet, "personas", packet.personas, manifest.personas); assertReferences(packet, "boards", packet.boards, boardIds); assertReferences(packet, "profiles", packet.profiles, profileIds); assertReferences(packet, "viewports", packet.viewports, viewportIds); assertReferences(packet, "zooms", packet.zooms, manifest.zooms); assertReferences(packet, "inputs", packet.inputs, manifest.inputs); if (packet.widgetKinds) { const expectedGroup = expectedWidgetGroups[packet.id]; if (!expectedGroup || JSON.stringify(packet.widgetKinds) !== JSON.stringify(expectedGroup)) { errors.push(`${packet.id}: widget allocation does not match the required group`); } if (JSON.stringify(packet.viewports) !== JSON.stringify(viewportIds)) { errors.push(`${packet.id}: every widget kind must cover all ${viewportIds.length} required viewports`); } } } for (const prNumber of prNumbers) { if (!manifest.packets.some((packet) => packet.prRefs.includes(prNumber))) errors.push(`PR #${prNumber} has no packet coverage`); } const widgetSource = await readFile(join(repoRoot, "packages/definitions/src/widget.ts"), "utf8"); const widgetBlock = widgetSource.match(/export const widgetKinds = \[([\s\S]*?)\] as const;/)?.[1]; if (!widgetBlock) throw new Error("Could not parse widgetKinds from packages/definitions/src/widget.ts"); const sourceWidgetKinds = [...widgetBlock.matchAll(/"([^"]+)"/g)].map((match) => match[1]); const assignedWidgetKinds = manifest.packets.flatMap((packet) => packet.widgetKinds ?? []); for (const widgetKind of sourceWidgetKinds) { const count = assignedWidgetKinds.filter((assigned) => assigned === widgetKind).length; if (count !== 1) errors.push(`widget ${widgetKind}: expected exactly one assignment, found ${count}`); } for (const widgetKind of assignedWidgetKinds) { if (!sourceWidgetKinds.includes(widgetKind)) errors.push(`unknown widget assignment ${widgetKind}`); } if (assignedWidgetKinds.length !== sourceWidgetKinds.length) { errors.push( `widget coverage count differs: ${assignedWidgetKinds.length} assigned, ${sourceWidgetKinds.length} declared`, ); } await validateRequestedRuntime(options, sourceWidgetKinds, expectedSharedBoardNames); for (const packet of manifest.packets) { const reportPath = join(reportsRoot, packet.id, "report.md"); await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`); let content: string; try { content = await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`); } catch { errors.push(`${packet.id}: report.md is missing`); continue; } const rawMetadata = parseReportMetadata(content, packet.id); if (rawMetadata === undefined) continue; const expectedWidgetChecks = (packet.widgetKinds ?? []).flatMap((widgetKind) => packet.viewports.map((viewport) => ({ widgetKind, viewport, sizeRequirement: requiredSizeInstruction(widgetKind), status: "not-reached" as const, })), ); const normalized = normalizeReportMetadata(rawMetadata, { packetId: packet.id, expectedCaseIds: expandedCases(manifest, packet).map(({ id }) => id), expectedWidgetChecks, }); errors.push(...normalized.errors); const metadata = normalized.metadata as ReportMetadata; errors.push(...collectHumanWidgetStatusErrors(content, packet.id, metadata.widgetChecks)); if (metadata.packetId !== packet.id) errors.push(`${packet.id}: metadata packetId is ${metadata.packetId}`); if (!allowedStatuses.includes(metadata.status)) errors.push(`${packet.id}: invalid report status ${String(metadata.status)}`); if (!Array.isArray(metadata.findings)) errors.push(`${packet.id}: findings must be an array`); if (!Array.isArray(metadata.artifacts)) errors.push(`${packet.id}: artifacts must be an array`); if (metadata.execution?.candidateSha !== campaignCandidateSha) { errors.push(`${packet.id}: candidateSha must equal the pinned campaign candidate ${campaignCandidateSha}`); } if (!content.includes(`| Candidate SHA | ${campaignCandidateSha} |`)) { errors.push(`${packet.id}: human metadata must show the pinned campaign candidate`); } if (!metadata.execution || typeof metadata.execution !== "object") errors.push(`${packet.id}: execution metadata is missing`); if (!content.includes("## Dogfood evidence")) errors.push(`${packet.id}: dogfood evidence section is missing`); if (!content.includes(resolve(repoRoot, ".screenshots/release-v2", packet.id))) { errors.push(`${packet.id}: absolute artifact directory link is missing`); } if (!content.includes(`| Status | ${metadata.status} |`)) errors.push(`${packet.id}: human metadata status differs from report metadata`); const packetCases = expandedCases(manifest, packet).map(({ id }) => id); if (!sameSet(Object.keys(metadata.caseStatuses), packetCases)) errors.push(`${packet.id}: case status IDs differ from manifest`); for (const [caseId, status] of Object.entries(metadata.caseStatuses)) { if (!allowedStatuses.includes(status)) errors.push(`${packet.id}/${caseId}: invalid case status ${String(status)}`); if (!content.includes(`| ${caseId} | ${status} |`)) errors.push(`${packet.id}/${caseId}: human evidence row/status is missing`); } const rollup = expectedRollup(Object.values(metadata.caseStatuses)); if (metadata.status !== rollup) errors.push(`${packet.id}: report status ${metadata.status} does not match case rollup ${rollup}`); for (const finding of Array.isArray(metadata.findings) ? metadata.findings : []) { if (!["P0", "P1", "P2", "P3"].includes(finding.severity)) errors.push(`${packet.id}: finding has invalid severity ${String(finding.severity)}`); if (typeof finding.title !== "string" || finding.title.trim().length === 0) errors.push(`${packet.id}: finding title is required`); for (const caseId of finding.caseIds ?? []) { if (!packetCases.includes(caseId)) errors.push(`${packet.id}: finding references unknown case ${caseId}`); } for (const evidence of finding.evidence ?? []) { const artifactError = await validateResolvedArtifactPath( resolve(repoRoot, ".screenshots/release-v2", packet.id), evidence, `${packet.id}: finding evidence`, ); if (artifactError) errors.push(artifactError); } } if (packet.widgetKinds) { const checks = Array.isArray(metadata.widgetChecks) ? metadata.widgetChecks : []; const expectedCheckKeys = packet.widgetKinds.flatMap((widgetKind) => packet.viewports.map((viewport) => `${widgetKind}\u0000${viewport}`), ); const actualCheckKeys = checks.map((check) => `${check.widgetKind}\u0000${check.viewport}`); if (!sameSet(actualCheckKeys, expectedCheckKeys)) { errors.push(`${packet.id}: structured widget checks do not match the assigned kind × viewport matrix`); } if (duplicateValues(actualCheckKeys).length > 0) { errors.push(`${packet.id}: structured widget checks contain duplicate kind × viewport rows`); } for (const check of checks) { if (!allowedStatuses.includes(check.status)) { errors.push(`${packet.id}/${check.widgetKind}/${check.viewport}: invalid widget check status`); } const expectedInstruction = requiredSizeInstruction(check.widgetKind); if (check.sizeRequirement !== expectedInstruction) { errors.push(`${packet.id}/${check.widgetKind}: incorrect size-threshold instruction`); } } for (const heading of [ "Size", "States", "Permission/read-only", "Options/persistence", "Recovery", "Evidence", "Status", ]) { if (!content.includes(heading)) errors.push(`${packet.id}: widget checklist is missing ${heading}`); } } else if ((metadata.widgetChecks?.length ?? 0) > 0) { errors.push(`${packet.id}: non-widget report must not contain widget checks`); } if (!metadata.performance || !Array.isArray(metadata.performance.measurements)) { errors.push(`${packet.id}: performance measurements must be an array`); } else { for (const measurement of metadata.performance.measurements) { if (!measurement.name?.trim() || !measurement.unit?.trim() || !measurement.threshold?.trim()) { errors.push(`${packet.id}: performance measurement name, unit, and threshold are required`); } if (measurement.value !== null && !Number.isFinite(measurement.value)) { errors.push(`${packet.id}: performance measurement value must be finite or null`); } if (!allowedStatuses.includes(measurement.status)) { errors.push(`${packet.id}: performance measurement has invalid status`); } for (const evidence of measurement.evidence ?? []) { const artifactError = await validateResolvedArtifactPath( resolve(repoRoot, ".screenshots/release-v2", packet.id), evidence, `${packet.id}: performance evidence`, ); if (artifactError) errors.push(artifactError); } } } if (!Array.isArray(metadata.performance?.limitations)) { errors.push(`${packet.id}: performance limitations must be an array`); } if (!Array.isArray(metadata.independentReproductions)) { errors.push(`${packet.id}: independentReproductions must be an array`); } else { for (const reproduction of metadata.independentReproductions) { if (!reproduction.findingFingerprint?.trim() || !reproduction.agentId?.trim()) { errors.push(`${packet.id}: independent reproduction requires a finding fingerprint and agent ID`); } if (!["reproduced", "not-reproduced", "blocked", "not-reached"].includes(reproduction.outcome)) { errors.push(`${packet.id}: independent reproduction has invalid outcome`); } for (const evidence of reproduction.evidence ?? []) { const artifactError = await validateResolvedArtifactPath( resolve(repoRoot, ".screenshots/release-v2", packet.id), evidence, `${packet.id}: reproduction evidence`, ); if (artifactError) errors.push(artifactError); } } } if (metadata.status !== "not-reached" && metadata.execution) { const execution = metadata.execution; const requiredText = [ execution.candidateSha, execution.url, execution.runtimeProfile, execution.persona, execution.sessionId, execution.timestamp, execution.viewport, execution.input, ]; if (requiredText.some((value) => typeof value !== "string" || value.trim().length === 0)) { errors.push(`${packet.id}: completed/blocked report has incomplete execution metadata`); } if (!Number.isInteger(execution.actualPort) || (execution.actualPort ?? 0) <= 0) errors.push(`${packet.id}: actualPort is required after execution`); if (!Number.isFinite(execution.zoom) || (execution.zoom ?? 0) <= 0) errors.push(`${packet.id}: zoom is required after execution`); if (execution.viewport && !packet.viewports.includes(execution.viewport)) errors.push(`${packet.id}: execution viewport is outside packet coverage`); if (execution.input && !packet.inputs.includes(execution.input)) errors.push(`${packet.id}: execution input is outside packet coverage`); if (execution.zoom && !packet.zooms.includes(execution.zoom)) errors.push(`${packet.id}: execution zoom is outside packet coverage`); if (execution.persona && !packet.personas.includes(execution.persona)) errors.push(`${packet.id}: execution persona is outside packet coverage`); if (execution.runtimeProfile && !packet.profiles.includes(execution.runtimeProfile)) errors.push(`${packet.id}: execution profile is outside packet coverage`); const expectedFlags = execution.runtimeProfile ? expectedProfileFlags[execution.runtimeProfile] : undefined; if (!expectedFlags || !sameSet(execution.runtimeFlags, expectedFlags)) { errors.push( `${packet.id}: runtime flags do not exactly match profile ${execution.runtimeProfile ?? "unknown"}`, ); } if (!execution.url?.match(/^https?:\/\//)) errors.push(`${packet.id}: URL must be absolute HTTP(S)`); if (!execution.timestamp || Number.isNaN(Date.parse(execution.timestamp))) errors.push(`${packet.id}: timestamp must be ISO-compatible`); for (const value of [...requiredText, execution.actualPort, execution.zoom]) { if (value !== null && value !== undefined && !content.includes(String(value))) errors.push(`${packet.id}: human metadata does not include execution value ${String(value)}`); } if (metadata.artifacts.length === 0) errors.push(`${packet.id}: executed report must link at least one artifact`); for (const artifact of metadata.artifacts) { const artifactError = await validateResolvedArtifactPath( resolve(repoRoot, ".screenshots/release-v2", packet.id), artifact, `${packet.id}: artifact`, ); if (artifactError) errors.push(artifactError); } } const ledgerEntry = ledger.packetStatuses[packet.id]; if (!ledgerEntry) { errors.push(`${packet.id}: ledger entry is missing`); continue; } if (ledgerEntry.status !== metadata.status) errors.push(`${packet.id}: ledger/report packet statuses differ`); if (JSON.stringify(ledgerEntry.cases) !== JSON.stringify(metadata.caseStatuses)) { errors.push(`${packet.id}: ledger/report case statuses differ`); } } for (const ledgerPacketId of Object.keys(ledger.packetStatuses)) { if (!packetIds.includes(ledgerPacketId)) errors.push(`ledger contains unknown packet ${ledgerPacketId}`); } if (errors.length > 0) { console.error(`release-v2 QA verification failed with ${errors.length} error(s):`); for (const error of errors) console.error(`- ${error}`); process.exitCode = 1; return; } console.log( `release-v2 QA verification passed: ${manifest.packets.length} packets, ${allCases.length} cases, ${sourceWidgetKinds.length} widgets`, ); }; await main();