diff --git a/apps/nextjs/src/components/layout/header/configurable-header.module.css b/apps/nextjs/src/components/layout/header/configurable-header.module.css index 27e2c8402..ace02eee7 100644 --- a/apps/nextjs/src/components/layout/header/configurable-header.module.css +++ b/apps/nextjs/src/components/layout/header/configurable-header.module.css @@ -103,7 +103,7 @@ } .mobileZone[data-zone="right"] { - justify-content: flex-end; + justify-content: flex-start; max-width: 72vw; } diff --git a/qa/release-v2/README.md b/qa/release-v2/README.md index a8a004fac..25df47e89 100644 --- a/qa/release-v2/README.md +++ b/qa/release-v2/README.md @@ -22,7 +22,9 @@ Each populated fixture manifest keeps the human-readable persona label in `perso Generated placements are validated before the seed transaction commits. Root items and containers must fit the active main/left/right lane after gutters, nested containers must fit their parent content grid without cycles, and every item must fit its selected root or container in every layout. `qa-grid-24` includes a dedicated unguttered 24-column custom layout that preserves exact width coverage `1–24`; its guttered Base layout and all narrower layouts keep lane-valid projected widths. -Use `report -- --init` to create missing agent reports. Use `report -- --refresh-placeholders` only after changing manifest templates; it refuses to overwrite reports with execution results, widget results, measurements, limitations, or reproduction data. +Use `report -- --init` to create missing agent reports. Use `report -- --refresh-placeholders` after changing manifest templates or rolling the checkout candidate. It refreshes only safe, untouched `not-reached` placeholders, reports refreshed and protected-skip counts, and preserves reports with execution or case results, widget results, findings, artifacts, measurements, limitations, or reproduction data byte-for-byte. Malformed reports are protected skips rather than overwrite candidates. + +When a release-blocking defect has been fixed and downstream packets contain no feature result beyond that one blocker, explicitly roll only those reports to fresh `not-reached` templates with `report -- --rollover-resolved-blocker `. The rollover rejects reports with another finding, passed or failed feature cases or widgets, performance measurements, malformed metadata, or a concurrent content/mtime change. It never scans report directories outside the 45 manifest packets, so independent reproducer reports remain untouched. ## Managed runtime @@ -44,7 +46,7 @@ The selected app port remains bound by a temporary loopback reservation through `--reset` is intentionally narrow: after validating the QA run-root marker, it stops only the selected running manifest, removes only that slot directory, and clears only that slot's `.next-qa` cache. The run root must be a direct child of the canonical operating-system temporary directory returned by Node, its basename must be `homarr-release-v2-qa` or start with `homarr-release-v2-qa-`, and only safe suffix characters are accepted. Temporary-directory aliases are canonicalized before containment checks. Existing run roots and slot paths must not be symlinks or contain symlink components, and destructive paths are rechecked immediately before mutation. The Linux default `/tmp/homarr-release-v2-qa` remains unchanged. -This host's `fs.inotify.max_user_instances=128` is too low for concurrent native Watchpack watchers. Spawned QA apps keep Turbopack and set `WATCHPACK_POLLING=1000`, avoiding reliance on another inotify instance while keeping all three isolated slots available. Filesystem changes can take up to one polling interval to reach HMR. This is Watchpack's watcher mode, not a switch to the Webpack bundler. +This host's `fs.inotify.max_user_instances=128` is too low for concurrent native Watchpack watchers. Spawned QA apps use Next.js Webpack development mode (`--webpack`) and set `WATCHPACK_POLLING=1000`, avoiding reliance on another inotify instance while keeping all three isolated slots available. Webpack also gives clean browser sessions a deterministic development chunk graph instead of retaining stale Turbopack chunk URLs. Filesystem changes can take up to one polling interval to reach HMR. The runtime manifest records `bundler: "webpack"`, and live verification rejects another bundler. Existing slots must be restarted before they satisfy this contract. ## Profiles diff --git a/scripts/release-v2-qa/report.mts b/scripts/release-v2-qa/report.mts index d872d2ebc..eb6d66f0a 100644 --- a/scripts/release-v2-qa/report.mts +++ b/scripts/release-v2-qa/report.mts @@ -1,6 +1,6 @@ -import { mkdir, readFile } from "node:fs/promises"; +import { mkdir, readFile, stat } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { aggregateQaFindings, @@ -17,6 +17,7 @@ import { writeSafeReportFile, } from "./report-path-integrity.mts"; import { resolveCheckoutCandidateSha } from "./provenance.mts"; +import { createRuntimeExecutionContract } from "./runtime.mts"; type Status = "passed" | "failed" | "blocked" | "not-reached"; type Severity = "P0" | "P1" | "P2" | "P3"; @@ -105,12 +106,47 @@ interface IndependentReproduction { notes: string; } +interface PlaceholderRefreshOptions { + read: (entry: T) => Promise; + isSafePlaceholder: (content: string, entry: T) => boolean; + render: (entry: T) => string; + write: (entry: T, content: string) => Promise; +} + +interface PlaceholderRefreshCounts { + refreshed: number; + skipped: number; +} + +interface ReportSnapshot { + content: string; + mtimeMs: number; + size: number; +} + +interface ResolvedBlockerRolloverOptions { + read: (entry: T) => Promise; + isEligible: (content: string, entry: T) => boolean; + isUnchanged: (entry: T, snapshot: TSnapshot) => Promise; + render: (entry: T) => string; + write: (entry: T, content: string) => Promise; +} + +interface ResolvedBlockerRolloverCounts { + reset: number; + skipped: number; + concurrentlyChanged: number; +} + const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const qaRoot = join(repoRoot, "qa/release-v2"); const reportsRoot = join(qaRoot, "reports"); const campaignCandidateSha = resolveCheckoutCandidateSha(repoRoot); -const hostRuntimeLimitation = - "The host's fs.inotify.max_user_instances=128 is too low for concurrent native Watchpack watchers. Spawned QA apps keep Turbopack and use 1000ms Watchpack polling; filesystem changes can take up to one polling interval to reach HMR."; +const runtimeExecutionContract = createRuntimeExecutionContract(); +export const hostRuntimeLimitation = + `The host's fs.inotify.max_user_instances=128 is too low for concurrent native Watchpack watchers. ` + + `Spawned QA apps use the ${runtimeExecutionContract.bundler} bundler with ${runtimeExecutionContract.watcher.watchpackPollingIntervalMs}ms Watchpack polling; ` + + "filesystem changes can take up to one polling interval to reach HMR."; const expectedProfileFlags: Record = { "main-writable": ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], "main-readonly": ["DEMO_MODE=true", "DEMO_READ_ONLY=true", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], @@ -139,9 +175,20 @@ const exhaustiveSizeWidgetKinds = new Set([ const manifest = JSON.parse(await readFile(join(qaRoot, "coverage-manifest.json"), "utf8")) as Manifest; const init = process.argv.includes("--init"); const refresh = process.argv.includes("--refresh-placeholders"); +const rolloverFlag = "--rollover-resolved-blocker"; +const rolloverFlagIndex = process.argv.indexOf(rolloverFlag); +const rolloverResolvedBlocker = rolloverFlagIndex === -1 ? null : (process.argv[rolloverFlagIndex + 1] ?? null); + +if (rolloverFlagIndex !== -1 && (!rolloverResolvedBlocker || rolloverResolvedBlocker.startsWith("--"))) { + throw new Error(`${rolloverFlag} requires a finding fingerprint`); +} +if ([init, refresh, rolloverResolvedBlocker !== null].filter(Boolean).length > 1) { + throw new Error("Use only one of --init, --refresh-placeholders, or --rollover-resolved-blocker"); +} const escapeCell = (value: string): string => value.replaceAll("|", "\\|").replaceAll("\n", " "); const list = (values: (string | number)[]): string => (values.length === 0 ? "None" : values.join(", ")); +const isBlockedOrNotReached = (status: Status): boolean => status === "blocked" || status === "not-reached"; const expandedCases = (packet: Packet): { id: string; expected: string }[] => { const dimensions = manifest.caseDimensions[packet.wave] ?? []; @@ -215,7 +262,7 @@ const renderPacketReportBase = (packet: Packet): string => { return `\n\n# ${packet.id}: ${packet.title}\n\n| Metadata | Value |\n| --- | --- |\n| Status | not-reached |\n| Wave | ${packet.wave} |\n| PR refs | ${packet.prRefs.map((number) => `#${number}`).join(", ")} |\n| Personas | ${list(packet.personas)} |\n| Boards | ${list(packet.boards)} |\n| Profiles | ${list(packet.profiles)} |\n| Chromium viewports | ${list(packet.viewports)} |\n| Zoom | ${list(packet.zooms.map((zoom) => `${zoom}%`))} |\n| Input | ${list(packet.inputs)} |\n| Widget kinds | ${list(packet.widgetKinds ?? [])} |\n| Artifact directory | [${artifactDirectory}](${artifactDirectory}) |\n| Candidate SHA | not-recorded |\n| URL / actual port | not-recorded |\n| Runtime profile / flags | not-recorded |\n| Executed persona | not-recorded |\n| Browser session ID | not-recorded |\n| Timestamp | not-recorded |\n| Executed viewport / input / zoom | not-recorded |\n\n## Dogfood evidence\n\n| Case ID | Status | Evidence | Observed | Expected |\n| --- | --- | --- | --- | --- |\n${cases}\n${widgetSection}\n## Findings\n\nNo findings recorded. Add structured findings to the metadata block and mirror them here.\n\n## Blockers and gaps\n\nNot executed.\n\n## Cleanup\n\nRecord created resources, cleanup outcome, console errors, and network failures. Never include secret values.\n`; }; -const renderPacketReport = (packet: Packet): string => +export const renderPacketReport = (packet: Packet): string => renderPacketReportBase(packet) .replace("| Candidate SHA | not-recorded |", `| Candidate SHA | ${campaignCandidateSha} |`) .replace( @@ -247,6 +294,104 @@ const parseMetadata = (content: string, packet: Packet): { metadata: ReportMetad }; }; +export const reportMetadataHasEvidence = (metadata: ReportMetadata): boolean => { + const execution = metadata.execution; + const hasExecutionMetadata = + execution.url !== null || + execution.actualPort !== null || + execution.runtimeProfile !== null || + execution.runtimeFlags.length > 0 || + execution.persona !== null || + execution.sessionId !== null || + execution.timestamp !== null || + execution.viewport !== null || + execution.input !== null || + execution.zoom !== null; + + return ( + metadata.status !== "not-reached" || + Object.values(metadata.caseStatuses).some((status) => status !== "not-reached") || + metadata.widgetChecks.some((check) => check.status !== "not-reached") || + metadata.findings.length > 0 || + metadata.artifacts.length > 0 || + metadata.performance.measurements.length > 0 || + metadata.performance.limitations.length > 0 || + metadata.independentReproductions.length > 0 || + hasExecutionMetadata + ); +}; + +export const reportIsOnlyBlockedBy = (metadata: ReportMetadata, findingFingerprint: string): boolean => { + if (!isBlockedOrNotReached(metadata.status)) return false; + if (!Object.values(metadata.caseStatuses).every(isBlockedOrNotReached)) return false; + if (!metadata.widgetChecks.every((check) => isBlockedOrNotReached(check.status))) return false; + if (metadata.performance.measurements.length > 0) return false; + if (metadata.findings.length === 0) return false; + if (metadata.findings.some((finding) => finding.fingerprint !== findingFingerprint)) return false; + if (metadata.independentReproductions.some((reproduction) => reproduction.findingFingerprint !== findingFingerprint)) + return false; + return true; +}; + +export const reportContentIsOnlyBlockedBy = (content: string, packet: Packet, findingFingerprint: string): boolean => { + const parsed = parseMetadata(content, packet); + return parsed.errors.length === 0 && reportIsOnlyBlockedBy(parsed.metadata, findingFingerprint); +}; + +export const rolloverResolvedBlockerReports = async ( + entries: T[], + options: ResolvedBlockerRolloverOptions, +): Promise => { + const counts: ResolvedBlockerRolloverCounts = { reset: 0, skipped: 0, concurrentlyChanged: 0 }; + for (const entry of entries) { + let snapshot: TSnapshot; + try { + snapshot = await options.read(entry); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + counts.skipped += 1; + continue; + } + + if (!options.isEligible(snapshot.content, entry)) { + counts.skipped += 1; + continue; + } + if (!(await options.isUnchanged(entry, snapshot))) { + counts.concurrentlyChanged += 1; + continue; + } + + await options.write(entry, options.render(entry)); + counts.reset += 1; + } + return counts; +}; + +export const refreshPlaceholderReports = async ( + entries: T[], + options: PlaceholderRefreshOptions, +): Promise => { + const counts: PlaceholderRefreshCounts = { refreshed: 0, skipped: 0 }; + for (const entry of entries) { + let existingContent: string | null = null; + try { + existingContent = await options.read(entry); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + + if (existingContent !== null && !options.isSafePlaceholder(existingContent, entry)) { + counts.skipped += 1; + continue; + } + + await options.write(entry, options.render(entry)); + counts.refreshed += 1; + } + return counts; +}; + const collectHumanReportErrors = (content: string, packet: Packet, metadata: ReportMetadata): string[] => { const errors: string[] = []; if (!content.includes(`| Status | ${metadata.status} |`)) { @@ -323,33 +468,72 @@ const collectArtifactErrors = async (packet: Packet, metadata: ReportMetadata): }; const ensureReports = async (): Promise => { + if (refresh || rolloverResolvedBlocker) { + for (const packet of manifest.packets) { + const packetDirectory = join(reportsRoot, packet.id); + const reportPath = join(packetDirectory, "report.md"); + await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`); + await mkdir(packetDirectory, { recursive: true }); + await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`); + } + } + + if (rolloverResolvedBlocker) { + const reportSnapshot = async (packet: Packet): Promise => { + const reportPath = join(reportsRoot, packet.id, "report.md"); + const content = await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`); + const metadata = await stat(reportPath); + return { content, mtimeMs: metadata.mtimeMs, size: metadata.size }; + }; + const counts = await rolloverResolvedBlockerReports(manifest.packets, { + read: reportSnapshot, + isEligible: (content, packet) => reportContentIsOnlyBlockedBy(content, packet, rolloverResolvedBlocker), + isUnchanged: async (packet, snapshot) => { + const current = await reportSnapshot(packet); + return ( + current.content === snapshot.content && current.mtimeMs === snapshot.mtimeMs && current.size === snapshot.size + ); + }, + render: renderPacketReport, + write: async (packet, content) => { + const reportPath = join(reportsRoot, packet.id, "report.md"); + await writeSafeReportFile(qaRoot, reportPath, content, `${packet.id} report path`); + }, + }); + console.log( + `release-v2 QA resolved-blocker rollover (${rolloverResolvedBlocker}): ${counts.reset} reset; ${counts.skipped} protected report(s) skipped; ${counts.concurrentlyChanged} concurrently changed report(s) skipped`, + ); + return; + } + + if (refresh) { + const counts = await refreshPlaceholderReports(manifest.packets, { + read: async (packet) => { + const reportPath = join(reportsRoot, packet.id, "report.md"); + return readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`); + }, + isSafePlaceholder: (content, packet) => { + const parsed = parseMetadata(content, packet); + return parsed.errors.length === 0 && !reportMetadataHasEvidence(parsed.metadata); + }, + render: renderPacketReport, + write: async (packet, content) => { + const reportPath = join(reportsRoot, packet.id, "report.md"); + await writeSafeReportFile(qaRoot, reportPath, content, `${packet.id} report path`); + }, + }); + console.log( + `release-v2 QA placeholder refresh: ${counts.refreshed} refreshed; ${counts.skipped} protected report(s) skipped`, + ); + return; + } + for (const packet of manifest.packets) { const packetDirectory = join(reportsRoot, packet.id); const reportPath = join(packetDirectory, "report.md"); await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`); await mkdir(packetDirectory, { recursive: true }); await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`); - if (refresh) { - try { - const parsed = parseMetadata(await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`), packet); - if (parsed.errors.length > 0) throw new Error(parsed.errors.join("; ")); - const existing = parsed.metadata; - const hasExecution = - existing.status !== "not-reached" || - Object.values(existing.caseStatuses).some((status) => status !== "not-reached") || - existing.findings.length > 0 || - existing.artifacts.length > 0 || - (existing.widgetChecks ?? []).some((check) => check.status !== "not-reached") || - (existing.performance?.measurements.length ?? 0) > 0 || - (existing.performance?.limitations.length ?? 0) > 0 || - (existing.independentReproductions?.length ?? 0) > 0; - if (hasExecution) throw new Error(`${packet.id}: refusing to replace a report containing execution results`); - } catch (error) { - if (!(error instanceof Error) || !error.message.includes("ENOENT")) throw error; - } - await writeSafeReportFile(qaRoot, reportPath, renderPacketReport(packet), `${packet.id} report path`); - continue; - } try { await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`); } catch (error) { @@ -402,7 +586,7 @@ const coverageTable = (assignments: CoverageAssignment[], labelHeading: string) }; const main = async (): Promise => { - if (init || refresh) await ensureReports(); + if (init || refresh || rolloverResolvedBlocker) await ensureReports(); const reports: { packet: Packet; metadata: ReportMetadata; metadataErrors: string[] }[] = []; for (const packet of manifest.packets) { @@ -627,4 +811,5 @@ const main = async (): Promise => { } }; -await main(); +const entrypoint = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : null; +if (entrypoint === import.meta.url) await main(); diff --git a/scripts/release-v2-qa/report.spec.ts b/scripts/release-v2-qa/report.spec.ts new file mode 100644 index 000000000..90a99e38d --- /dev/null +++ b/scripts/release-v2-qa/report.spec.ts @@ -0,0 +1,281 @@ +// @vitest-environment node + +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { resolveCheckoutCandidateSha } from "./provenance.mts"; +import { + hostRuntimeLimitation, + refreshPlaceholderReports, + renderPacketReport, + reportContentIsOnlyBlockedBy, + reportMetadataHasEvidence, + rolloverResolvedBlockerReports, +} from "./report.mts"; +import { createRuntimeExecutionContract } from "./runtime.mts"; + +const repoRoot = path.resolve(import.meta.dirname, "../.."); +const candidateSha = resolveCheckoutCandidateSha(repoRoot); +const blockerFingerprint = "release-v2-auth-locale-redirect-loop"; + +describe("release-v2 QA report runtime limitation", () => { + it("describes the bundler and polling interval from the runtime contract", () => { + const runtimeExecutionContract = createRuntimeExecutionContract(); + + expect(hostRuntimeLimitation).toContain(`the ${runtimeExecutionContract.bundler} bundler`); + expect(hostRuntimeLimitation).toContain( + `${runtimeExecutionContract.watcher.watchpackPollingIntervalMs}ms Watchpack polling`, + ); + expect(hostRuntimeLimitation).not.toContain("Turbopack"); + }); +}); + +interface TestPacket { + id: string; + wave: string; + title: string; + prRefs: number[]; + personas: string[]; + boards: string[]; + profiles: string[]; + viewports: string[]; + zooms: number[]; + inputs: string[]; + widgetKinds?: string[]; + cases: string[]; +} + +const coverageManifest = JSON.parse( + await readFile(path.join(repoRoot, "qa/release-v2/coverage-manifest.json"), "utf8"), +) as { packets: TestPacket[] }; +const packet = (packetId: string): TestPacket => { + const result = coverageManifest.packets.find((entry) => entry.id === packetId); + if (!result) throw new Error(`Missing test packet ${packetId}`); + return result; +}; + +const blockedByResolvedFinding = (testPacket: TestPacket): string => { + const placeholder = renderPacketReport(testPacket); + const rawMetadata = placeholder.match(//)?.[1]; + if (!rawMetadata) throw new Error(`Missing generated metadata for ${testPacket.id}`); + const metadata = JSON.parse(rawMetadata) as Record & { + status: string; + caseStatuses: Record; + execution: Record; + findings: unknown[]; + artifacts: string[]; + widgetChecks: { status: string }[]; + performance: { measurements: unknown[]; limitations: string[] }; + independentReproductions: unknown[]; + }; + metadata.status = "blocked"; + metadata.caseStatuses = Object.fromEntries(Object.keys(metadata.caseStatuses).map((caseId) => [caseId, "blocked"])); + metadata.widgetChecks = metadata.widgetChecks.map((check) => ({ ...check, status: "blocked" })); + metadata.execution = { + ...metadata.execution, + url: "http://127.0.0.1:34401", + actualPort: 34401, + runtimeProfile: "main-writable", + runtimeFlags: ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"], + persona: testPacket.personas[0] ?? "Avery Admin", + sessionId: `qa-v2-${testPacket.id}`, + timestamp: "2026-08-31T15:00:00.000Z", + viewport: testPacket.viewports[0] ?? "desktop-1440", + input: testPacket.inputs[0] ?? "mouse", + zoom: testPacket.zooms[0] ?? 100, + }; + metadata.findings = [ + { + fingerprint: blockerFingerprint, + severity: "P1", + title: "Locale rewrite redirect loop", + evidence: [`/tmp/${testPacket.id}/auth-loop.png`], + }, + ]; + metadata.artifacts = [`/tmp/${testPacket.id}/auth-loop.png`]; + metadata.performance.limitations = ["The application never rendered because of the resolved auth blocker."]; + metadata.independentReproductions = [ + { + findingFingerprint: blockerFingerprint, + agentId: "preflight-01", + outcome: "reproduced", + evidence: [], + notes: "Auth-only reproduction.", + }, + ]; + return placeholder.replace(rawMetadata, JSON.stringify(metadata, null, 2)); +}; + +const untouchedMetadata = () => ({ + packetId: "preflight-01", + status: "not-reached" as const, + caseStatuses: { "PF01-FIXTURE-STATE": "not-reached" as const }, + execution: { + candidateSha: "f57a660088d6777c86aca22977354fa8b810e2be", + url: null, + actualPort: null, + runtimeProfile: null, + runtimeFlags: [], + persona: null, + sessionId: null, + timestamp: null, + viewport: null, + input: null, + zoom: null, + }, + findings: [], + artifacts: [], + widgetChecks: [], + performance: { measurements: [], limitations: [] }, + independentReproductions: [], + notes: "Not executed.", +}); + +describe("release-v2 QA report placeholder refresh", () => { + it("preserves an executed report before refreshing a later untouched placeholder", async () => { + const reports = new Map([ + ["preflight-01", "executed report evidence"], + ["board-01", "untouched placeholder from the previous candidate"], + ]); + + const counts = await refreshPlaceholderReports(["preflight-01", "board-01"], { + read: async (packetId) => reports.get(packetId) ?? "", + isSafePlaceholder: (content) => content.startsWith("untouched placeholder"), + render: (packetId) => `refreshed placeholder for ${packetId}`, + write: async (packetId, content) => { + reports.set(packetId, content); + }, + }); + + expect(counts).toEqual({ refreshed: 1, skipped: 1 }); + expect(reports.get("preflight-01")).toBe("executed report evidence"); + expect(reports.get("board-01")).toBe("refreshed placeholder for board-01"); + }); + + it("treats execution metadata as evidence even before a case status changes", () => { + const metadata = untouchedMetadata(); + metadata.execution.url = "http://127.0.0.1:34401"; + + expect(reportMetadataHasEvidence(metadata)).toBe(true); + }); + + it("protects every evidence-bearing report while allowing a candidate-only placeholder", () => { + expect(reportMetadataHasEvidence(untouchedMetadata())).toBe(false); + + const evidenceReports = [ + () => ({ ...untouchedMetadata(), status: "passed" as const }), + () => ({ ...untouchedMetadata(), caseStatuses: { "PF01-FIXTURE-STATE": "blocked" as const } }), + () => ({ + ...untouchedMetadata(), + widgetChecks: [ + { widgetKind: "weather", viewport: "320x568", sizeRequirement: "minimum", status: "passed" as const }, + ], + }), + () => ({ + ...untouchedMetadata(), + findings: [{ severity: "P3" as const, title: "Observed defect" }], + }), + () => ({ ...untouchedMetadata(), artifacts: ["/tmp/evidence.png"] }), + () => ({ + ...untouchedMetadata(), + performance: { + measurements: [ + { + name: "render", + value: 1, + unit: "ms", + threshold: "under 10 ms", + status: "passed" as const, + evidence: [], + }, + ], + limitations: [], + }, + }), + () => ({ ...untouchedMetadata(), performance: { measurements: [], limitations: ["Host limitation"] } }), + () => ({ + ...untouchedMetadata(), + independentReproductions: [ + { + findingFingerprint: "finding", + agentId: "reproducer", + outcome: "reproduced" as const, + evidence: [], + notes: "Confirmed.", + }, + ], + }), + ]; + + for (const evidenceReport of evidenceReports) expect(reportMetadataHasEvidence(evidenceReport())).toBe(true); + }); +}); + +describe("release-v2 QA resolved-blocker rollover", () => { + it("resets actual board and widget packet shapes while preserving an executed preflight", async () => { + const preflightPacket = packet("preflight-01"); + const boardPacket = packet("board-01"); + const widgetPacket = packet("widgets-01"); + const executedPreflight = renderPacketReport(preflightPacket).replace( + '"status": "not-reached"', + '"status": "passed"', + ); + const reports = new Map([ + [preflightPacket.id, { content: executedPreflight, revision: 1 }], + [boardPacket.id, { content: blockedByResolvedFinding(boardPacket), revision: 1 }], + [widgetPacket.id, { content: blockedByResolvedFinding(widgetPacket), revision: 1 }], + ]); + + const counts = await rolloverResolvedBlockerReports([preflightPacket, boardPacket, widgetPacket], { + read: async (entry) => { + const report = reports.get(entry.id); + if (!report) throw new Error(`Missing report ${entry.id}`); + return { ...report }; + }, + isEligible: (content, entry) => reportContentIsOnlyBlockedBy(content, entry, blockerFingerprint), + isUnchanged: async (entry, snapshot) => { + const current = reports.get(entry.id); + return current?.content === snapshot.content && current.revision === snapshot.revision; + }, + render: renderPacketReport, + write: async (entry, content) => { + reports.set(entry.id, { content, revision: 2 }); + }, + }); + + expect(counts).toEqual({ reset: 2, skipped: 1, concurrentlyChanged: 0 }); + expect(reports.get(preflightPacket.id)?.content).toBe(executedPreflight); + for (const packetId of [boardPacket.id, widgetPacket.id]) { + const refreshed = reports.get(packetId)?.content ?? ""; + expect(refreshed).toContain(`"candidateSha": "${candidateSha}"`); + expect(refreshed).toContain(`| Candidate SHA | ${candidateSha} |`); + expect(refreshed).toContain("| Status | not-reached |"); + expect(refreshed).not.toContain(blockerFingerprint); + } + }); + + it("skips a report changed by an agent after eligibility was checked", async () => { + const boardPacket = packet("board-01"); + const initial = blockedByResolvedFinding(boardPacket); + const agentUpdate = `${initial}\nAgent completed new evidence.\n`; + const reports = new Map([[boardPacket.id, { content: initial, revision: 1 }]]); + + const counts = await rolloverResolvedBlockerReports([boardPacket], { + read: async () => ({ content: initial, revision: 1 }), + isEligible: (content, entry) => reportContentIsOnlyBlockedBy(content, entry, blockerFingerprint), + isUnchanged: async () => { + reports.set(boardPacket.id, { content: agentUpdate, revision: 2 }); + return false; + }, + render: renderPacketReport, + write: async (entry, content) => { + reports.set(entry.id, { content, revision: 3 }); + }, + }); + + expect(counts).toEqual({ reset: 0, skipped: 0, concurrentlyChanged: 1 }); + expect(reports.get(boardPacket.id)?.content).toBe(agentUpdate); + }); +}); diff --git a/scripts/release-v2-qa/runtime.mts b/scripts/release-v2-qa/runtime.mts index 9b2d85044..066f64713 100644 --- a/scripts/release-v2-qa/runtime.mts +++ b/scripts/release-v2-qa/runtime.mts @@ -20,6 +20,7 @@ const repoRoot = path.resolve(import.meta.dirname, "../.."); const defaultRunRoot = path.join(tmpdir(), "homarr-release-v2-qa"); const markerName = ".homarr-release-v2-qa-root.json"; const watchpackPollingIntervalMs = 1_000; +const runtimeBundler = "webpack" as const; const profiles = ["main-writable", "main-readonly", "onboarding-fresh", "degraded"] as const; type Profile = (typeof profiles)[number]; @@ -30,6 +31,53 @@ export const assertRuntimeCommandPlatform = (command: RuntimeCommand, platform: assertSupportedRuntimePlatform(platform); }; +interface RuntimeExecutionContract { + bundler: typeof runtimeBundler; + watcher: { watchpackPollingIntervalMs: number }; +} + +export const createRuntimeExecutionContract = (): RuntimeExecutionContract => ({ + bundler: runtimeBundler, + watcher: { watchpackPollingIntervalMs }, +}); + +export const validateRuntimeExecutionContract = (value: { + bundler?: unknown; + watcher?: { watchpackPollingIntervalMs?: unknown }; +}): string[] => { + const contractErrors: string[] = []; + if (value.bundler !== runtimeBundler) contractErrors.push("bundler must be webpack"); + if (value.watcher?.watchpackPollingIntervalMs !== watchpackPollingIntervalMs) { + contractErrors.push("Watchpack polling interval must be 1000ms"); + } + return contractErrors; +}; + +export const createAppLaunchCommand = ( + appPort: number, + options: { platform?: NodeJS.Platform; prlimitAvailable?: boolean } = {}, +) => { + const platform = options.platform ?? process.platform; + const prlimitAvailable = options.prlimitAvailable ?? existsSync("/usr/bin/prlimit"); + const pnpmArguments = [ + "--filter", + "@homarr/nextjs", + "dev", + "--webpack", + "--hostname", + "127.0.0.1", + "--port", + String(appPort), + ]; + if (platform === "linux" && prlimitAvailable) { + return { + command: "/usr/bin/prlimit", + arguments: ["--nofile=65536:1048576", "--", "pnpm", ...pnpmArguments], + }; + } + return { command: "pnpm", arguments: pnpmArguments }; +}; + interface Options { command: RuntimeCommand; slot: number; @@ -59,6 +107,7 @@ interface RuntimeManifest { fixtureUrl: string; ports: { app: number; fixture: number; redis: number }; flags: { demoMode: boolean; demoReadOnly: boolean; unsafeMockIntegration: boolean }; + bundler: typeof runtimeBundler; watcher: { watchpackPollingIntervalMs: number }; processes: { app: { pid: number; logPath: string }; @@ -582,7 +631,6 @@ const startWithLease = async (options: Options) => { runForeground("pnpm", ["qa:release-v2:seed", "--", "--profile", options.profile, "--output", slotDir], environment); await assertSafeContainedPath(options.runRoot, fixtureManifestPath, "QA fixture manifest"); - let appCommand = "pnpm"; const appEnvironment = { ...sanitizeAppEnvironment(environment), WATCHPACK_POLLING: String(watchpackPollingIntervalMs), @@ -592,18 +640,12 @@ const startWithLease = async (options: Options) => { async (attemptPort) => { appPort = attemptPort; url = `http://127.0.0.1:${appPort}`; - let appArguments = ["--filter", "@homarr/nextjs", "dev", "--hostname", "127.0.0.1", "--port", String(appPort)]; - if (process.platform === "linux" && existsSync("/usr/bin/prlimit")) { - appCommand = "/usr/bin/prlimit"; - appArguments = ["--nofile=65536:1048576", "--", "pnpm", ...appArguments]; - } else { - appCommand = "pnpm"; - } + const appLaunch = createAppLaunchCommand(appPort); const logOffset = await stat(appLogPath) .then((stats) => stats.size) .catch(() => 0); await assertSafeContainedPath(options.runRoot, appLogPath, "QA app log"); - const spawned = spawnLogged(appCommand, appArguments, appLogPath, { + const spawned = spawnLogged(appLaunch.command, appLaunch.arguments, appLogPath, { ...appEnvironment, HOMARR_WEBSITE_URL: url, }); @@ -641,7 +683,7 @@ const startWithLease = async (options: Options) => { fixtureUrl: fixtureReady.url, ports: { app: appPort, fixture: fixtureReady.port, redis: redisPort }, flags, - watcher: { watchpackPollingIntervalMs }, + ...createRuntimeExecutionContract(), processes: { app: { pid: appPid, logPath: appLogPath }, fixture: { pid: fixturePid, logPath: fixtureLogPath }, diff --git a/scripts/release-v2-qa/runtime.spec.ts b/scripts/release-v2-qa/runtime.spec.ts index 28163ee40..3e4672799 100644 --- a/scripts/release-v2-qa/runtime.spec.ts +++ b/scripts/release-v2-qa/runtime.spec.ts @@ -4,7 +4,14 @@ import { createServer } from "node:net"; import type { Server } from "node:net"; import { afterEach, describe, expect, test } from "vitest"; -import { assertRuntimeCommandPlatform, launchWithPortRetry, reserveLoopbackPort } from "./runtime.mts"; +import { + assertRuntimeCommandPlatform, + createAppLaunchCommand, + createRuntimeExecutionContract, + launchWithPortRetry, + reserveLoopbackPort, + validateRuntimeExecutionContract, +} from "./runtime.mts"; const openServers: Server[] = []; @@ -79,3 +86,46 @@ describe("release-v2 QA runtime platform boundary", () => { expect(() => assertRuntimeCommandPlatform(command, "linux")).not.toThrow(); }); }); + +describe("release-v2 QA app execution contract", () => { + test("uses Webpack dev while preserving the Linux file-descriptor wrapper", () => { + const launch = createAppLaunchCommand(31_337, { platform: "linux", prlimitAvailable: true }); + const launchWithoutPrlimit = createAppLaunchCommand(31_337, { platform: "linux", prlimitAvailable: false }); + + expect(launch).toEqual({ + command: "/usr/bin/prlimit", + arguments: [ + "--nofile=65536:1048576", + "--", + "pnpm", + "--filter", + "@homarr/nextjs", + "dev", + "--webpack", + "--hostname", + "127.0.0.1", + "--port", + "31337", + ], + }); + expect(launchWithoutPrlimit.command).toBe("pnpm"); + expect(launchWithoutPrlimit.arguments).toContain("--webpack"); + }); + + test("records and validates the deterministic bundler and watcher settings", () => { + const execution = createRuntimeExecutionContract(); + + expect(execution).toEqual({ + bundler: "webpack", + watcher: { watchpackPollingIntervalMs: 1_000 }, + }); + expect(validateRuntimeExecutionContract(execution)).toEqual([]); + expect( + validateRuntimeExecutionContract({ + bundler: "turbopack", + watcher: { watchpackPollingIntervalMs: 500 }, + }), + ).toEqual(["bundler must be webpack", "Watchpack polling interval must be 1000ms"]); + expect(validateRuntimeExecutionContract({ watcher: execution.watcher })).toEqual(["bundler must be webpack"]); + }); +}); diff --git a/scripts/release-v2-qa/verify.mts b/scripts/release-v2-qa/verify.mts index 96ef764da..c53bc7d2f 100644 --- a/scripts/release-v2-qa/verify.mts +++ b/scripts/release-v2-qa/verify.mts @@ -9,6 +9,7 @@ import { findFixtureGeometryErrors } from "./geometry.mts"; import { resolveCheckoutCandidateSha } from "./provenance.mts"; import { collectHumanWidgetStatusErrors, normalizeReportMetadata } from "./report-integrity.mts"; import { assertSafeReportPath, readSafeReportFile, validateResolvedArtifactPath } from "./report-path-integrity.mts"; +import { validateRuntimeExecutionContract } from "./runtime.mts"; import { assertSafeRunRoot, isPathWithin, validateFixtureUrl } from "./safety.mts"; @@ -117,6 +118,7 @@ interface RuntimeManifest { fixtureUrl: string; ports: { app: number; fixture: number; redis: number }; flags: { demoMode: boolean; demoReadOnly: boolean; unsafeMockIntegration: boolean }; + bundler: string; watcher: { watchpackPollingIntervalMs: number }; processes: { app: { pid: number; logPath: string }; @@ -629,8 +631,8 @@ const validateRuntimeManifest = async ( if (JSON.stringify(runtime.flags) !== JSON.stringify(expectedFlags)) { errors.push(`${label}: runtime flags do not match profile ${runtime.profile}`); } - if (runtime.watcher?.watchpackPollingIntervalMs !== 1_000) { - errors.push(`${label}: Watchpack polling interval must be 1000ms`); + for (const executionError of validateRuntimeExecutionContract(runtime)) { + errors.push(`${label}: ${executionError}`); } if (resolve(runtime.repoRoot) !== repoRoot) errors.push(`${label}: repoRoot does not match this checkout`); try {