test: add release v2 browser QA campaign
This commit is contained in:
ajnart committed
1 parent
f57a660088
commit
ec95d18245
83 files changed
+20922
-19
No files matched your search
@@ -0,0 +1,349 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { readFile, mkdir, open, rename, rm } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { validateLoopbackHost } from "./safety.mts";
|
||||
|
||||
const DEFAULT_HOST = "127.0.0.1";
|
||||
const MAX_BODY_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_SLOW_MS = 30_000;
|
||||
|
||||
const png = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
);
|
||||
|
||||
const videoPath = path.resolve(
|
||||
import.meta.dirname,
|
||||
"../../apps/docs/src/components/pages/home/drag-and-drop/showcase-dark.mp4",
|
||||
);
|
||||
|
||||
const createWav = () => {
|
||||
const sampleRate = 8_000;
|
||||
const sampleCount = sampleRate / 4;
|
||||
const dataLength = sampleCount * 2;
|
||||
const buffer = Buffer.alloc(44 + dataLength);
|
||||
buffer.write("RIFF", 0);
|
||||
buffer.writeUInt32LE(36 + dataLength, 4);
|
||||
buffer.write("WAVEfmt ", 8);
|
||||
buffer.writeUInt32LE(16, 16);
|
||||
buffer.writeUInt16LE(1, 20);
|
||||
buffer.writeUInt16LE(1, 22);
|
||||
buffer.writeUInt32LE(sampleRate, 24);
|
||||
buffer.writeUInt32LE(sampleRate * 2, 28);
|
||||
buffer.writeUInt16LE(2, 32);
|
||||
buffer.writeUInt16LE(16, 34);
|
||||
buffer.write("data", 36);
|
||||
buffer.writeUInt32LE(dataLength, 40);
|
||||
for (let index = 0; index < sampleCount; index += 1) {
|
||||
const value = Math.sin((2 * Math.PI * 440 * index) / sampleRate) * 0.2;
|
||||
buffer.writeInt16LE(Math.round(value * 32_767), 44 + index * 2);
|
||||
}
|
||||
return buffer;
|
||||
};
|
||||
|
||||
const wav = createWav();
|
||||
|
||||
const customWidgetPayload = {
|
||||
title: "Release v2 QA fixture",
|
||||
status: "ok",
|
||||
items: [
|
||||
{ id: "alpha", label: "Alpha", value: 42 },
|
||||
{ id: "beta", label: "Beta", value: 7 },
|
||||
],
|
||||
};
|
||||
|
||||
const parseArgs = () => {
|
||||
const args = process.argv.slice(2).filter((argument) => argument !== "--");
|
||||
let host = DEFAULT_HOST;
|
||||
let port = 0;
|
||||
let readyFile: string | undefined;
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
const value = args[index + 1];
|
||||
if (argument === "--host" && value) {
|
||||
host = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--port" && value) {
|
||||
port = Number(value);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--ready-file" && value) {
|
||||
readyFile = path.resolve(value);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown or incomplete argument: ${argument ?? "<missing>"}`);
|
||||
}
|
||||
|
||||
if (!Number.isInteger(port) || port < 0 || port > 65_535) {
|
||||
throw new Error("--port must be an integer between 0 and 65535");
|
||||
}
|
||||
return { host: validateLoopbackHost(host), port, readyFile };
|
||||
};
|
||||
|
||||
const setCommonHeaders = (response: ServerResponse) => {
|
||||
response.setHeader("Access-Control-Allow-Origin", "*");
|
||||
response.setHeader("Access-Control-Allow-Headers", "Content-Type, X-QA-Fixture");
|
||||
response.setHeader("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, OPTIONS");
|
||||
response.setHeader("Cache-Control", "no-store");
|
||||
response.setHeader("X-QA-Fixture", "release-v2");
|
||||
};
|
||||
|
||||
const send = (response: ServerResponse, status: number, contentType: string, body: string | Buffer) => {
|
||||
setCommonHeaders(response);
|
||||
response.writeHead(status, { "Content-Type": contentType });
|
||||
response.end(body);
|
||||
};
|
||||
|
||||
const sendJson = (response: ServerResponse, status: number, value: unknown) => {
|
||||
send(response, status, "application/json; charset=utf-8", `${JSON.stringify(value)}\n`);
|
||||
};
|
||||
|
||||
const readBody = async (request: IncomingMessage) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
for await (const chunk of request) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
size += buffer.length;
|
||||
if (size > MAX_BODY_BYTES) throw new Error("UPLOAD_TOO_LARGE");
|
||||
chunks.push(buffer);
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
};
|
||||
|
||||
const handleRequest = async (request: IncomingMessage, response: ServerResponse) => {
|
||||
const url = new URL(request.url ?? "/", `http://${request.headers.host ?? DEFAULT_HOST}`);
|
||||
const method = request.method ?? "GET";
|
||||
|
||||
if (method === "OPTIONS") {
|
||||
setCommonHeaders(response);
|
||||
response.writeHead(204);
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/health") {
|
||||
sendJson(response, 200, { status: "ok", fixture: "release-v2-qa" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/json") {
|
||||
sendJson(response, 200, {
|
||||
id: "qa-json-001",
|
||||
name: "Release v2 deterministic JSON",
|
||||
active: true,
|
||||
count: 3,
|
||||
tags: ["homarr", "release-v2", "qa"],
|
||||
nested: { score: 99, nullable: null },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/qa/custom-widget") {
|
||||
sendJson(response, 200, customWidgetPayload);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/rss") {
|
||||
send(
|
||||
response,
|
||||
200,
|
||||
"application/rss+xml; charset=utf-8",
|
||||
`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"><channel><title>Release v2 QA</title><link>https://example.invalid/qa</link><description>Deterministic feed</description><item><guid>qa-entry-1</guid><title>Alpha release note</title><link>https://example.invalid/qa/alpha</link><pubDate>Thu, 01 Jan 2026 00:00:00 GMT</pubDate><description>Stable fixture entry</description></item><item><guid>qa-entry-2</guid><title>Beta release note</title><link>https://example.invalid/qa/beta</link><pubDate>Fri, 02 Jan 2026 00:00:00 GMT</pubDate><description>Second stable fixture entry</description></item></channel></rss>\n`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/media/image.png") {
|
||||
send(response, 200, "image/png", png);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/media/image.svg") {
|
||||
send(
|
||||
response,
|
||||
200,
|
||||
"image/svg+xml; charset=utf-8",
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="320" height="180" viewBox="0 0 320 180"><rect width="320" height="180" fill="#111827"/><circle cx="72" cy="90" r="40" fill="#38bdf8"/><text x="132" y="98" fill="#f8fafc" font-family="sans-serif" font-size="22">Release v2 QA</text></svg>\n',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/media/audio.wav") {
|
||||
send(response, 200, "audio/wav", wav);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/media/video.mp4") {
|
||||
const video = await readFile(videoPath);
|
||||
const range = request.headers.range;
|
||||
setCommonHeaders(response);
|
||||
response.setHeader("Accept-Ranges", "bytes");
|
||||
if (!range) {
|
||||
response.writeHead(200, { "Content-Type": "video/mp4", "Content-Length": video.length });
|
||||
response.end(method === "HEAD" ? undefined : video);
|
||||
return;
|
||||
}
|
||||
|
||||
const match = /^bytes=(\d+)-(\d*)$/.exec(range);
|
||||
if (!match) {
|
||||
response.writeHead(416, { "Content-Range": `bytes */${video.length}` });
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
const start = Number(match[1]);
|
||||
const requestedEnd = match[2] ? Number(match[2]) : video.length - 1;
|
||||
const end = Math.min(requestedEnd, video.length - 1);
|
||||
if (start > end || start >= video.length) {
|
||||
response.writeHead(416, { "Content-Range": `bytes */${video.length}` });
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
const body = video.subarray(start, end + 1);
|
||||
response.writeHead(206, {
|
||||
"Content-Type": "video/mp4",
|
||||
"Content-Length": body.length,
|
||||
"Content-Range": `bytes ${start}-${end}/${video.length}`,
|
||||
});
|
||||
response.end(method === "HEAD" ? undefined : body);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/iframe") {
|
||||
send(
|
||||
response,
|
||||
200,
|
||||
"text/html; charset=utf-8",
|
||||
'<!doctype html><html><head><meta charset="utf-8"><title>Release v2 QA iframe</title></head><body><main data-qa-fixture="iframe"><h1>Deterministic iframe</h1><p>release-v2-qa</p></main></body></html>\n',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/api/qa/download/sample.txt") {
|
||||
setCommonHeaders(response);
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"Content-Disposition": 'attachment; filename="release-v2-qa-sample.txt"',
|
||||
});
|
||||
response.end("Release v2 QA deterministic download\nline=2\n");
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/upload") {
|
||||
if (method !== "POST" && method !== "PUT") {
|
||||
sendJson(response, 405, { error: "METHOD_NOT_ALLOWED", allowed: ["POST", "PUT"] });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const body = await readBody(request);
|
||||
sendJson(response, 200, {
|
||||
uploaded: true,
|
||||
method,
|
||||
bytes: body.length,
|
||||
contentType: request.headers["content-type"] ?? null,
|
||||
sha256: createHash("sha256").update(body).digest("hex"),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === "UPLOAD_TOO_LARGE") {
|
||||
sendJson(response, 413, { error: "UPLOAD_TOO_LARGE", maxBytes: MAX_BODY_BYTES });
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/empty") {
|
||||
setCommonHeaders(response);
|
||||
response.writeHead(204);
|
||||
response.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/malformed" || url.pathname === "/malformed/json") {
|
||||
send(response, 200, "application/json; charset=utf-8", '{"status":"broken","items":[1,2,}\n');
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/slow") {
|
||||
const requestedMs = Number(url.searchParams.get("ms") ?? "1500");
|
||||
const delayMs = Number.isFinite(requestedMs) ? Math.max(0, Math.min(Math.trunc(requestedMs), MAX_SLOW_MS)) : 1500;
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
sendJson(response, 200, { status: "ok", delayedMs: delayMs });
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/error" || url.pathname.startsWith("/error/")) {
|
||||
const pathStatus = url.pathname.split("/")[2];
|
||||
const requestedStatus = Number(url.searchParams.get("status") ?? pathStatus ?? "503");
|
||||
const status =
|
||||
Number.isInteger(requestedStatus) && requestedStatus >= 400 && requestedStatus <= 599 ? requestedStatus : 503;
|
||||
sendJson(response, status, { error: "QA_CONFIGURED_HTTP_ERROR", status });
|
||||
return;
|
||||
}
|
||||
|
||||
sendJson(response, 404, { error: "NOT_FOUND", path: url.pathname });
|
||||
};
|
||||
|
||||
const writeJsonAtomic = async (filePath: string, value: unknown) => {
|
||||
await mkdir(path.dirname(filePath), { recursive: true });
|
||||
const temporaryPath = `${filePath}.${process.pid}.tmp`;
|
||||
const handle = await open(
|
||||
temporaryPath,
|
||||
constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW,
|
||||
0o600,
|
||||
);
|
||||
try {
|
||||
await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
try {
|
||||
await rename(temporaryPath, filePath);
|
||||
} catch (error) {
|
||||
await rm(temporaryPath, { force: true });
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const options = parseArgs();
|
||||
const server = createServer((request, response) => {
|
||||
void handleRequest(request, response).catch((error: unknown) => {
|
||||
console.error(error);
|
||||
if (!response.headersSent) sendJson(response, 500, { error: "FIXTURE_INTERNAL_ERROR" });
|
||||
else response.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(options.port, options.host, resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Fixture server did not expose a TCP address");
|
||||
const urlHost = options.host.includes(":") ? `[${options.host}]` : options.host;
|
||||
const url = `http://${urlHost}:${address.port}`;
|
||||
const ready = { schemaVersion: 1, pid: process.pid, host: options.host, port: address.port, url };
|
||||
if (options.readyFile) await writeJsonAtomic(options.readyFile, ready);
|
||||
process.stdout.write(`${JSON.stringify(ready)}\n`);
|
||||
|
||||
const shutdown = () => {
|
||||
server.close(() => process.exit(0));
|
||||
setTimeout(() => process.exit(1), 5_000).unref();
|
||||
};
|
||||
process.once("SIGINT", shutdown);
|
||||
process.once("SIGTERM", shutdown);
|
||||
};
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,152 @@
|
||||
import type { LayoutRole } from "../../packages/definitions/index.ts";
|
||||
import { getBoardLaneColumnCount, getRootSectionLane } from "../../packages/definitions/index.ts";
|
||||
|
||||
interface GeometryLayout {
|
||||
id: string;
|
||||
boardId: string;
|
||||
columnCount: number;
|
||||
leftGutterColumnCount: number;
|
||||
rightGutterColumnCount: number;
|
||||
role: LayoutRole;
|
||||
}
|
||||
|
||||
interface GeometrySection {
|
||||
id: string;
|
||||
boardId: string;
|
||||
kind: "empty" | "container";
|
||||
xOffset: number | null;
|
||||
}
|
||||
|
||||
interface GeometryPlacement {
|
||||
layoutId: string;
|
||||
sectionId: string;
|
||||
xOffset: number;
|
||||
yOffset: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
interface GeometrySectionLayout extends GeometryPlacement {
|
||||
parentSectionId: string | null;
|
||||
}
|
||||
|
||||
interface GeometryItemLayout extends GeometryPlacement {
|
||||
itemId: string;
|
||||
boardId: string;
|
||||
}
|
||||
|
||||
export interface FixtureGeometry {
|
||||
layouts: GeometryLayout[];
|
||||
sections: GeometrySection[];
|
||||
sectionLayouts: GeometrySectionLayout[];
|
||||
itemLayouts: GeometryItemLayout[];
|
||||
}
|
||||
|
||||
const placementIsValid = (placement: GeometryPlacement) =>
|
||||
Number.isInteger(placement.xOffset) &&
|
||||
Number.isInteger(placement.yOffset) &&
|
||||
Number.isInteger(placement.width) &&
|
||||
Number.isInteger(placement.height) &&
|
||||
placement.xOffset >= 0 &&
|
||||
placement.yOffset >= 0 &&
|
||||
placement.width >= 1 &&
|
||||
placement.height >= 1;
|
||||
|
||||
export const findFixtureGeometryErrors = (geometry: FixtureGeometry) => {
|
||||
const errors: string[] = [];
|
||||
const layoutsById = new Map(geometry.layouts.map((layout) => [layout.id, layout]));
|
||||
const sectionsById = new Map(geometry.sections.map((section) => [section.id, section]));
|
||||
const sectionLayoutsByKey = new Map<string, GeometrySectionLayout>();
|
||||
|
||||
for (const sectionLayout of geometry.sectionLayouts) {
|
||||
const key = `${sectionLayout.layoutId}\0${sectionLayout.sectionId}`;
|
||||
if (sectionLayoutsByKey.has(key)) {
|
||||
errors.push(`container ${sectionLayout.sectionId} has duplicate placement in layout ${sectionLayout.layoutId}`);
|
||||
continue;
|
||||
}
|
||||
sectionLayoutsByKey.set(key, sectionLayout);
|
||||
}
|
||||
|
||||
const parentCapacity = (
|
||||
layout: GeometryLayout,
|
||||
parentSection: GeometrySection,
|
||||
): { columns: number; kind: "lane" | "container" } | undefined => {
|
||||
if (parentSection.kind === "empty") {
|
||||
return {
|
||||
columns: getBoardLaneColumnCount(layout, getRootSectionLane(parentSection.xOffset)),
|
||||
kind: "lane",
|
||||
};
|
||||
}
|
||||
|
||||
const parentLayout = sectionLayoutsByKey.get(`${layout.id}\0${parentSection.id}`);
|
||||
if (!parentLayout) return undefined;
|
||||
return { columns: parentLayout.width, kind: "container" };
|
||||
};
|
||||
|
||||
for (const sectionLayout of geometry.sectionLayouts) {
|
||||
const layout = layoutsById.get(sectionLayout.layoutId);
|
||||
const section = sectionsById.get(sectionLayout.sectionId);
|
||||
const parent = sectionLayout.parentSectionId ? sectionsById.get(sectionLayout.parentSectionId) : undefined;
|
||||
if (!layout || !section || !parent) {
|
||||
errors.push(
|
||||
`container ${sectionLayout.sectionId} in layout ${sectionLayout.layoutId} has unknown geometry references`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (section.kind !== "container" || section.boardId !== layout.boardId || parent.boardId !== layout.boardId) {
|
||||
errors.push(`container ${section.id} in layout ${layout.id} crosses a board or root boundary`);
|
||||
continue;
|
||||
}
|
||||
if (!placementIsValid(sectionLayout)) {
|
||||
errors.push(`container ${section.id} in layout ${layout.id} has an invalid placement`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const capacity = parentCapacity(layout, parent);
|
||||
if (!capacity) {
|
||||
errors.push(`container ${section.id} in layout ${layout.id} has an unplaced parent container`);
|
||||
} else if (sectionLayout.xOffset + sectionLayout.width > capacity.columns) {
|
||||
errors.push(
|
||||
`container ${section.id} in layout ${layout.id} exceeds its ${capacity.columns}-column parent ${capacity.kind}`,
|
||||
);
|
||||
}
|
||||
|
||||
const visited = new Set<string>();
|
||||
let currentSectionId: string | null = section.id;
|
||||
while (currentSectionId) {
|
||||
if (visited.has(currentSectionId)) {
|
||||
errors.push(`container cycle detected in layout ${layout.id} at ${currentSectionId}`);
|
||||
break;
|
||||
}
|
||||
visited.add(currentSectionId);
|
||||
const current = sectionsById.get(currentSectionId);
|
||||
if (!current || current.kind === "empty") break;
|
||||
currentSectionId = sectionLayoutsByKey.get(`${layout.id}\0${currentSectionId}`)?.parentSectionId ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
for (const itemLayout of geometry.itemLayouts) {
|
||||
const layout = layoutsById.get(itemLayout.layoutId);
|
||||
const parent = sectionsById.get(itemLayout.sectionId);
|
||||
if (!layout || !parent || itemLayout.boardId !== layout.boardId || parent.boardId !== layout.boardId) {
|
||||
errors.push(`item ${itemLayout.itemId} in layout ${itemLayout.layoutId} crosses a board or section boundary`);
|
||||
continue;
|
||||
}
|
||||
if (!placementIsValid(itemLayout)) {
|
||||
errors.push(`item ${itemLayout.itemId} in layout ${layout.id} has an invalid placement`);
|
||||
continue;
|
||||
}
|
||||
const capacity = parentCapacity(layout, parent);
|
||||
if (!capacity) {
|
||||
errors.push(`item ${itemLayout.itemId} in layout ${layout.id} has an unplaced parent container`);
|
||||
continue;
|
||||
}
|
||||
if (itemLayout.xOffset + itemLayout.width > capacity.columns) {
|
||||
errors.push(
|
||||
`item ${itemLayout.itemId} in layout ${layout.id} exceeds its ${capacity.columns}-column parent ${capacity.kind}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
};
|
||||
@@ -0,0 +1,119 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import type { FixtureGeometry } from "./geometry.mts";
|
||||
import { findFixtureGeometryErrors } from "./geometry.mts";
|
||||
|
||||
const baseGeometry = (): FixtureGeometry => ({
|
||||
layouts: [
|
||||
{
|
||||
id: "layout-mobile",
|
||||
boardId: "board",
|
||||
columnCount: 3,
|
||||
leftGutterColumnCount: 0,
|
||||
rightGutterColumnCount: 0,
|
||||
role: "mobile",
|
||||
},
|
||||
],
|
||||
sections: [
|
||||
{ id: "root", boardId: "board", kind: "empty", xOffset: 0 },
|
||||
{ id: "level-1", boardId: "board", kind: "container", xOffset: null },
|
||||
{ id: "level-2", boardId: "board", kind: "container", xOffset: null },
|
||||
{ id: "level-3", boardId: "board", kind: "container", xOffset: null },
|
||||
],
|
||||
sectionLayouts: [
|
||||
{
|
||||
sectionId: "level-1",
|
||||
layoutId: "layout-mobile",
|
||||
parentSectionId: "root",
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
width: 3,
|
||||
height: 6,
|
||||
},
|
||||
{
|
||||
sectionId: "level-2",
|
||||
layoutId: "layout-mobile",
|
||||
parentSectionId: "level-1",
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
width: 2,
|
||||
height: 5,
|
||||
},
|
||||
{
|
||||
sectionId: "level-3",
|
||||
layoutId: "layout-mobile",
|
||||
parentSectionId: "level-2",
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
width: 1,
|
||||
height: 4,
|
||||
},
|
||||
],
|
||||
itemLayouts: [
|
||||
{
|
||||
itemId: "nested-item",
|
||||
boardId: "board",
|
||||
sectionId: "level-3",
|
||||
layoutId: "layout-mobile",
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
width: 1,
|
||||
height: 2,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
describe("release-v2 QA fixture geometry", () => {
|
||||
test("rejects an item that fits total columns but overflows the gutter-reduced main lane", () => {
|
||||
const geometry = baseGeometry();
|
||||
geometry.layouts[0] = {
|
||||
id: "layout-mobile",
|
||||
boardId: "board",
|
||||
role: "base",
|
||||
columnCount: 12,
|
||||
leftGutterColumnCount: 2,
|
||||
rightGutterColumnCount: 2,
|
||||
};
|
||||
geometry.sections = [{ id: "root", boardId: "board", kind: "empty", xOffset: 0 }];
|
||||
geometry.sectionLayouts = [];
|
||||
geometry.itemLayouts = [
|
||||
{
|
||||
itemId: "gutter-overflow",
|
||||
boardId: "board",
|
||||
sectionId: "root",
|
||||
layoutId: "layout-mobile",
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
width: 9,
|
||||
height: 1,
|
||||
},
|
||||
];
|
||||
|
||||
expect(findFixtureGeometryErrors(geometry)).toContain(
|
||||
"item gutter-overflow in layout layout-mobile exceeds its 8-column parent lane",
|
||||
);
|
||||
});
|
||||
|
||||
test("accepts valid three-level mobile nesting and rejects a child wider than its container", () => {
|
||||
const geometry = baseGeometry();
|
||||
expect(findFixtureGeometryErrors(geometry)).toEqual([]);
|
||||
|
||||
const nestedItem = geometry.itemLayouts.at(0);
|
||||
if (!nestedItem) throw new Error("Missing nested fixture item");
|
||||
nestedItem.width = 2;
|
||||
expect(findFixtureGeometryErrors(geometry)).toContain(
|
||||
"item nested-item in layout layout-mobile exceeds its 1-column parent container",
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects recursive container parent cycles", () => {
|
||||
const geometry = baseGeometry();
|
||||
const firstSection = geometry.sectionLayouts.at(0);
|
||||
if (!firstSection) throw new Error("Missing nested fixture section");
|
||||
firstSection.parentSectionId = "level-3";
|
||||
|
||||
expect(findFixtureGeometryErrors(geometry).some((error) => error.includes("container cycle"))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const fullShaPattern = /^[0-9a-f]{40}$/iu;
|
||||
|
||||
export const releaseV2QaProfiles = ["main-writable", "main-readonly", "onboarding-fresh", "degraded"] as const;
|
||||
export type ReleaseV2QaProfile = (typeof releaseV2QaProfiles)[number];
|
||||
|
||||
export interface ReleaseV2QaProfileFlags {
|
||||
demoMode: boolean;
|
||||
demoReadOnly: boolean;
|
||||
unsafeMockIntegration: boolean;
|
||||
}
|
||||
|
||||
const expectedProfileFlags: Record<ReleaseV2QaProfile, ReleaseV2QaProfileFlags> = {
|
||||
"main-writable": { demoMode: true, demoReadOnly: false, unsafeMockIntegration: true },
|
||||
"main-readonly": { demoMode: true, demoReadOnly: true, unsafeMockIntegration: true },
|
||||
"onboarding-fresh": { demoMode: false, demoReadOnly: false, unsafeMockIntegration: true },
|
||||
degraded: { demoMode: true, demoReadOnly: false, unsafeMockIntegration: true },
|
||||
};
|
||||
|
||||
const profileFlagEnvironmentNames = {
|
||||
demoMode: "DEMO_MODE",
|
||||
demoReadOnly: "DEMO_READ_ONLY",
|
||||
unsafeMockIntegration: "UNSAFE_ENABLE_MOCK_INTEGRATION",
|
||||
} as const satisfies Record<keyof ReleaseV2QaProfileFlags, string>;
|
||||
|
||||
const parseStrictBoolean = (value: string | undefined) => {
|
||||
if (value === "true") return true;
|
||||
if (value === "false") return false;
|
||||
return null;
|
||||
};
|
||||
|
||||
export const assertReleaseV2QaProfileEnvironment = (
|
||||
profile: ReleaseV2QaProfile,
|
||||
environment: NodeJS.ProcessEnv,
|
||||
): ReleaseV2QaProfileFlags => {
|
||||
const expected = expectedProfileFlags[profile];
|
||||
const invalidFlags: string[] = [];
|
||||
|
||||
for (const [flag, environmentName] of Object.entries(profileFlagEnvironmentNames) as Array<
|
||||
[keyof ReleaseV2QaProfileFlags, string]
|
||||
>) {
|
||||
const expectedValue = expected[flag];
|
||||
const actualValue = parseStrictBoolean(environment[environmentName]);
|
||||
if (actualValue !== expectedValue) invalidFlags.push(`${environmentName} (expected ${String(expectedValue)})`);
|
||||
}
|
||||
|
||||
if (invalidFlags.length > 0) {
|
||||
throw new Error(`QA profile '${profile}' has invalid or missing environment flags: ${invalidFlags.join(", ")}`);
|
||||
}
|
||||
|
||||
return { ...expected };
|
||||
};
|
||||
|
||||
export const validateCandidateSha = (candidateSha: string, checkoutSha: string) => {
|
||||
if (!fullShaPattern.test(candidateSha)) {
|
||||
throw new Error("QA candidate must be a full 40-character Git SHA");
|
||||
}
|
||||
if (!fullShaPattern.test(checkoutSha)) {
|
||||
throw new Error("The checked-out HEAD did not resolve to a full 40-character Git SHA");
|
||||
}
|
||||
if (candidateSha.toLowerCase() !== checkoutSha.toLowerCase()) {
|
||||
throw new Error("QA candidate does not match the checked-out HEAD");
|
||||
}
|
||||
return checkoutSha.toLowerCase();
|
||||
};
|
||||
|
||||
export const resolveCheckoutCandidateSha = (repoRoot: string) => {
|
||||
const result = spawnSync("git", ["rev-parse", "--verify", "HEAD^{commit}"], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (result.status !== 0) throw new Error("Could not resolve the checked-out HEAD for QA provenance");
|
||||
const checkoutSha = result.stdout.trim();
|
||||
return validateCandidateSha(checkoutSha, checkoutSha);
|
||||
};
|
||||
@@ -0,0 +1,133 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { access, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import Database from "better-sqlite3";
|
||||
import { afterEach, describe, expect, test } from "vitest";
|
||||
|
||||
import { assertReleaseV2QaProfileEnvironment, releaseV2QaProfiles, validateCandidateSha } from "./provenance.mts";
|
||||
import type { ReleaseV2QaProfile } from "./provenance.mts";
|
||||
|
||||
const disposableDirectories: string[] = [];
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const profileEnvironmentNames = ["DEMO_MODE", "DEMO_READ_ONLY", "UNSAFE_ENABLE_MOCK_INTEGRATION"] as const;
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
disposableDirectories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })),
|
||||
);
|
||||
});
|
||||
|
||||
const environmentForProfile = (profile: ReleaseV2QaProfile): NodeJS.ProcessEnv => {
|
||||
const environment: NodeJS.ProcessEnv = {
|
||||
DEMO_MODE: "true",
|
||||
DEMO_READ_ONLY: "false",
|
||||
UNSAFE_ENABLE_MOCK_INTEGRATION: "true",
|
||||
};
|
||||
if (profile === "main-readonly") environment.DEMO_READ_ONLY = "true";
|
||||
if (profile === "onboarding-fresh") environment.DEMO_MODE = "false";
|
||||
return environment;
|
||||
};
|
||||
|
||||
describe("release-v2 QA candidate provenance", () => {
|
||||
test("accepts only the exact checked-out commit", () => {
|
||||
const checkoutSha = "a".repeat(40);
|
||||
|
||||
expect(validateCandidateSha(checkoutSha.toUpperCase(), checkoutSha)).toBe(checkoutSha);
|
||||
expect(() => validateCandidateSha("b".repeat(40), checkoutSha)).toThrow("checked-out HEAD");
|
||||
expect(() => validateCandidateSha("not-a-sha", checkoutSha)).toThrow("full 40-character Git SHA");
|
||||
});
|
||||
});
|
||||
|
||||
describe("release-v2 QA profile provenance", () => {
|
||||
test.each(releaseV2QaProfiles)("accepts the exact %s profile matrix", (profile) => {
|
||||
expect(assertReleaseV2QaProfileEnvironment(profile, environmentForProfile(profile))).toEqual({
|
||||
demoMode: profile !== "onboarding-fresh",
|
||||
demoReadOnly: profile === "main-readonly",
|
||||
unsafeMockIntegration: true,
|
||||
});
|
||||
});
|
||||
|
||||
test.each(profileEnvironmentNames)("rejects an unset %s flag", (environmentName) => {
|
||||
const environment = environmentForProfile("main-writable");
|
||||
delete environment[environmentName];
|
||||
|
||||
expect(() => assertReleaseV2QaProfileEnvironment("main-writable", environment)).toThrow(
|
||||
new RegExp(`${environmentName} \\(expected (?:true|false)\\)`, "u"),
|
||||
);
|
||||
});
|
||||
|
||||
test.each(["1", "0", "TRUE", "False", "yes", " true ", "", "unexpected-private-marker"])(
|
||||
"rejects the malformed strict boolean %j without echoing it",
|
||||
(malformedValue) => {
|
||||
for (const environmentName of profileEnvironmentNames) {
|
||||
const environment = environmentForProfile("main-writable");
|
||||
environment[environmentName] = malformedValue;
|
||||
|
||||
expect(() => assertReleaseV2QaProfileEnvironment("main-writable", environment)).toThrow(environmentName);
|
||||
try {
|
||||
assertReleaseV2QaProfileEnvironment("main-writable", environment);
|
||||
} catch (error) {
|
||||
expect((error as Error).message).not.toContain(malformedValue || "unexpected-private-marker");
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test.each(releaseV2QaProfiles)("rejects every mismatched flag for %s", (profile) => {
|
||||
const expectedEnvironment = environmentForProfile(profile);
|
||||
for (const environmentName of profileEnvironmentNames) {
|
||||
const environment = { ...expectedEnvironment };
|
||||
environment[environmentName] = environment[environmentName] === "true" ? "false" : "true";
|
||||
expect(() => assertReleaseV2QaProfileEnvironment(profile, environment)).toThrow(environmentName);
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects a writable ambient environment before a read-only reset or manifest write", async () => {
|
||||
const disposableDirectory = await mkdtemp(path.join(tmpdir(), "homarr-release-v2-qa-profile-flags-"));
|
||||
disposableDirectories.push(disposableDirectory);
|
||||
const databasePath = path.join(disposableDirectory, "fixture.sqlite");
|
||||
const outputDirectory = path.join(disposableDirectory, "output");
|
||||
const database = new Database(databasePath);
|
||||
database.exec("CREATE TABLE mutation_guard (value TEXT NOT NULL); INSERT INTO mutation_guard VALUES ('preserved')");
|
||||
database.close();
|
||||
const databaseBefore = await readFile(databasePath);
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
path.join(repoRoot, "scripts/release-v2-qa/seed.mts"),
|
||||
"--profile",
|
||||
"main-readonly",
|
||||
"--output",
|
||||
outputDirectory,
|
||||
"--reset",
|
||||
],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
DB_DRIVER: "better-sqlite3",
|
||||
DB_URL: databasePath,
|
||||
DEMO_MODE: "true",
|
||||
DEMO_READ_ONLY: "false",
|
||||
UNSAFE_ENABLE_MOCK_INTEGRATION: "true",
|
||||
},
|
||||
timeout: 30_000,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain("DEMO_READ_ONLY (expected true)");
|
||||
expect(result.stderr).not.toContain("no such table");
|
||||
expect(await readFile(databasePath)).toEqual(databaseBefore);
|
||||
await expect(access(path.join(outputDirectory, "fixture-manifest.json"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,338 @@
|
||||
import { lstat, readFile, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { resolveCheckoutCandidateSha } from "./provenance.mts";
|
||||
import { validateResolvedArtifactPath, writeSafeReportFile } from "./report-path-integrity.mts";
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const qaRoot = path.join(repoRoot, "qa/release-v2");
|
||||
const candidateSha = resolveCheckoutCandidateSha(repoRoot);
|
||||
|
||||
type Status = "passed" | "failed" | "blocked" | "not-reached";
|
||||
|
||||
interface Packet {
|
||||
id: string;
|
||||
wave: string;
|
||||
title: string;
|
||||
prRefs: number[];
|
||||
personas: string[];
|
||||
boards: string[];
|
||||
profiles: string[];
|
||||
viewports: string[];
|
||||
zooms: number[];
|
||||
inputs: string[];
|
||||
widgetKinds?: string[];
|
||||
cases: string[];
|
||||
}
|
||||
|
||||
interface Manifest {
|
||||
caseDimensions: Record<string, { id: string; expected: string }[]>;
|
||||
packets: Packet[];
|
||||
}
|
||||
|
||||
interface Options {
|
||||
packetId: string;
|
||||
url: string;
|
||||
profile: string;
|
||||
persona: string;
|
||||
session: string;
|
||||
viewport: string;
|
||||
input: string;
|
||||
zoom: number;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
const profileFlags: Record<string, string[]> = {
|
||||
"main-writable": ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
"main-readonly": ["DEMO_MODE=true", "DEMO_READ_ONLY=true", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
"onboarding-fresh": ["DEMO_MODE=false", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
degraded: ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
};
|
||||
|
||||
const exhaustiveSizeWidgetKinds = new Set([
|
||||
"downloads",
|
||||
"calendar",
|
||||
"mediaServer",
|
||||
"mediaRequests-requestList",
|
||||
"mediaRequests-requestStats",
|
||||
"systemResources",
|
||||
"systemDisks",
|
||||
"dockerContainers",
|
||||
"networkControllerSummary",
|
||||
"networkControllerStatus",
|
||||
"beszelSystemStats",
|
||||
"uptimeKuma",
|
||||
"bookmarks",
|
||||
"notebook",
|
||||
"iframe",
|
||||
"customApi",
|
||||
"assistant",
|
||||
]);
|
||||
|
||||
const parseOptions = (): Options => {
|
||||
const values = new Map<string, string>();
|
||||
const args = process.argv.slice(2).filter((argument) => argument !== "--");
|
||||
for (let index = 0; index < args.length; index += 2) {
|
||||
const key = args[index];
|
||||
const value = args[index + 1];
|
||||
if (!key?.startsWith("--") || !value) throw new Error(`Invalid argument ${key ?? "<missing>"}`);
|
||||
values.set(key.slice(2), value);
|
||||
}
|
||||
|
||||
const required = (key: string) => {
|
||||
const value = values.get(key);
|
||||
if (!value) throw new Error(`--${key} is required`);
|
||||
return value;
|
||||
};
|
||||
const zoom = Number(required("zoom"));
|
||||
if (!Number.isFinite(zoom) || zoom <= 0) throw new Error("--zoom must be a positive number");
|
||||
return {
|
||||
packetId: required("packet"),
|
||||
url: required("url"),
|
||||
profile: required("profile"),
|
||||
persona: required("persona"),
|
||||
session: required("session"),
|
||||
viewport: required("viewport"),
|
||||
input: required("input"),
|
||||
zoom,
|
||||
timestamp: values.get("timestamp") ?? new Date().toISOString(),
|
||||
};
|
||||
};
|
||||
|
||||
const listFiles = async (directory: string): Promise<string[]> => {
|
||||
const output: string[] = [];
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const entryPath = path.join(directory, entry.name);
|
||||
if (entry.isDirectory()) output.push(...(await listFiles(entryPath)));
|
||||
else if (entry.isFile()) {
|
||||
const artifactError = await validateResolvedArtifactPath(directory, entryPath, "blocked-report artifact");
|
||||
if (artifactError) throw new Error(artifactError);
|
||||
const file = await lstat(entryPath);
|
||||
if (!entryPath.endsWith(".webm") || file.size > 0) output.push(entryPath);
|
||||
} else {
|
||||
throw new Error(`Blocked-report artifact tree contains a non-regular entry: ${entry.name}`);
|
||||
}
|
||||
}
|
||||
return output.toSorted();
|
||||
};
|
||||
|
||||
const escapeCell = (value: string) => value.replaceAll("|", "\\|").replaceAll("\n", " ");
|
||||
const list = (values: (string | number)[]) => {
|
||||
if (values.length === 0) return "None";
|
||||
return values.join(", ");
|
||||
};
|
||||
|
||||
const sizeRequirement = (widgetKind: string) => {
|
||||
if (exhaustiveSizeWidgetKinds.has(widgetKind)) return "every width 1-24 × every height 1-6";
|
||||
return "minimum / canonical / wide / tall / maximum / overflow / behavior-changing thresholds";
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const options = parseOptions();
|
||||
const manifest = JSON.parse(await readFile(path.join(qaRoot, "coverage-manifest.json"), "utf8")) as Manifest;
|
||||
const packet = manifest.packets.find((candidate) => candidate.id === options.packetId);
|
||||
if (!packet) throw new Error(`Unknown packet ${options.packetId}`);
|
||||
if (!packet.profiles.includes(options.profile)) throw new Error(`${packet.id} does not include ${options.profile}`);
|
||||
if (!packet.personas.includes(options.persona)) throw new Error(`${packet.id} does not include ${options.persona}`);
|
||||
if (!packet.viewports.includes(options.viewport))
|
||||
throw new Error(`${packet.id} does not include ${options.viewport}`);
|
||||
if (!packet.inputs.includes(options.input)) throw new Error(`${packet.id} does not include ${options.input}`);
|
||||
if (!packet.zooms.includes(options.zoom)) throw new Error(`${packet.id} does not include ${options.zoom}%`);
|
||||
const flags = profileFlags[options.profile];
|
||||
if (!flags) throw new Error(`Unknown runtime profile ${options.profile}`);
|
||||
|
||||
const runtimeUrl = new URL(options.url);
|
||||
const actualPort = Number(runtimeUrl.port);
|
||||
if (!Number.isInteger(actualPort) || actualPort <= 0) throw new Error("Runtime URL must include an explicit port");
|
||||
const artifactDirectory = path.join(repoRoot, ".screenshots/release-v2", packet.id);
|
||||
const artifacts = await listFiles(artifactDirectory);
|
||||
if (artifacts.length === 0) throw new Error(`${packet.id} has no browser artifacts`);
|
||||
|
||||
const dimensions = manifest.caseDimensions[packet.wave] ?? [];
|
||||
const cases = packet.cases.flatMap((baseId) =>
|
||||
dimensions.map((dimension) => ({ id: `${baseId}-${dimension.id}`, expected: dimension.expected })),
|
||||
);
|
||||
const caseStatuses: Record<string, Status> = {};
|
||||
for (const testCase of cases) {
|
||||
let status: Status = "blocked";
|
||||
if (packet.wave === "preflight" && testCase.id.endsWith("-ENVIRONMENT")) status = "failed";
|
||||
if (packet.wave === "preflight" && testCase.id.endsWith("-EVIDENCE")) status = "passed";
|
||||
caseStatuses[testCase.id] = status;
|
||||
}
|
||||
const status: Status = Object.values(caseStatuses).includes("failed") ? "failed" : "blocked";
|
||||
const blockedCaseIds = Object.entries(caseStatuses)
|
||||
.filter(([, caseStatus]) => caseStatus !== "passed")
|
||||
.map(([caseId]) => caseId);
|
||||
const findingEvidence = artifacts.filter(
|
||||
(artifact) => artifact.endsWith(".png") || artifact.endsWith(".webm") || artifact.includes("network"),
|
||||
);
|
||||
const evidence = findingEvidence.length > 0 ? findingEvidence : artifacts;
|
||||
const hasRetryEvidence = artifacts.some((artifact) => /fresh|retry/u.test(path.basename(artifact)));
|
||||
let findingSummary =
|
||||
"Chromium repeatedly requests the same entry route and ends on ERR_TOO_MANY_REDIRECTS, preventing the assigned browser flow.";
|
||||
let retryLimitation =
|
||||
"The packet's retry did not produce a separate artifact; the same finding has independent reproductions in the preflight reports.";
|
||||
let retryStep = "5. Compare the result with the independently reproduced preflight evidence.";
|
||||
let reproducibility = "reproduced independently by the preflight agents";
|
||||
if (hasRetryEvidence) {
|
||||
findingSummary =
|
||||
"A fresh Chromium session repeatedly requests the same entry route and ends on ERR_TOO_MANY_REDIRECTS; a fresh-context retry does the same.";
|
||||
retryLimitation =
|
||||
"The finding was retried in a fresh Chromium context; console output was empty while document requests repeated until Chromium stopped the loop.";
|
||||
retryStep = "5. Close the context, retry in a fresh context, and observe the same result.";
|
||||
reproducibility = "reproduced on retry and independently by the preflight agents";
|
||||
}
|
||||
const reproductionAgent = packet.id === "preflight-01" ? "qa-v2-preflight-02" : "qa-v2-preflight-01";
|
||||
const widgetChecks = (packet.widgetKinds ?? []).flatMap((widgetKind) =>
|
||||
packet.viewports.map((viewport) => ({
|
||||
widgetKind,
|
||||
viewport,
|
||||
sizeRequirement: sizeRequirement(widgetKind),
|
||||
status: "blocked" as const,
|
||||
})),
|
||||
);
|
||||
const metadata = {
|
||||
packetId: packet.id,
|
||||
status,
|
||||
caseStatuses,
|
||||
execution: {
|
||||
candidateSha,
|
||||
url: options.url,
|
||||
actualPort,
|
||||
runtimeProfile: options.profile,
|
||||
runtimeFlags: flags,
|
||||
persona: options.persona,
|
||||
sessionId: options.session,
|
||||
timestamp: options.timestamp,
|
||||
viewport: options.viewport,
|
||||
input: options.input,
|
||||
zoom: options.zoom,
|
||||
},
|
||||
findings: [
|
||||
{
|
||||
id: "RV2-P1-001",
|
||||
fingerprint: "release-v2-auth-locale-redirect-loop",
|
||||
severity: "P1",
|
||||
area: "routing/authentication",
|
||||
title: "Locale rewrite causes an infinite browser redirect before the application renders",
|
||||
summary: findingSummary,
|
||||
caseIds: blockedCaseIds,
|
||||
evidence,
|
||||
},
|
||||
],
|
||||
artifacts,
|
||||
widgetChecks,
|
||||
performance: {
|
||||
measurements: [],
|
||||
limitations: [
|
||||
"The application never rendered, so assigned feature, mutation, persistence, permission, viewport, and performance checks were blocked.",
|
||||
"No credentials were submitted and no product state was mutated.",
|
||||
retryLimitation,
|
||||
],
|
||||
},
|
||||
independentReproductions: [
|
||||
{
|
||||
findingFingerprint: "release-v2-auth-locale-redirect-loop",
|
||||
agentId: reproductionAgent,
|
||||
outcome: "reproduced",
|
||||
evidence: [],
|
||||
notes: `See ${reproductionAgent.replace("qa-v2-", "")} for an independent Luna Max reproduction.`,
|
||||
},
|
||||
],
|
||||
notes: "Report-only black-box run. No defect was fixed and no secret was recorded.",
|
||||
};
|
||||
|
||||
const caseRows = cases
|
||||
.map((testCase) => {
|
||||
const caseStatus = caseStatuses[testCase.id];
|
||||
return `| ${testCase.id} | ${caseStatus} | [browser artifacts](${artifactDirectory}) | The application entry repeated redirects until Chromium stopped navigation; the assigned flow was not reachable. | ${escapeCell(testCase.expected)} |`;
|
||||
})
|
||||
.join("\n");
|
||||
let widgetSection = "";
|
||||
if (widgetChecks.length > 0) {
|
||||
const widgetRows = widgetChecks
|
||||
.map(
|
||||
(check) =>
|
||||
`| ${check.widgetKind} | ${check.viewport} | ${check.sizeRequirement} | blocked before render | blocked before authentication | not reached | redirect retry reproduced | [artifacts](${artifactDirectory}) | ${check.status} |`,
|
||||
)
|
||||
.join("\n");
|
||||
widgetSection = `\n## Widget kind × viewport checklist\n\n| Widget kind | Viewport | Size | States | Permission/read-only | Options/persistence | Recovery | Evidence | Status |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n${widgetRows}\n`;
|
||||
}
|
||||
const artifactRows = artifacts.map((artifact) => `- [${path.basename(artifact)}](${artifact})`).join("\n");
|
||||
const report = `<!-- release-v2-qa-report
|
||||
${JSON.stringify(metadata, null, 2)}
|
||||
-->
|
||||
|
||||
# ${packet.id}: ${packet.title}
|
||||
|
||||
| Metadata | Value |
|
||||
| --- | --- |
|
||||
| Status | ${status} |
|
||||
| Wave | ${packet.wave} |
|
||||
| PR refs | ${packet.prRefs.map((number) => `#${number}`).join(", ")} |
|
||||
| Personas | ${list(packet.personas)} |
|
||||
| Boards | ${list(packet.boards)} |
|
||||
| Profiles | ${list(packet.profiles)} |
|
||||
| Chromium viewports | ${list(packet.viewports)} |
|
||||
| Zoom | ${list(packet.zooms.map((zoom) => `${zoom}%`))} |
|
||||
| Input | ${list(packet.inputs)} |
|
||||
| Widget kinds | ${list(packet.widgetKinds ?? [])} |
|
||||
| Artifact directory | [${artifactDirectory}](${artifactDirectory}) |
|
||||
| Candidate SHA | ${candidateSha} |
|
||||
| URL / actual port | ${options.url} / ${actualPort} |
|
||||
| Runtime profile / flags | ${options.profile}; ${flags.join(", ")} |
|
||||
| Executed persona | ${options.persona} |
|
||||
| Browser session ID | ${options.session} |
|
||||
| Timestamp | ${options.timestamp} |
|
||||
| Executed viewport / input / zoom | ${options.viewport} / ${options.input} / ${options.zoom}% |
|
||||
|
||||
## Dogfood evidence
|
||||
|
||||
| Case ID | Status | Evidence | Observed | Expected |
|
||||
| --- | --- | --- | --- | --- |
|
||||
${caseRows}
|
||||
${widgetSection}
|
||||
## Performance measurements and limitations
|
||||
|
||||
No valid feature timing could be recorded because no application document rendered. Cold/warm, polling, rendering, memory, and recovery measurements remain blocked.
|
||||
|
||||
## Independent reproductions
|
||||
|
||||
The common routing failure was independently reproduced by ${reproductionAgent}; the corresponding tracked packet contains its own browser evidence.
|
||||
|
||||
## Findings
|
||||
|
||||
### RV2-P1-001 — P1: locale rewrite causes an infinite browser redirect
|
||||
|
||||
Preconditions: candidate ${candidateSha}, ${options.profile}, ${options.viewport}, ${options.zoom}% zoom, ${options.input}, fresh named Chromium session ${options.session}.
|
||||
|
||||
1. Launch a fresh Chromium context with the Chromium argument --no-sandbox.
|
||||
2. Set the assigned viewport and open ${options.url} or its authentication entry.
|
||||
3. Observe repeated document requests to the same route.
|
||||
4. Wait until Chromium displays ERR_TOO_MANY_REDIRECTS or chrome-error://chromewebdata/.
|
||||
${retryStep}
|
||||
|
||||
Expected: the application renders the assigned entry, permitting authentication and the packet flow. Actual: the redirect loop prevents the application from rendering. Reproducibility: ${reproducibility}. Console output was empty; network and visual evidence are linked below.
|
||||
|
||||
## Artifacts
|
||||
|
||||
${artifactRows}
|
||||
|
||||
## Mutations and persistence
|
||||
|
||||
No credentials were submitted and no product mutation occurred. Reload and fresh-session retries reproduced the routing failure, so application persistence was not reachable.
|
||||
|
||||
## Blockers and gaps
|
||||
|
||||
Every non-evidence case listed as blocked remains untested beyond the routing entry. No blocked cell is treated as passed.
|
||||
|
||||
## Cleanup
|
||||
|
||||
The named browser session was closed by the browser agent. Artifacts remain under the untracked packet directory; no secret values were captured.
|
||||
`;
|
||||
const reportPath = path.join(qaRoot, "reports", packet.id, "report.md");
|
||||
await writeSafeReportFile(qaRoot, reportPath, report, `${packet.id} report path`);
|
||||
console.log(`Normalized blocked browser report ${packet.id} with ${artifacts.length} artifact(s)`);
|
||||
};
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,663 @@
|
||||
export type QaStatus = "passed" | "failed" | "blocked" | "not-reached";
|
||||
|
||||
export interface QaWidgetCheck {
|
||||
widgetKind: string;
|
||||
viewport: string;
|
||||
sizeRequirement: string;
|
||||
status: QaStatus;
|
||||
}
|
||||
|
||||
export type QaSeverity = "P0" | "P1" | "P2" | "P3";
|
||||
|
||||
export interface QaFinding {
|
||||
id?: string;
|
||||
fingerprint?: string;
|
||||
severity: QaSeverity;
|
||||
title: string;
|
||||
area?: string;
|
||||
summary?: string;
|
||||
caseIds?: string[];
|
||||
evidence?: string[];
|
||||
}
|
||||
|
||||
export interface QaIndependentReproduction {
|
||||
findingFingerprint: string;
|
||||
agentId: string;
|
||||
outcome: string;
|
||||
evidence: string[];
|
||||
notes: string;
|
||||
}
|
||||
|
||||
export interface AggregatedQaFinding extends QaFinding {
|
||||
packets: string[];
|
||||
caseIds: string[];
|
||||
evidence: string[];
|
||||
reproductions: QaIndependentReproduction[];
|
||||
}
|
||||
|
||||
export interface QaReportMetadata {
|
||||
packetId: string;
|
||||
status: QaStatus;
|
||||
caseStatuses: Record<string, QaStatus>;
|
||||
execution: {
|
||||
candidateSha: string | null;
|
||||
url: string | null;
|
||||
actualPort: number | null;
|
||||
runtimeProfile: string | null;
|
||||
runtimeFlags: string[];
|
||||
persona: string | null;
|
||||
sessionId: string | null;
|
||||
timestamp: string | null;
|
||||
viewport: string | null;
|
||||
input: string | null;
|
||||
zoom: number | null;
|
||||
};
|
||||
findings: QaFinding[];
|
||||
artifacts: string[];
|
||||
widgetChecks: QaWidgetCheck[];
|
||||
performance: {
|
||||
measurements: {
|
||||
name: string;
|
||||
value: number | null;
|
||||
unit: string;
|
||||
threshold: string;
|
||||
status: QaStatus;
|
||||
evidence: string[];
|
||||
}[];
|
||||
limitations: string[];
|
||||
};
|
||||
independentReproductions: QaIndependentReproduction[];
|
||||
notes: string;
|
||||
}
|
||||
|
||||
export interface CoverageGap {
|
||||
packetId: string;
|
||||
kind: "case" | "widget" | "metadata";
|
||||
itemId: string;
|
||||
status: QaStatus;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
interface NormalizeOptions {
|
||||
packetId: string;
|
||||
expectedCaseIds: string[];
|
||||
expectedWidgetChecks: QaWidgetCheck[];
|
||||
}
|
||||
|
||||
export interface ReportIntegrityOptions {
|
||||
campaignCandidateSha: string;
|
||||
packetId: string;
|
||||
profiles: string[];
|
||||
personas: string[];
|
||||
viewports: string[];
|
||||
zooms: number[];
|
||||
inputs: string[];
|
||||
expectedProfileFlags: Record<string, string[]>;
|
||||
}
|
||||
|
||||
const allowedStatuses = new Set<QaStatus>(["passed", "failed", "blocked", "not-reached"]);
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
const textOrNull = (value: unknown): string | null => (typeof value === "string" ? value : null);
|
||||
const numberOrNull = (value: unknown): number | null => (typeof value === "number" ? value : null);
|
||||
const strings = (value: unknown): string[] =>
|
||||
Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === "string") : [];
|
||||
|
||||
const statusOrNotReached = (value: unknown): QaStatus =>
|
||||
allowedStatuses.has(value as QaStatus) ? (value as QaStatus) : "not-reached";
|
||||
const severityOrP3 = (value: unknown): QaSeverity =>
|
||||
["P0", "P1", "P2", "P3"].includes(String(value)) ? (value as QaSeverity) : "P3";
|
||||
|
||||
const checkKey = (check: Pick<QaWidgetCheck, "widgetKind" | "viewport">): string =>
|
||||
`${check.widgetKind}\u0000${check.viewport}`;
|
||||
|
||||
const sameSet = <T,>(left: T[], right: T[]): boolean =>
|
||||
left.length === right.length && new Set(left).size === left.length && left.every((value) => right.includes(value));
|
||||
|
||||
const severityRank: Record<QaSeverity, number> = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
||||
|
||||
const uniqueStrings = (values: string[]): string[] => [...new Set(values)];
|
||||
|
||||
const reproductionKey = (reproduction: QaIndependentReproduction): string =>
|
||||
`${reproduction.findingFingerprint}\u0000${reproduction.agentId}\u0000${reproduction.outcome}`;
|
||||
|
||||
const mergeReproduction = (finding: AggregatedQaFinding, reproduction: QaIndependentReproduction): void => {
|
||||
const key = reproductionKey(reproduction);
|
||||
const existing = finding.reproductions.find((candidate) => reproductionKey(candidate) === key);
|
||||
if (!existing) {
|
||||
finding.reproductions.push({ ...reproduction, evidence: uniqueStrings(reproduction.evidence) });
|
||||
return;
|
||||
}
|
||||
existing.evidence = uniqueStrings([...existing.evidence, ...reproduction.evidence]);
|
||||
if (reproduction.notes && !existing.notes.includes(reproduction.notes)) {
|
||||
existing.notes = [existing.notes, reproduction.notes].filter(Boolean).join("; ");
|
||||
}
|
||||
};
|
||||
|
||||
export const aggregateQaFindings = (
|
||||
reports: {
|
||||
packetId: string;
|
||||
findings: QaFinding[];
|
||||
independentReproductions: QaIndependentReproduction[];
|
||||
}[],
|
||||
): AggregatedQaFinding[] => {
|
||||
const findings = new Map<string, AggregatedQaFinding>();
|
||||
for (const report of reports) {
|
||||
for (const finding of report.findings) {
|
||||
const key =
|
||||
finding.fingerprint ?? `${finding.severity}:${finding.area ?? "general"}:${finding.title}`.toLowerCase();
|
||||
const existing = findings.get(key);
|
||||
if (!existing) {
|
||||
findings.set(key, {
|
||||
...finding,
|
||||
packets: [report.packetId],
|
||||
caseIds: uniqueStrings(finding.caseIds ?? []),
|
||||
evidence: uniqueStrings(finding.evidence ?? []),
|
||||
reproductions: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
existing.packets = uniqueStrings([...existing.packets, report.packetId]);
|
||||
existing.caseIds = uniqueStrings([...existing.caseIds, ...(finding.caseIds ?? [])]);
|
||||
existing.evidence = uniqueStrings([...existing.evidence, ...(finding.evidence ?? [])]);
|
||||
if (severityRank[finding.severity] < severityRank[existing.severity]) {
|
||||
existing.severity = finding.severity;
|
||||
existing.id = finding.id ?? existing.id;
|
||||
existing.title = finding.title;
|
||||
existing.area = finding.area ?? existing.area;
|
||||
existing.summary = finding.summary ?? existing.summary;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const report of reports) {
|
||||
for (const reproduction of report.independentReproductions) {
|
||||
const finding = findings.get(reproduction.findingFingerprint);
|
||||
if (finding) mergeReproduction(finding, reproduction);
|
||||
}
|
||||
}
|
||||
|
||||
return [...findings.values()].toSorted(
|
||||
(left, right) =>
|
||||
severityRank[left.severity] - severityRank[right.severity] || left.title.localeCompare(right.title),
|
||||
);
|
||||
};
|
||||
|
||||
const markdownTableCells = (line: string): string[] | null => {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith("|") || !trimmed.endsWith("|")) return null;
|
||||
return trimmed
|
||||
.slice(1, -1)
|
||||
.split(/(?<!\\)\|/u)
|
||||
.map((cell) => cell.trim().replaceAll("\\|", "|"));
|
||||
};
|
||||
|
||||
export const collectHumanWidgetStatusErrors = (
|
||||
content: string,
|
||||
packetId: string,
|
||||
expectedChecks: QaWidgetCheck[],
|
||||
): string[] => {
|
||||
const heading = "## Widget kind × viewport checklist";
|
||||
const sectionStart = content.indexOf(heading);
|
||||
if (expectedChecks.length === 0) {
|
||||
if (sectionStart >= 0) return [`${packetId}: non-widget report must not contain a human widget checklist`];
|
||||
return [];
|
||||
}
|
||||
if (sectionStart < 0) return [`${packetId}: human widget checklist is missing`];
|
||||
|
||||
const sectionBodyStart = sectionStart + heading.length;
|
||||
const duplicateSection = content.indexOf(heading, sectionBodyStart);
|
||||
const nextSection = content.indexOf("\n## ", sectionBodyStart);
|
||||
const section = content.slice(sectionBodyStart, nextSection < 0 ? undefined : nextSection);
|
||||
const rows = new Map<string, { status: string; count: number }>();
|
||||
for (const line of section.split("\n")) {
|
||||
const cells = markdownTableCells(line);
|
||||
if (!cells || cells.length !== 9 || cells[0] === "Widget kind" || cells.every((cell) => /^-+$/u.test(cell))) {
|
||||
continue;
|
||||
}
|
||||
const key = `${cells[0]}\u0000${cells[1]}`;
|
||||
const existing = rows.get(key);
|
||||
rows.set(key, { status: cells[8] ?? "", count: (existing?.count ?? 0) + 1 });
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
if (duplicateSection >= 0) errors.push(`${packetId}: human widget checklist is duplicated`);
|
||||
const expectedKeys = new Set(expectedChecks.map(checkKey));
|
||||
for (const check of expectedChecks) {
|
||||
const key = checkKey(check);
|
||||
const row = rows.get(key);
|
||||
if (!row) {
|
||||
errors.push(`${packetId}: human widget row is missing ${check.widgetKind} at ${check.viewport}`);
|
||||
continue;
|
||||
}
|
||||
if (row.count !== 1) {
|
||||
errors.push(`${packetId}: human widget row is duplicated ${check.widgetKind} at ${check.viewport}`);
|
||||
}
|
||||
if (row.status !== check.status) {
|
||||
errors.push(
|
||||
`${packetId}/${check.widgetKind}/${check.viewport}: human widget status ${row.status || "<empty>"} differs from structured status ${check.status}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const key of rows.keys()) {
|
||||
if (expectedKeys.has(key)) continue;
|
||||
const [widgetKind, viewport] = key.split("\u0000");
|
||||
errors.push(`${packetId}: human widget checklist contains unassigned row ${widgetKind}@${viewport}`);
|
||||
}
|
||||
return errors;
|
||||
};
|
||||
|
||||
export const rollupQaStatuses = (statuses: QaStatus[]): QaStatus => {
|
||||
if (statuses.includes("failed")) return "failed";
|
||||
if (statuses.includes("blocked")) return "blocked";
|
||||
if (statuses.includes("not-reached")) return "not-reached";
|
||||
return "passed";
|
||||
};
|
||||
|
||||
const hasExecutionResults = (metadata: QaReportMetadata): boolean =>
|
||||
metadata.status !== "not-reached" ||
|
||||
Object.values(metadata.caseStatuses).some((status) => status !== "not-reached") ||
|
||||
metadata.widgetChecks.some((check) => check.status !== "not-reached") ||
|
||||
metadata.findings.length > 0 ||
|
||||
metadata.artifacts.length > 0 ||
|
||||
metadata.performance.measurements.length > 0 ||
|
||||
metadata.performance.limitations.length > 0 ||
|
||||
metadata.independentReproductions.length > 0;
|
||||
|
||||
export const collectReportIntegrityErrors = (metadata: QaReportMetadata, options: ReportIntegrityOptions): string[] => {
|
||||
const errors: string[] = [];
|
||||
if (metadata.packetId !== options.packetId) {
|
||||
errors.push(`${options.packetId}: metadata packetId is ${metadata.packetId}`);
|
||||
}
|
||||
|
||||
const caseRollup = rollupQaStatuses(Object.values(metadata.caseStatuses));
|
||||
if (metadata.status !== caseRollup) {
|
||||
errors.push(`${options.packetId}: report status ${metadata.status} does not match case rollup ${caseRollup}`);
|
||||
}
|
||||
|
||||
if (metadata.execution.candidateSha !== options.campaignCandidateSha) {
|
||||
errors.push(`${options.packetId}: candidateSha must equal the campaign candidate ${options.campaignCandidateSha}`);
|
||||
}
|
||||
|
||||
if (!hasExecutionResults(metadata)) return errors;
|
||||
|
||||
const execution = metadata.execution;
|
||||
const requiredText = {
|
||||
candidateSha: execution.candidateSha,
|
||||
url: execution.url,
|
||||
runtimeProfile: execution.runtimeProfile,
|
||||
persona: execution.persona,
|
||||
sessionId: execution.sessionId,
|
||||
timestamp: execution.timestamp,
|
||||
viewport: execution.viewport,
|
||||
input: execution.input,
|
||||
};
|
||||
for (const [field, value] of Object.entries(requiredText)) {
|
||||
if (typeof value !== "string" || value.trim().length === 0) {
|
||||
errors.push(`${options.packetId}: execution.${field} is required after execution`);
|
||||
}
|
||||
}
|
||||
if (!Number.isInteger(execution.actualPort) || (execution.actualPort ?? 0) <= 0) {
|
||||
errors.push(`${options.packetId}: execution.actualPort must be a positive integer after execution`);
|
||||
}
|
||||
if (!Number.isFinite(execution.zoom) || (execution.zoom ?? 0) <= 0) {
|
||||
errors.push(`${options.packetId}: execution.zoom must be positive after execution`);
|
||||
}
|
||||
if (metadata.artifacts.length === 0) {
|
||||
errors.push(`${options.packetId}: executed report must link at least one artifact`);
|
||||
}
|
||||
|
||||
if (execution.runtimeProfile && !options.profiles.includes(execution.runtimeProfile)) {
|
||||
errors.push(`${options.packetId}: execution.runtimeProfile is outside the packet assignment`);
|
||||
}
|
||||
if (execution.persona && !options.personas.includes(execution.persona)) {
|
||||
errors.push(`${options.packetId}: execution.persona is outside the packet assignment`);
|
||||
}
|
||||
if (execution.viewport && !options.viewports.includes(execution.viewport)) {
|
||||
errors.push(`${options.packetId}: execution.viewport is outside the packet assignment`);
|
||||
}
|
||||
if (execution.input && !options.inputs.includes(execution.input)) {
|
||||
errors.push(`${options.packetId}: execution.input is outside the packet assignment`);
|
||||
}
|
||||
if (execution.zoom && !options.zooms.includes(execution.zoom)) {
|
||||
errors.push(`${options.packetId}: execution.zoom is outside the packet assignment`);
|
||||
}
|
||||
|
||||
const expectedFlags = execution.runtimeProfile ? options.expectedProfileFlags[execution.runtimeProfile] : undefined;
|
||||
if (!expectedFlags || !sameSet(execution.runtimeFlags, expectedFlags)) {
|
||||
errors.push(
|
||||
`${options.packetId}: execution.runtimeFlags do not match profile ${execution.runtimeProfile ?? "unknown"}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (execution.timestamp && Number.isNaN(Date.parse(execution.timestamp))) {
|
||||
errors.push(`${options.packetId}: execution.timestamp must be ISO-compatible`);
|
||||
}
|
||||
|
||||
if (execution.url) {
|
||||
try {
|
||||
const url = new URL(execution.url);
|
||||
if (!["http:", "https:"].includes(url.protocol)) {
|
||||
errors.push(`${options.packetId}: execution.url must use HTTP(S)`);
|
||||
}
|
||||
const defaultPort = url.protocol === "https:" ? 443 : 80;
|
||||
const urlPort = url.port.length > 0 ? Number(url.port) : defaultPort;
|
||||
if (execution.actualPort !== null && urlPort !== execution.actualPort) {
|
||||
errors.push(`${options.packetId}: execution.url port does not match execution.actualPort`);
|
||||
}
|
||||
} catch {
|
||||
errors.push(`${options.packetId}: execution.url must be an absolute HTTP(S) URL`);
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
};
|
||||
|
||||
export const normalizeReportMetadata = (
|
||||
value: unknown,
|
||||
options: NormalizeOptions,
|
||||
): { metadata: QaReportMetadata; errors: string[] } => {
|
||||
const errors: string[] = [];
|
||||
const root = isRecord(value) ? value : {};
|
||||
if (!isRecord(value)) errors.push(`${options.packetId}: report metadata must be an object`);
|
||||
if (typeof root.packetId !== "string") errors.push(`${options.packetId}: packetId must be a string`);
|
||||
|
||||
const caseStatuses: Record<string, QaStatus> = {};
|
||||
if (!isRecord(root.caseStatuses)) {
|
||||
errors.push(`${options.packetId}: caseStatuses must be an object`);
|
||||
}
|
||||
const rawCaseStatuses = isRecord(root.caseStatuses) ? root.caseStatuses : {};
|
||||
for (const caseId of options.expectedCaseIds) {
|
||||
const rawStatus = rawCaseStatuses[caseId];
|
||||
if (rawStatus === undefined) {
|
||||
errors.push(`${options.packetId}: caseStatuses is missing assigned case ${caseId}`);
|
||||
} else if (!allowedStatuses.has(rawStatus as QaStatus)) {
|
||||
errors.push(`${options.packetId}/${caseId}: invalid case status ${String(rawStatus)}`);
|
||||
}
|
||||
caseStatuses[caseId] = statusOrNotReached(rawStatus);
|
||||
}
|
||||
for (const caseId of Object.keys(rawCaseStatuses)) {
|
||||
if (!options.expectedCaseIds.includes(caseId)) {
|
||||
errors.push(`${options.packetId}: caseStatuses contains unknown case ${caseId}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!Array.isArray(root.widgetChecks)) {
|
||||
errors.push(`${options.packetId}: widgetChecks must be an array`);
|
||||
}
|
||||
const expectedChecks = new Map(options.expectedWidgetChecks.map((check) => [checkKey(check), check]));
|
||||
const suppliedChecks = new Map<string, QaWidgetCheck>();
|
||||
for (const [index, valueAtIndex] of (Array.isArray(root.widgetChecks) ? root.widgetChecks : []).entries()) {
|
||||
if (!isRecord(valueAtIndex)) {
|
||||
errors.push(`${options.packetId}: widgetChecks[${index}] must be an object`);
|
||||
continue;
|
||||
}
|
||||
const widgetKind = typeof valueAtIndex.widgetKind === "string" ? valueAtIndex.widgetKind : "";
|
||||
const viewport = typeof valueAtIndex.viewport === "string" ? valueAtIndex.viewport : "";
|
||||
const key = checkKey({ widgetKind, viewport });
|
||||
if (!expectedChecks.has(key)) {
|
||||
errors.push(`${options.packetId}: widgetChecks contains unknown row ${widgetKind}@${viewport}`);
|
||||
continue;
|
||||
}
|
||||
if (suppliedChecks.has(key)) {
|
||||
errors.push(`${options.packetId}: widgetChecks contains duplicate row ${widgetKind}@${viewport}`);
|
||||
continue;
|
||||
}
|
||||
if (!allowedStatuses.has(valueAtIndex.status as QaStatus)) {
|
||||
errors.push(`${options.packetId}/${widgetKind}/${viewport}: invalid widget check status`);
|
||||
}
|
||||
if (typeof valueAtIndex.sizeRequirement !== "string") {
|
||||
errors.push(`${options.packetId}/${widgetKind}/${viewport}: sizeRequirement must be a string`);
|
||||
} else if (valueAtIndex.sizeRequirement !== expectedChecks.get(key)?.sizeRequirement) {
|
||||
errors.push(`${options.packetId}/${widgetKind}/${viewport}: sizeRequirement does not match the assignment`);
|
||||
}
|
||||
suppliedChecks.set(key, {
|
||||
widgetKind,
|
||||
viewport,
|
||||
sizeRequirement: typeof valueAtIndex.sizeRequirement === "string" ? valueAtIndex.sizeRequirement : "",
|
||||
status: statusOrNotReached(valueAtIndex.status),
|
||||
});
|
||||
}
|
||||
const widgetChecks = options.expectedWidgetChecks.map((expected) => {
|
||||
const supplied = suppliedChecks.get(checkKey(expected));
|
||||
if (supplied) return supplied;
|
||||
errors.push(
|
||||
`${options.packetId}: widgetChecks is missing assigned row ${expected.widgetKind}@${expected.viewport}`,
|
||||
);
|
||||
return { ...expected, status: "not-reached" as const };
|
||||
});
|
||||
|
||||
if (!Array.isArray(root.findings)) errors.push(`${options.packetId}: findings must be an array`);
|
||||
const findings = (Array.isArray(root.findings) ? root.findings : []).flatMap((finding, index) => {
|
||||
if (!isRecord(finding)) {
|
||||
errors.push(`${options.packetId}: findings[${index}] must be an object`);
|
||||
return [];
|
||||
}
|
||||
if (!["P0", "P1", "P2", "P3"].includes(String(finding.severity))) {
|
||||
errors.push(`${options.packetId}: findings[${index}] has an invalid severity`);
|
||||
}
|
||||
if (typeof finding.title !== "string") {
|
||||
errors.push(`${options.packetId}: findings[${index}].title must be a string`);
|
||||
}
|
||||
if (finding.caseIds !== undefined && !Array.isArray(finding.caseIds)) {
|
||||
errors.push(`${options.packetId}: findings[${index}].caseIds must be an array`);
|
||||
} else if (Array.isArray(finding.caseIds) && finding.caseIds.some((caseId) => typeof caseId !== "string")) {
|
||||
errors.push(`${options.packetId}: findings[${index}].caseIds must contain only strings`);
|
||||
} else if (
|
||||
Array.isArray(finding.caseIds) &&
|
||||
finding.caseIds.some((caseId) => !options.expectedCaseIds.includes(String(caseId)))
|
||||
) {
|
||||
errors.push(`${options.packetId}: findings[${index}].caseIds contains an unknown assigned case`);
|
||||
}
|
||||
if (finding.evidence !== undefined && !Array.isArray(finding.evidence)) {
|
||||
errors.push(`${options.packetId}: findings[${index}].evidence must be an array`);
|
||||
} else if (Array.isArray(finding.evidence) && finding.evidence.some((entry) => typeof entry !== "string")) {
|
||||
errors.push(`${options.packetId}: findings[${index}].evidence must contain only strings`);
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: typeof finding.id === "string" ? finding.id : undefined,
|
||||
fingerprint: typeof finding.fingerprint === "string" ? finding.fingerprint : undefined,
|
||||
severity: severityOrP3(finding.severity),
|
||||
title: typeof finding.title === "string" ? finding.title : "",
|
||||
area: typeof finding.area === "string" ? finding.area : undefined,
|
||||
summary: typeof finding.summary === "string" ? finding.summary : undefined,
|
||||
caseIds: Array.isArray(finding.caseIds) ? strings(finding.caseIds) : undefined,
|
||||
evidence: Array.isArray(finding.evidence) ? strings(finding.evidence) : undefined,
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
if (!Array.isArray(root.artifacts)) errors.push(`${options.packetId}: artifacts must be an array`);
|
||||
if (Array.isArray(root.artifacts) && root.artifacts.some((artifact) => typeof artifact !== "string")) {
|
||||
errors.push(`${options.packetId}: artifacts must contain only strings`);
|
||||
}
|
||||
const artifacts = strings(root.artifacts);
|
||||
|
||||
const execution = isRecord(root.execution) ? root.execution : {};
|
||||
if (!isRecord(root.execution)) errors.push(`${options.packetId}: execution metadata must be an object`);
|
||||
if (!Array.isArray(execution.runtimeFlags)) {
|
||||
errors.push(`${options.packetId}: execution.runtimeFlags must be an array`);
|
||||
} else if (execution.runtimeFlags.some((flag) => typeof flag !== "string")) {
|
||||
errors.push(`${options.packetId}: execution.runtimeFlags must contain only strings`);
|
||||
}
|
||||
for (const field of [
|
||||
"candidateSha",
|
||||
"url",
|
||||
"runtimeProfile",
|
||||
"persona",
|
||||
"sessionId",
|
||||
"timestamp",
|
||||
"viewport",
|
||||
"input",
|
||||
]) {
|
||||
const fieldValue = execution[field];
|
||||
if (fieldValue !== undefined && fieldValue !== null && typeof fieldValue !== "string") {
|
||||
errors.push(`${options.packetId}: execution.${field} must be a string or null`);
|
||||
}
|
||||
}
|
||||
for (const field of ["actualPort", "zoom"]) {
|
||||
const fieldValue = execution[field];
|
||||
if (fieldValue !== undefined && fieldValue !== null && typeof fieldValue !== "number") {
|
||||
errors.push(`${options.packetId}: execution.${field} must be a number or null`);
|
||||
}
|
||||
}
|
||||
|
||||
const performance = isRecord(root.performance) ? root.performance : {};
|
||||
if (!isRecord(root.performance)) errors.push(`${options.packetId}: performance must be an object`);
|
||||
if (!Array.isArray(performance.measurements)) {
|
||||
errors.push(`${options.packetId}: performance.measurements must be an array`);
|
||||
}
|
||||
if (!Array.isArray(performance.limitations)) {
|
||||
errors.push(`${options.packetId}: performance.limitations must be an array`);
|
||||
} else if (performance.limitations.some((limitation) => typeof limitation !== "string")) {
|
||||
errors.push(`${options.packetId}: performance.limitations must contain only strings`);
|
||||
}
|
||||
const measurements = (Array.isArray(performance.measurements) ? performance.measurements : []).flatMap(
|
||||
(measurement, index) => {
|
||||
if (!isRecord(measurement)) {
|
||||
errors.push(`${options.packetId}: performance.measurements[${index}] must be an object`);
|
||||
return [];
|
||||
}
|
||||
for (const field of ["name", "unit", "threshold"]) {
|
||||
if (typeof measurement[field] !== "string") {
|
||||
errors.push(`${options.packetId}: performance.measurements[${index}].${field} must be a string`);
|
||||
}
|
||||
}
|
||||
if (measurement.value !== null && typeof measurement.value !== "number") {
|
||||
errors.push(`${options.packetId}: performance.measurements[${index}].value must be a number or null`);
|
||||
}
|
||||
if (!allowedStatuses.has(measurement.status as QaStatus)) {
|
||||
errors.push(`${options.packetId}: performance.measurements[${index}].status is invalid`);
|
||||
}
|
||||
if (!Array.isArray(measurement.evidence)) {
|
||||
errors.push(`${options.packetId}: performance.measurements[${index}].evidence must be an array`);
|
||||
} else if (measurement.evidence.some((entry) => typeof entry !== "string")) {
|
||||
errors.push(`${options.packetId}: performance.measurements[${index}].evidence must contain only strings`);
|
||||
}
|
||||
return [
|
||||
{
|
||||
name: typeof measurement.name === "string" ? measurement.name : "",
|
||||
value: numberOrNull(measurement.value),
|
||||
unit: typeof measurement.unit === "string" ? measurement.unit : "",
|
||||
threshold: typeof measurement.threshold === "string" ? measurement.threshold : "",
|
||||
status: statusOrNotReached(measurement.status),
|
||||
evidence: strings(measurement.evidence),
|
||||
},
|
||||
];
|
||||
},
|
||||
);
|
||||
|
||||
if (!Array.isArray(root.independentReproductions)) {
|
||||
errors.push(`${options.packetId}: independentReproductions must be an array`);
|
||||
}
|
||||
const independentReproductions = (
|
||||
Array.isArray(root.independentReproductions) ? root.independentReproductions : []
|
||||
).flatMap((reproduction, index) => {
|
||||
if (!isRecord(reproduction)) {
|
||||
errors.push(`${options.packetId}: independentReproductions[${index}] must be an object`);
|
||||
return [];
|
||||
}
|
||||
for (const field of ["findingFingerprint", "agentId", "notes"]) {
|
||||
if (typeof reproduction[field] !== "string") {
|
||||
errors.push(`${options.packetId}: independentReproductions[${index}].${field} must be a string`);
|
||||
}
|
||||
}
|
||||
if (!["reproduced", "not-reproduced", "blocked", "not-reached"].includes(String(reproduction.outcome))) {
|
||||
errors.push(`${options.packetId}: independentReproductions[${index}].outcome is invalid`);
|
||||
}
|
||||
if (!Array.isArray(reproduction.evidence)) {
|
||||
errors.push(`${options.packetId}: independentReproductions[${index}].evidence must be an array`);
|
||||
} else if (reproduction.evidence.some((entry) => typeof entry !== "string")) {
|
||||
errors.push(`${options.packetId}: independentReproductions[${index}].evidence must contain only strings`);
|
||||
}
|
||||
return [
|
||||
{
|
||||
findingFingerprint: typeof reproduction.findingFingerprint === "string" ? reproduction.findingFingerprint : "",
|
||||
agentId: typeof reproduction.agentId === "string" ? reproduction.agentId : "",
|
||||
outcome: typeof reproduction.outcome === "string" ? reproduction.outcome : "not-reached",
|
||||
evidence: strings(reproduction.evidence),
|
||||
notes: typeof reproduction.notes === "string" ? reproduction.notes : "",
|
||||
},
|
||||
];
|
||||
});
|
||||
|
||||
if (!allowedStatuses.has(root.status as QaStatus)) {
|
||||
errors.push(`${options.packetId}: invalid report status ${String(root.status)}`);
|
||||
}
|
||||
if (typeof root.notes !== "string") errors.push(`${options.packetId}: notes must be a string`);
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
packetId: typeof root.packetId === "string" ? root.packetId : options.packetId,
|
||||
status: statusOrNotReached(root.status),
|
||||
caseStatuses,
|
||||
execution: {
|
||||
candidateSha: textOrNull(execution.candidateSha),
|
||||
url: textOrNull(execution.url),
|
||||
actualPort: numberOrNull(execution.actualPort),
|
||||
runtimeProfile: textOrNull(execution.runtimeProfile),
|
||||
runtimeFlags: strings(execution.runtimeFlags),
|
||||
persona: textOrNull(execution.persona),
|
||||
sessionId: textOrNull(execution.sessionId),
|
||||
timestamp: textOrNull(execution.timestamp),
|
||||
viewport: textOrNull(execution.viewport),
|
||||
input: textOrNull(execution.input),
|
||||
zoom: numberOrNull(execution.zoom),
|
||||
},
|
||||
findings,
|
||||
artifacts,
|
||||
widgetChecks,
|
||||
performance: {
|
||||
measurements,
|
||||
limitations: strings(performance.limitations),
|
||||
},
|
||||
independentReproductions,
|
||||
notes: typeof root.notes === "string" ? root.notes : "",
|
||||
},
|
||||
errors,
|
||||
};
|
||||
};
|
||||
|
||||
export const collectCriticalCoverageGaps = (
|
||||
reports: {
|
||||
packetId: string;
|
||||
caseStatuses: Record<string, QaStatus>;
|
||||
widgetChecks: QaWidgetCheck[];
|
||||
metadataErrors?: string[];
|
||||
}[],
|
||||
): CoverageGap[] =>
|
||||
reports.flatMap((report) => {
|
||||
const caseGaps = Object.entries(report.caseStatuses)
|
||||
.filter(([, status]) => status !== "passed")
|
||||
.map(([caseId, status]) => ({
|
||||
packetId: report.packetId,
|
||||
kind: "case" as const,
|
||||
itemId: caseId,
|
||||
status,
|
||||
}));
|
||||
const widgetGaps = report.widgetChecks
|
||||
.filter((check) => check.status !== "passed")
|
||||
.map((check) => ({
|
||||
packetId: report.packetId,
|
||||
kind: "widget" as const,
|
||||
itemId: `${check.widgetKind}@${check.viewport}`,
|
||||
status: check.status,
|
||||
}));
|
||||
const metadataGaps = (report.metadataErrors ?? []).map((detail, index) => ({
|
||||
packetId: report.packetId,
|
||||
kind: "metadata" as const,
|
||||
itemId: `metadata-${index + 1}`,
|
||||
status: "blocked" as const,
|
||||
detail,
|
||||
}));
|
||||
return [...caseGaps, ...widgetGaps, ...metadataGaps];
|
||||
});
|
||||
|
||||
export const releaseDecision = (severityTotals: { P0: number; P1: number }, gaps: CoverageGap[]): "GO" | "NO-GO" => {
|
||||
if (severityTotals.P0 > 0 || severityTotals.P1 > 0 || gaps.length > 0) return "NO-GO";
|
||||
return "GO";
|
||||
};
|
||||
@@ -0,0 +1,281 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
aggregateQaFindings,
|
||||
collectCriticalCoverageGaps,
|
||||
collectHumanWidgetStatusErrors,
|
||||
collectReportIntegrityErrors,
|
||||
normalizeReportMetadata,
|
||||
releaseDecision,
|
||||
} from "./report-integrity.mts";
|
||||
import type { QaReportMetadata, QaWidgetCheck, ReportIntegrityOptions } from "./report-integrity.mts";
|
||||
|
||||
const assignedWidgetCheck: QaWidgetCheck = {
|
||||
widgetKind: "weather",
|
||||
viewport: "320x568",
|
||||
sizeRequirement: "minimum / canonical / wide / tall / maximum / overflow / behavior-changing thresholds",
|
||||
status: "not-reached",
|
||||
};
|
||||
const campaignCandidateSha = "f57a660088d6777c86aca22977354fa8b810e2be";
|
||||
const integrityOptions: ReportIntegrityOptions = {
|
||||
campaignCandidateSha,
|
||||
packetId: "preflight-01",
|
||||
profiles: ["main-writable"],
|
||||
personas: ["avery-admin"],
|
||||
viewports: ["1280x900"],
|
||||
zooms: [100],
|
||||
inputs: ["mouse"],
|
||||
expectedProfileFlags: {
|
||||
"main-writable": ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
},
|
||||
};
|
||||
const validMetadata = (): QaReportMetadata => ({
|
||||
packetId: "preflight-01",
|
||||
status: "passed",
|
||||
caseStatuses: { "PF01-FIXTURE-STATE": "passed" },
|
||||
execution: {
|
||||
candidateSha: campaignCandidateSha,
|
||||
url: "http://127.0.0.1:34401",
|
||||
actualPort: 34401,
|
||||
runtimeProfile: "main-writable",
|
||||
runtimeFlags: ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
persona: "avery-admin",
|
||||
sessionId: "qa-v2-main-avery-admin",
|
||||
timestamp: "2026-09-01T00:00:00.000Z",
|
||||
viewport: "1280x900",
|
||||
input: "mouse",
|
||||
zoom: 100,
|
||||
},
|
||||
findings: [],
|
||||
artifacts: ["/tmp/qa-artifact.png"],
|
||||
widgetChecks: [],
|
||||
performance: { measurements: [], limitations: [] },
|
||||
independentReproductions: [],
|
||||
notes: "Complete.",
|
||||
});
|
||||
|
||||
describe("release-v2 report integrity", () => {
|
||||
it("keeps the highest severity while merging duplicate finding evidence and reproductions", () => {
|
||||
const fingerprint = "same-defect";
|
||||
const findings = aggregateQaFindings([
|
||||
{
|
||||
packetId: "widgets-01",
|
||||
findings: [
|
||||
{
|
||||
fingerprint,
|
||||
severity: "P3",
|
||||
title: "Initial low-severity title",
|
||||
caseIds: ["W01-RENDER-STATE"],
|
||||
evidence: ["/tmp/first.png"],
|
||||
},
|
||||
],
|
||||
independentReproductions: [
|
||||
{
|
||||
findingFingerprint: fingerprint,
|
||||
agentId: "reproducer-01",
|
||||
outcome: "reproduced",
|
||||
evidence: ["/tmp/repro-first.png"],
|
||||
notes: "First replay.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
packetId: "widgets-02",
|
||||
findings: [
|
||||
{
|
||||
fingerprint,
|
||||
severity: "P1",
|
||||
title: "Confirmed release blocker",
|
||||
caseIds: ["W02-RENDER-STATE"],
|
||||
evidence: ["/tmp/first.png", "/tmp/second.png"],
|
||||
},
|
||||
],
|
||||
independentReproductions: [],
|
||||
},
|
||||
{
|
||||
packetId: "repro-widgets-02",
|
||||
findings: [],
|
||||
independentReproductions: [
|
||||
{
|
||||
findingFingerprint: fingerprint,
|
||||
agentId: "reproducer-01",
|
||||
outcome: "reproduced",
|
||||
evidence: ["/tmp/repro-second.png"],
|
||||
notes: "Second replay.",
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(findings).toHaveLength(1);
|
||||
expect(findings[0]).toMatchObject({
|
||||
severity: "P1",
|
||||
title: "Confirmed release blocker",
|
||||
packets: ["widgets-01", "widgets-02"],
|
||||
caseIds: ["W01-RENDER-STATE", "W02-RENDER-STATE"],
|
||||
evidence: ["/tmp/first.png", "/tmp/second.png"],
|
||||
});
|
||||
expect(findings[0]?.reproductions).toEqual([
|
||||
{
|
||||
findingFingerprint: fingerprint,
|
||||
agentId: "reproducer-01",
|
||||
outcome: "reproduced",
|
||||
evidence: ["/tmp/repro-first.png", "/tmp/repro-second.png"],
|
||||
notes: "First replay.; Second replay.",
|
||||
},
|
||||
]);
|
||||
const severityTotals = { P0: 0, P1: findings.filter((finding) => finding.severity === "P1").length };
|
||||
expect(releaseDecision(severityTotals, [])).toBe("NO-GO");
|
||||
});
|
||||
|
||||
it("requires each human widget status to exactly match the structured status", () => {
|
||||
const passedCheck = { ...assignedWidgetCheck, status: "passed" as const };
|
||||
const content = `## Widget kind × viewport checklist
|
||||
|
||||
| Widget kind | Viewport | Size | States | Permission/read-only | Options/persistence | Recovery | Evidence | Status |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| weather | 320x568 | ${assignedWidgetCheck.sizeRequirement} | healthy | owner | reload | recovered | evidence | not-reached |
|
||||
|
||||
## Findings`;
|
||||
|
||||
expect(collectHumanWidgetStatusErrors(content, "widgets-01", [passedCheck])).toEqual([
|
||||
"widgets-01/weather/320x568: human widget status not-reached differs from structured status passed",
|
||||
]);
|
||||
expect(
|
||||
collectHumanWidgetStatusErrors(content.replace("| not-reached |", "| passed |"), "widgets-01", [passedCheck]),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the release NO-GO when cases pass but an assigned widget check is not reached", () => {
|
||||
const gaps = collectCriticalCoverageGaps([
|
||||
{
|
||||
packetId: "widgets-01",
|
||||
caseStatuses: { "W01-RENDER-STATE": "passed" },
|
||||
widgetChecks: [assignedWidgetCheck],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(gaps).toEqual([
|
||||
{
|
||||
packetId: "widgets-01",
|
||||
kind: "widget",
|
||||
itemId: "weather@320x568",
|
||||
status: "not-reached",
|
||||
},
|
||||
]);
|
||||
expect(releaseDecision({ P0: 0, P1: 0 }, gaps)).toBe("NO-GO");
|
||||
});
|
||||
|
||||
it("reports null and malformed metadata without throwing", () => {
|
||||
const normalize = () =>
|
||||
normalizeReportMetadata(
|
||||
{
|
||||
packetId: "widgets-01",
|
||||
status: "passed",
|
||||
caseStatuses: null,
|
||||
execution: { candidateSha: {}, runtimeFlags: null },
|
||||
findings: null,
|
||||
artifacts: null,
|
||||
widgetChecks: null,
|
||||
performance: null,
|
||||
independentReproductions: null,
|
||||
},
|
||||
{
|
||||
packetId: "widgets-01",
|
||||
expectedCaseIds: ["W01-RENDER-STATE"],
|
||||
expectedWidgetChecks: [assignedWidgetCheck],
|
||||
},
|
||||
);
|
||||
|
||||
expect(normalize).not.toThrow();
|
||||
const result = normalize();
|
||||
expect(result.errors).toEqual(
|
||||
expect.arrayContaining([
|
||||
"widgets-01: caseStatuses must be an object",
|
||||
"widgets-01: findings must be an array",
|
||||
"widgets-01: widgetChecks must be an array",
|
||||
"widgets-01: execution.runtimeFlags must be an array",
|
||||
"widgets-01: execution.candidateSha must be a string or null",
|
||||
]),
|
||||
);
|
||||
expect(result.metadata.caseStatuses).toEqual({ "W01-RENDER-STATE": "not-reached" });
|
||||
expect(result.metadata.findings).toEqual([]);
|
||||
expect(result.metadata.widgetChecks).toEqual([assignedWidgetCheck]);
|
||||
|
||||
const gaps = collectCriticalCoverageGaps([
|
||||
{
|
||||
packetId: "widgets-01",
|
||||
caseStatuses: result.metadata.caseStatuses,
|
||||
widgetChecks: result.metadata.widgetChecks,
|
||||
metadataErrors: result.errors,
|
||||
},
|
||||
]);
|
||||
expect(gaps.some((gap) => gap.kind === "metadata")).toBe(true);
|
||||
expect(releaseDecision({ P0: 0, P1: 0 }, gaps)).toBe("NO-GO");
|
||||
});
|
||||
|
||||
it("turns a null metadata root into deterministic validation errors", () => {
|
||||
const result = normalizeReportMetadata(null, {
|
||||
packetId: "widgets-01",
|
||||
expectedCaseIds: ["W01-RENDER-STATE"],
|
||||
expectedWidgetChecks: [assignedWidgetCheck],
|
||||
});
|
||||
|
||||
expect(result.errors[0]).toBe("widgets-01: report metadata must be an object");
|
||||
expect(result.metadata.caseStatuses["W01-RENDER-STATE"]).toBe("not-reached");
|
||||
expect(result.metadata.widgetChecks[0]?.status).toBe("not-reached");
|
||||
});
|
||||
|
||||
it("forces a metadata gap and NO-GO for an executed report from the wrong candidate", () => {
|
||||
const metadata = validMetadata();
|
||||
metadata.execution.candidateSha = "0000000000000000000000000000000000000000";
|
||||
const metadataErrors = collectReportIntegrityErrors(metadata, integrityOptions);
|
||||
const gaps = collectCriticalCoverageGaps([
|
||||
{ packetId: metadata.packetId, caseStatuses: metadata.caseStatuses, widgetChecks: [], metadataErrors },
|
||||
]);
|
||||
|
||||
expect(metadataErrors).toContain(
|
||||
`preflight-01: candidateSha must equal the campaign candidate ${campaignCandidateSha}`,
|
||||
);
|
||||
expect(gaps.some((gap) => gap.kind === "metadata")).toBe(true);
|
||||
expect(releaseDecision({ P0: 0, P1: 0 }, gaps)).toBe("NO-GO");
|
||||
});
|
||||
|
||||
it("forces a metadata gap and NO-GO when packet status disagrees with passed cases", () => {
|
||||
const metadata = validMetadata();
|
||||
metadata.status = "failed";
|
||||
const metadataErrors = collectReportIntegrityErrors(metadata, integrityOptions);
|
||||
const gaps = collectCriticalCoverageGaps([
|
||||
{ packetId: metadata.packetId, caseStatuses: metadata.caseStatuses, widgetChecks: [], metadataErrors },
|
||||
]);
|
||||
|
||||
expect(metadataErrors).toContain("preflight-01: report status failed does not match case rollup passed");
|
||||
expect(releaseDecision({ P0: 0, P1: 0 }, gaps)).toBe("NO-GO");
|
||||
});
|
||||
|
||||
it("reports missing execution metadata after any assigned case was reached", () => {
|
||||
const metadata = validMetadata();
|
||||
metadata.execution.url = null;
|
||||
metadata.execution.actualPort = null;
|
||||
|
||||
expect(collectReportIntegrityErrors(metadata, integrityOptions)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"preflight-01: execution.url is required after execution",
|
||||
"preflight-01: execution.actualPort must be a positive integer after execution",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves valid structured report metadata unchanged", () => {
|
||||
const input = validMetadata();
|
||||
const normalized = normalizeReportMetadata(input, {
|
||||
packetId: "preflight-01",
|
||||
expectedCaseIds: ["PF01-FIXTURE-STATE"],
|
||||
expectedWidgetChecks: [],
|
||||
});
|
||||
|
||||
expect(normalized.errors).toEqual([]);
|
||||
expect(normalized.metadata).toEqual(input);
|
||||
expect(collectReportIntegrityErrors(normalized.metadata, integrityOptions)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import { assertSafeContainedPath, isPathWithin } from "./safety.mts";
|
||||
|
||||
const errorCode = (error: unknown): string | undefined => (error as NodeJS.ErrnoException).code;
|
||||
|
||||
export const assertSafeReportPath = async (qaRoot: string, target: string, label: string): Promise<string> =>
|
||||
assertSafeContainedPath(qaRoot, target, label);
|
||||
|
||||
export const readSafeReportFile = async (qaRoot: string, target: string, label: string): Promise<string> => {
|
||||
await assertSafeReportPath(qaRoot, target, label);
|
||||
const handle = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
return await handle.readFile("utf8");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
|
||||
export const writeSafeReportFile = async (
|
||||
qaRoot: string,
|
||||
target: string,
|
||||
content: string,
|
||||
label: string,
|
||||
): Promise<void> => {
|
||||
await assertSafeReportPath(qaRoot, target, label);
|
||||
const flags = constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW;
|
||||
const handle = await open(target, flags, 0o666);
|
||||
try {
|
||||
await handle.writeFile(content, "utf8");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
};
|
||||
|
||||
export const validateResolvedArtifactPath = async (
|
||||
expectedPacketDirectory: string,
|
||||
artifactPath: string,
|
||||
label: string,
|
||||
): Promise<string | null> => {
|
||||
if (!path.isAbsolute(artifactPath) || !isPathWithin(expectedPacketDirectory, artifactPath)) {
|
||||
return `${label} must use the packet's absolute artifact directory`;
|
||||
}
|
||||
|
||||
let resolvedPacketDirectory: string;
|
||||
try {
|
||||
resolvedPacketDirectory = await realpath(expectedPacketDirectory);
|
||||
} catch (error) {
|
||||
return `${label} packet artifact directory does not resolve (${errorCode(error) ?? "unknown error"})`;
|
||||
}
|
||||
if (resolvedPacketDirectory !== path.resolve(expectedPacketDirectory)) {
|
||||
return `${label} packet artifact directory must not contain symlinks`;
|
||||
}
|
||||
|
||||
let resolvedArtifact: string;
|
||||
try {
|
||||
resolvedArtifact = await realpath(artifactPath);
|
||||
} catch (error) {
|
||||
return `${label} does not resolve (${errorCode(error) ?? "unknown error"})`;
|
||||
}
|
||||
if (!isPathWithin(resolvedPacketDirectory, resolvedArtifact)) {
|
||||
return `${label} resolves outside the packet artifact directory`;
|
||||
}
|
||||
try {
|
||||
await assertSafeContainedPath(resolvedPacketDirectory, artifactPath, label);
|
||||
const artifactStats = await lstat(artifactPath);
|
||||
if (!artifactStats.isFile()) return `${label} must resolve to an existing regular file`;
|
||||
} catch (error) {
|
||||
return `${label} is not a safe regular file (${error instanceof Error ? error.message : "unknown error"})`;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
@@ -0,0 +1,120 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
assertSafeReportPath,
|
||||
readSafeReportFile,
|
||||
validateResolvedArtifactPath,
|
||||
writeSafeReportFile,
|
||||
} from "./report-path-integrity.mts";
|
||||
|
||||
const temporaryPaths: string[] = [];
|
||||
|
||||
const temporaryDirectory = async (suffix: string): Promise<string> => {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), `homarr-release-v2-qa-${suffix}-`));
|
||||
temporaryPaths.push(directory);
|
||||
return directory;
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temporaryPaths.splice(0).map((target) => rm(target, { force: true, recursive: true })));
|
||||
});
|
||||
|
||||
describe("release-v2 QA report path integrity", () => {
|
||||
it.each(["reports/preflight-01/report.md", "ledger.json", "master-report.md"])(
|
||||
"rejects a symlinked aggregate input or output path: %s",
|
||||
async (relativePath) => {
|
||||
const qaRoot = await temporaryDirectory("report-path");
|
||||
const target = path.join(qaRoot, "outside-target");
|
||||
const linkedPath = path.join(qaRoot, relativePath);
|
||||
await mkdir(path.dirname(linkedPath), { recursive: true });
|
||||
await writeFile(target, "outside");
|
||||
await symlink(target, linkedPath);
|
||||
|
||||
await expect(assertSafeReportPath(qaRoot, linkedPath, "QA report path")).rejects.toThrow("symlink");
|
||||
await expect(readSafeReportFile(qaRoot, linkedPath, "QA report path")).rejects.toThrow("symlink");
|
||||
await expect(writeSafeReportFile(qaRoot, linkedPath, "replacement", "QA report path")).rejects.toThrow("symlink");
|
||||
await expect(readFile(target, "utf8")).resolves.toBe("outside");
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects an artifact symlink that ultimately escapes its packet directory", async () => {
|
||||
const root = await temporaryDirectory("artifact-escape");
|
||||
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
|
||||
const outsideArtifact = path.join(root, "outside.png");
|
||||
const linkedArtifact = path.join(packetDirectory, "escape.png");
|
||||
await mkdir(packetDirectory, { recursive: true });
|
||||
await writeFile(outsideArtifact, "outside");
|
||||
await symlink(outsideArtifact, linkedArtifact);
|
||||
|
||||
await expect(
|
||||
validateResolvedArtifactPath(packetDirectory, linkedArtifact, "preflight-01: artifact"),
|
||||
).resolves.toContain("resolves outside");
|
||||
});
|
||||
|
||||
it("accepts a resolved regular artifact inside the packet directory", async () => {
|
||||
const root = await temporaryDirectory("artifact-valid");
|
||||
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
|
||||
const artifact = path.join(packetDirectory, "valid.png");
|
||||
await mkdir(packetDirectory, { recursive: true });
|
||||
await writeFile(artifact, "valid");
|
||||
|
||||
await expect(validateResolvedArtifactPath(packetDirectory, artifact, "preflight-01: artifact")).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("rejects an artifact directory and an in-directory artifact symlink", async () => {
|
||||
const root = await temporaryDirectory("artifact-not-file");
|
||||
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
|
||||
const artifactDirectory = path.join(packetDirectory, "looks-like-evidence.png");
|
||||
const realArtifact = path.join(packetDirectory, "real.png");
|
||||
const linkedArtifact = path.join(packetDirectory, "linked.png");
|
||||
await mkdir(artifactDirectory, { recursive: true });
|
||||
await writeFile(realArtifact, "valid");
|
||||
await symlink(realArtifact, linkedArtifact);
|
||||
|
||||
await expect(
|
||||
validateResolvedArtifactPath(packetDirectory, artifactDirectory, "preflight-01: artifact"),
|
||||
).resolves.toContain("regular file");
|
||||
await expect(
|
||||
validateResolvedArtifactPath(packetDirectory, linkedArtifact, "preflight-01: artifact"),
|
||||
).resolves.toContain("symlink");
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")("rejects a FIFO artifact", async () => {
|
||||
const root = await temporaryDirectory("artifact-fifo");
|
||||
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
|
||||
const fifoArtifact = path.join(packetDirectory, "capture.pipe");
|
||||
await mkdir(packetDirectory, { recursive: true });
|
||||
const result = spawnSync("mkfifo", [fifoArtifact]);
|
||||
expect(result.status).toBe(0);
|
||||
|
||||
await expect(
|
||||
validateResolvedArtifactPath(packetDirectory, fifoArtifact, "preflight-01: artifact"),
|
||||
).resolves.toContain("regular file");
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")("rejects a socket artifact", async () => {
|
||||
const root = await temporaryDirectory("artifact-socket");
|
||||
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
|
||||
const socketArtifact = path.join(packetDirectory, "capture.sock");
|
||||
await mkdir(packetDirectory, { recursive: true });
|
||||
const server = createServer();
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(socketArtifact, resolve);
|
||||
});
|
||||
try {
|
||||
await expect(
|
||||
validateResolvedArtifactPath(packetDirectory, socketArtifact, "preflight-01: artifact"),
|
||||
).resolves.toContain("regular file");
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,630 @@
|
||||
import { mkdir, readFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
aggregateQaFindings,
|
||||
collectCriticalCoverageGaps,
|
||||
collectHumanWidgetStatusErrors,
|
||||
collectReportIntegrityErrors,
|
||||
normalizeReportMetadata,
|
||||
releaseDecision,
|
||||
} from "./report-integrity.mts";
|
||||
import {
|
||||
assertSafeReportPath,
|
||||
readSafeReportFile,
|
||||
validateResolvedArtifactPath,
|
||||
writeSafeReportFile,
|
||||
} from "./report-path-integrity.mts";
|
||||
import { resolveCheckoutCandidateSha } from "./provenance.mts";
|
||||
|
||||
type Status = "passed" | "failed" | "blocked" | "not-reached";
|
||||
type Severity = "P0" | "P1" | "P2" | "P3";
|
||||
|
||||
interface Packet {
|
||||
id: string;
|
||||
wave: string;
|
||||
title: string;
|
||||
prRefs: number[];
|
||||
personas: string[];
|
||||
boards: string[];
|
||||
profiles: string[];
|
||||
viewports: string[];
|
||||
zooms: number[];
|
||||
inputs: string[];
|
||||
widgetKinds?: string[];
|
||||
cases: string[];
|
||||
}
|
||||
|
||||
interface Manifest {
|
||||
pullRequests: { number: number; label: string }[];
|
||||
caseDimensions: Record<string, { id: string; expected: string }[]>;
|
||||
packets: Packet[];
|
||||
}
|
||||
|
||||
interface Finding {
|
||||
id?: string;
|
||||
fingerprint?: string;
|
||||
severity: Severity;
|
||||
title: string;
|
||||
area?: string;
|
||||
summary?: string;
|
||||
caseIds?: string[];
|
||||
evidence?: string[];
|
||||
}
|
||||
|
||||
interface ReportMetadata {
|
||||
packetId: string;
|
||||
status: Status;
|
||||
caseStatuses: Record<string, Status>;
|
||||
execution: {
|
||||
candidateSha: string | null;
|
||||
url: string | null;
|
||||
actualPort: number | null;
|
||||
runtimeProfile: string | null;
|
||||
runtimeFlags: string[];
|
||||
persona: string | null;
|
||||
sessionId: string | null;
|
||||
timestamp: string | null;
|
||||
viewport: string | null;
|
||||
input: string | null;
|
||||
zoom: number | null;
|
||||
};
|
||||
findings: Finding[];
|
||||
artifacts: string[];
|
||||
widgetChecks: WidgetCheck[];
|
||||
performance: {
|
||||
measurements: PerformanceMeasurement[];
|
||||
limitations: string[];
|
||||
};
|
||||
independentReproductions: IndependentReproduction[];
|
||||
notes: string;
|
||||
}
|
||||
|
||||
interface WidgetCheck {
|
||||
widgetKind: string;
|
||||
viewport: string;
|
||||
sizeRequirement: string;
|
||||
status: Status;
|
||||
}
|
||||
|
||||
interface PerformanceMeasurement {
|
||||
name: string;
|
||||
value: number | null;
|
||||
unit: string;
|
||||
threshold: string;
|
||||
status: Status;
|
||||
evidence: string[];
|
||||
}
|
||||
|
||||
interface IndependentReproduction {
|
||||
findingFingerprint: string;
|
||||
agentId: string;
|
||||
outcome: "reproduced" | "not-reproduced" | "blocked" | "not-reached";
|
||||
evidence: string[];
|
||||
notes: string;
|
||||
}
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const qaRoot = join(repoRoot, "qa/release-v2");
|
||||
const reportsRoot = join(qaRoot, "reports");
|
||||
const campaignCandidateSha = resolveCheckoutCandidateSha(repoRoot);
|
||||
const hostRuntimeLimitation =
|
||||
"The host's fs.inotify.max_user_instances=128 is too low for concurrent native Watchpack watchers. Spawned QA apps keep Turbopack and use 1000ms Watchpack polling; filesystem changes can take up to one polling interval to reach HMR.";
|
||||
const expectedProfileFlags: Record<string, string[]> = {
|
||||
"main-writable": ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
"main-readonly": ["DEMO_MODE=true", "DEMO_READ_ONLY=true", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
"onboarding-fresh": ["DEMO_MODE=false", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
degraded: ["DEMO_MODE=true", "DEMO_READ_ONLY=false", "UNSAFE_ENABLE_MOCK_INTEGRATION=true"],
|
||||
};
|
||||
const exhaustiveSizeWidgetKinds = new Set([
|
||||
"downloads",
|
||||
"calendar",
|
||||
"mediaServer",
|
||||
"mediaRequests-requestList",
|
||||
"mediaRequests-requestStats",
|
||||
"systemResources",
|
||||
"systemDisks",
|
||||
"dockerContainers",
|
||||
"networkControllerSummary",
|
||||
"networkControllerStatus",
|
||||
"beszelSystemStats",
|
||||
"uptimeKuma",
|
||||
"bookmarks",
|
||||
"notebook",
|
||||
"iframe",
|
||||
"customApi",
|
||||
"assistant",
|
||||
]);
|
||||
const manifest = JSON.parse(await readFile(join(qaRoot, "coverage-manifest.json"), "utf8")) as Manifest;
|
||||
const init = process.argv.includes("--init");
|
||||
const refresh = process.argv.includes("--refresh-placeholders");
|
||||
|
||||
const escapeCell = (value: string): string => value.replaceAll("|", "\\|").replaceAll("\n", " ");
|
||||
const list = (values: (string | number)[]): string => (values.length === 0 ? "None" : values.join(", "));
|
||||
|
||||
const expandedCases = (packet: Packet): { id: string; expected: string }[] => {
|
||||
const dimensions = manifest.caseDimensions[packet.wave] ?? [];
|
||||
return packet.cases.flatMap((baseId) =>
|
||||
dimensions.map((dimension) => ({ id: `${baseId}-${dimension.id}`, expected: dimension.expected })),
|
||||
);
|
||||
};
|
||||
|
||||
const sizeRequirement = (widgetKind: string) => {
|
||||
if (exhaustiveSizeWidgetKinds.has(widgetKind)) {
|
||||
return "every width 1-24 × every height 1-6";
|
||||
}
|
||||
return "minimum / canonical / wide / tall / maximum / overflow / behavior-changing thresholds";
|
||||
};
|
||||
|
||||
const widgetChecks = (packet: Packet): WidgetCheck[] =>
|
||||
(packet.widgetKinds ?? []).flatMap((widgetKind) =>
|
||||
packet.viewports.map((viewport) => ({
|
||||
widgetKind,
|
||||
viewport,
|
||||
sizeRequirement: sizeRequirement(widgetKind),
|
||||
status: "not-reached",
|
||||
})),
|
||||
);
|
||||
|
||||
const reportMetadata = (packet: Packet): ReportMetadata => ({
|
||||
packetId: packet.id,
|
||||
status: "not-reached",
|
||||
caseStatuses: Object.fromEntries(expandedCases(packet).map(({ id }) => [id, "not-reached" as const])),
|
||||
execution: {
|
||||
candidateSha: campaignCandidateSha,
|
||||
url: null,
|
||||
actualPort: null,
|
||||
runtimeProfile: null,
|
||||
runtimeFlags: [],
|
||||
persona: null,
|
||||
sessionId: null,
|
||||
timestamp: null,
|
||||
viewport: null,
|
||||
input: null,
|
||||
zoom: null,
|
||||
},
|
||||
findings: [],
|
||||
artifacts: [],
|
||||
widgetChecks: widgetChecks(packet),
|
||||
performance: { measurements: [], limitations: [] },
|
||||
independentReproductions: [],
|
||||
notes: "Not executed.",
|
||||
});
|
||||
|
||||
const renderPacketReportBase = (packet: Packet): string => {
|
||||
const artifactDirectory = resolve(repoRoot, ".screenshots/release-v2", packet.id);
|
||||
const metadata = reportMetadata(packet);
|
||||
const cases = expandedCases(packet)
|
||||
.map(
|
||||
({ id, expected }) =>
|
||||
`| ${id} | not-reached | [planned screenshot](${artifactDirectory}/${id}.png) | Not executed | ${expected} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const widgetMatrix = metadata.widgetChecks
|
||||
.map(
|
||||
(check) =>
|
||||
`| ${check.widgetKind} | ${check.viewport} | ${check.sizeRequirement} | loading / populated / empty / error | Owner / Editor / Viewer / read-only | mutate / save / reload / reset | fail / retry / healthy | [planned artifact](${artifactDirectory}/${check.widgetKind}-${check.viewport}.png) | ${check.status} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const widgetSection =
|
||||
widgetMatrix.length === 0
|
||||
? ""
|
||||
: `\n## Widget kind × viewport checklist\n\n| Widget kind | Viewport | Size | States | Permission/read-only | Options/persistence | Recovery | Evidence | Status |\n| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n${widgetMatrix}\n`;
|
||||
|
||||
return `<!-- release-v2-qa-report\n${JSON.stringify(metadata, null, 2)}\n-->\n\n# ${packet.id}: ${packet.title}\n\n| Metadata | Value |\n| --- | --- |\n| Status | not-reached |\n| Wave | ${packet.wave} |\n| PR refs | ${packet.prRefs.map((number) => `#${number}`).join(", ")} |\n| Personas | ${list(packet.personas)} |\n| Boards | ${list(packet.boards)} |\n| Profiles | ${list(packet.profiles)} |\n| Chromium viewports | ${list(packet.viewports)} |\n| Zoom | ${list(packet.zooms.map((zoom) => `${zoom}%`))} |\n| Input | ${list(packet.inputs)} |\n| Widget kinds | ${list(packet.widgetKinds ?? [])} |\n| Artifact directory | [${artifactDirectory}](${artifactDirectory}) |\n| Candidate SHA | not-recorded |\n| URL / actual port | not-recorded |\n| Runtime profile / flags | not-recorded |\n| Executed persona | not-recorded |\n| Browser session ID | not-recorded |\n| Timestamp | not-recorded |\n| Executed viewport / input / zoom | not-recorded |\n\n## Dogfood evidence\n\n| Case ID | Status | Evidence | Observed | Expected |\n| --- | --- | --- | --- | --- |\n${cases}\n${widgetSection}\n## Findings\n\nNo findings recorded. Add structured findings to the metadata block and mirror them here.\n\n## Blockers and gaps\n\nNot executed.\n\n## Cleanup\n\nRecord created resources, cleanup outcome, console errors, and network failures. Never include secret values.\n`;
|
||||
};
|
||||
|
||||
const renderPacketReport = (packet: Packet): string =>
|
||||
renderPacketReportBase(packet)
|
||||
.replace("| Candidate SHA | not-recorded |", `| Candidate SHA | ${campaignCandidateSha} |`)
|
||||
.replace(
|
||||
"\n## Findings",
|
||||
"\n## Performance measurements and limitations\n\nRecord structured measurements in `performance.measurements` with units, thresholds, status, and evidence. Record every environmental or tooling limitation in `performance.limitations`; leave measurements empty instead of inventing a result.\n\n## Independent reproductions\n\nRecord a second agent's result in `independentReproductions` with the finding fingerprint, agent ID, outcome, evidence, and notes. An unattempted reproduction remains `not-reached`.\n\n## Findings",
|
||||
);
|
||||
|
||||
const parseMetadata = (content: string, packet: Packet): { metadata: ReportMetadata; errors: string[] } => {
|
||||
const metadata = content.match(/<!-- release-v2-qa-report\n([\s\S]*?)\n-->/)?.[1];
|
||||
let rawMetadata: unknown = null;
|
||||
const parseErrors: string[] = [];
|
||||
if (!metadata) {
|
||||
parseErrors.push(`${packet.id}: report metadata block is missing`);
|
||||
} else {
|
||||
try {
|
||||
rawMetadata = JSON.parse(metadata) as unknown;
|
||||
} catch (error) {
|
||||
parseErrors.push(`${packet.id}: report metadata is invalid JSON (${String(error)})`);
|
||||
}
|
||||
}
|
||||
const normalized = normalizeReportMetadata(rawMetadata, {
|
||||
packetId: packet.id,
|
||||
expectedCaseIds: expandedCases(packet).map(({ id }) => id),
|
||||
expectedWidgetChecks: widgetChecks(packet),
|
||||
});
|
||||
return {
|
||||
metadata: normalized.metadata as ReportMetadata,
|
||||
errors: [...parseErrors, ...normalized.errors],
|
||||
};
|
||||
};
|
||||
|
||||
const collectHumanReportErrors = (content: string, packet: Packet, metadata: ReportMetadata): string[] => {
|
||||
const errors: string[] = [];
|
||||
if (!content.includes(`| Status | ${metadata.status} |`)) {
|
||||
errors.push(`${packet.id}: human metadata status differs from structured metadata`);
|
||||
}
|
||||
if (!content.includes(`| Candidate SHA | ${campaignCandidateSha} |`)) {
|
||||
errors.push(`${packet.id}: human metadata does not show the campaign candidate`);
|
||||
}
|
||||
for (const [caseId, status] of Object.entries(metadata.caseStatuses)) {
|
||||
if (!content.includes(`| ${caseId} | ${status} |`)) {
|
||||
errors.push(`${packet.id}/${caseId}: human evidence row/status is missing`);
|
||||
}
|
||||
}
|
||||
errors.push(...collectHumanWidgetStatusErrors(content, packet.id, metadata.widgetChecks));
|
||||
|
||||
const hasExecution =
|
||||
metadata.status !== "not-reached" ||
|
||||
Object.values(metadata.caseStatuses).some((status) => status !== "not-reached") ||
|
||||
metadata.widgetChecks.some((check) => check.status !== "not-reached") ||
|
||||
metadata.findings.length > 0 ||
|
||||
metadata.artifacts.length > 0;
|
||||
if (!hasExecution) return errors;
|
||||
|
||||
const requiredValues = [
|
||||
metadata.execution.candidateSha,
|
||||
metadata.execution.url,
|
||||
metadata.execution.actualPort,
|
||||
metadata.execution.runtimeProfile,
|
||||
metadata.execution.persona,
|
||||
metadata.execution.sessionId,
|
||||
metadata.execution.timestamp,
|
||||
metadata.execution.viewport,
|
||||
metadata.execution.input,
|
||||
metadata.execution.zoom,
|
||||
];
|
||||
for (const value of requiredValues) {
|
||||
if (value !== null && !content.includes(String(value))) {
|
||||
errors.push(`${packet.id}: human metadata does not include execution value ${String(value)}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
};
|
||||
|
||||
const collectArtifactErrors = async (packet: Packet, metadata: ReportMetadata): Promise<string[]> => {
|
||||
const packetArtifactDirectory = resolve(repoRoot, ".screenshots/release-v2", packet.id);
|
||||
const artifacts = [
|
||||
...metadata.artifacts.map((artifact) => ({ artifact, label: `${packet.id}: artifact` })),
|
||||
...metadata.findings.flatMap((finding, findingIndex) =>
|
||||
(finding.evidence ?? []).map((artifact) => ({
|
||||
artifact,
|
||||
label: `${packet.id}: findings[${findingIndex}] evidence`,
|
||||
})),
|
||||
),
|
||||
...metadata.performance.measurements.flatMap((measurement, measurementIndex) =>
|
||||
measurement.evidence.map((artifact) => ({
|
||||
artifact,
|
||||
label: `${packet.id}: performance.measurements[${measurementIndex}] evidence`,
|
||||
})),
|
||||
),
|
||||
...metadata.independentReproductions.flatMap((reproduction, reproductionIndex) =>
|
||||
reproduction.evidence.map((artifact) => ({
|
||||
artifact,
|
||||
label: `${packet.id}: independentReproductions[${reproductionIndex}] evidence`,
|
||||
})),
|
||||
),
|
||||
];
|
||||
|
||||
const errors: string[] = [];
|
||||
for (const { artifact, label } of artifacts) {
|
||||
const error = await validateResolvedArtifactPath(packetArtifactDirectory, artifact, label);
|
||||
if (error) errors.push(error);
|
||||
}
|
||||
return errors;
|
||||
};
|
||||
|
||||
const ensureReports = async (): Promise<void> => {
|
||||
for (const packet of manifest.packets) {
|
||||
const packetDirectory = join(reportsRoot, packet.id);
|
||||
const reportPath = join(packetDirectory, "report.md");
|
||||
await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`);
|
||||
await mkdir(packetDirectory, { recursive: true });
|
||||
await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`);
|
||||
if (refresh) {
|
||||
try {
|
||||
const parsed = parseMetadata(await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`), packet);
|
||||
if (parsed.errors.length > 0) throw new Error(parsed.errors.join("; "));
|
||||
const existing = parsed.metadata;
|
||||
const hasExecution =
|
||||
existing.status !== "not-reached" ||
|
||||
Object.values(existing.caseStatuses).some((status) => status !== "not-reached") ||
|
||||
existing.findings.length > 0 ||
|
||||
existing.artifacts.length > 0 ||
|
||||
(existing.widgetChecks ?? []).some((check) => check.status !== "not-reached") ||
|
||||
(existing.performance?.measurements.length ?? 0) > 0 ||
|
||||
(existing.performance?.limitations.length ?? 0) > 0 ||
|
||||
(existing.independentReproductions?.length ?? 0) > 0;
|
||||
if (hasExecution) throw new Error(`${packet.id}: refusing to replace a report containing execution results`);
|
||||
} catch (error) {
|
||||
if (!(error instanceof Error) || !error.message.includes("ENOENT")) throw error;
|
||||
}
|
||||
await writeSafeReportFile(qaRoot, reportPath, renderPacketReport(packet), `${packet.id} report path`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
await writeSafeReportFile(qaRoot, reportPath, renderPacketReport(packet), `${packet.id} report path`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
interface CoverageAssignment {
|
||||
label: string;
|
||||
detail?: string;
|
||||
status: Status;
|
||||
}
|
||||
|
||||
const rollupStatuses = (statuses: Status[]): Status => {
|
||||
if (statuses.includes("failed")) return "failed";
|
||||
if (statuses.includes("blocked")) return "blocked";
|
||||
if (statuses.includes("not-reached")) return "not-reached";
|
||||
return "passed";
|
||||
};
|
||||
|
||||
const matchingCaseStatus = (metadata: ReportMetadata, fragments: string[]) => {
|
||||
const statuses = Object.entries(metadata.caseStatuses)
|
||||
.filter(([caseId]) => fragments.some((fragment) => caseId.includes(fragment)))
|
||||
.map(([, status]) => status);
|
||||
if (statuses.length === 0) return metadata.status;
|
||||
return rollupStatuses(statuses);
|
||||
};
|
||||
|
||||
const coverageTable = (assignments: CoverageAssignment[], labelHeading: string) => {
|
||||
const grouped = new Map<string, { detail: string; statuses: Status[] }>();
|
||||
for (const assignment of assignments) {
|
||||
const existing = grouped.get(assignment.label);
|
||||
if (existing) {
|
||||
existing.statuses.push(assignment.status);
|
||||
continue;
|
||||
}
|
||||
grouped.set(assignment.label, { detail: assignment.detail ?? "—", statuses: [assignment.status] });
|
||||
}
|
||||
if (grouped.size === 0)
|
||||
return `| ${labelHeading} | Detail | Passed | Failed | Blocked | Not reached | Total |\n| --- | --- | ---: | ---: | ---: | ---: | ---: |\n| — | Not recorded | 0 | 0 | 0 | 0 | 0 |`;
|
||||
const rows = [...grouped.entries()]
|
||||
.toSorted(([left], [right]) => left.localeCompare(right))
|
||||
.map(([label, value]) => {
|
||||
const count = (status: Status) => value.statuses.filter((current) => current === status).length;
|
||||
return `| ${escapeCell(label)} | ${escapeCell(value.detail)} | ${count("passed")} | ${count("failed")} | ${count("blocked")} | ${count("not-reached")} | ${value.statuses.length} |`;
|
||||
});
|
||||
return `| ${labelHeading} | Detail | Passed | Failed | Blocked | Not reached | Total |\n| --- | --- | ---: | ---: | ---: | ---: | ---: |\n${rows.join("\n")}`;
|
||||
};
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
if (init || refresh) await ensureReports();
|
||||
|
||||
const reports: { packet: Packet; metadata: ReportMetadata; metadataErrors: string[] }[] = [];
|
||||
for (const packet of manifest.packets) {
|
||||
const reportPath = join(reportsRoot, packet.id, "report.md");
|
||||
await assertSafeReportPath(qaRoot, reportPath, `${packet.id} report path`);
|
||||
let content = "";
|
||||
const readErrors: string[] = [];
|
||||
try {
|
||||
content = await readSafeReportFile(qaRoot, reportPath, `${packet.id} report path`);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
readErrors.push(`${packet.id}: report.md is missing`);
|
||||
}
|
||||
const parsed = parseMetadata(content, packet);
|
||||
const integrityErrors = collectReportIntegrityErrors(parsed.metadata, {
|
||||
campaignCandidateSha,
|
||||
packetId: packet.id,
|
||||
profiles: packet.profiles,
|
||||
personas: packet.personas,
|
||||
viewports: packet.viewports,
|
||||
zooms: packet.zooms,
|
||||
inputs: packet.inputs,
|
||||
expectedProfileFlags,
|
||||
});
|
||||
const humanReportErrors = collectHumanReportErrors(content, packet, parsed.metadata);
|
||||
const artifactErrors = await collectArtifactErrors(packet, parsed.metadata);
|
||||
reports.push({
|
||||
packet,
|
||||
metadata: parsed.metadata,
|
||||
metadataErrors: [...readErrors, ...parsed.errors, ...integrityErrors, ...humanReportErrors, ...artifactErrors],
|
||||
});
|
||||
}
|
||||
|
||||
const packetTotals: Record<Status, number> = { passed: 0, failed: 0, blocked: 0, "not-reached": 0 };
|
||||
const caseTotals: Record<Status, number> = { passed: 0, failed: 0, blocked: 0, "not-reached": 0 };
|
||||
for (const { metadata } of reports) {
|
||||
packetTotals[metadata.status] += 1;
|
||||
for (const status of Object.values(metadata.caseStatuses)) caseTotals[status] += 1;
|
||||
}
|
||||
|
||||
const findings = aggregateQaFindings(
|
||||
reports.map(({ packet, metadata }) => ({
|
||||
packetId: packet.id,
|
||||
findings: metadata.findings,
|
||||
independentReproductions: metadata.independentReproductions,
|
||||
})),
|
||||
);
|
||||
const severityTotals: Record<Severity, number> = { P0: 0, P1: 0, P2: 0, P3: 0 };
|
||||
for (const finding of findings) severityTotals[finding.severity] += 1;
|
||||
|
||||
const criticalGaps = collectCriticalCoverageGaps(
|
||||
reports.map(({ packet, metadata, metadataErrors }) => ({
|
||||
packetId: packet.id,
|
||||
caseStatuses: metadata.caseStatuses,
|
||||
widgetChecks: metadata.widgetChecks,
|
||||
metadataErrors,
|
||||
})),
|
||||
);
|
||||
const decision = releaseDecision(severityTotals, criticalGaps);
|
||||
const widgetTotals: Record<Status, number> = { passed: 0, failed: 0, blocked: 0, "not-reached": 0 };
|
||||
for (const { metadata } of reports) {
|
||||
for (const check of metadata.widgetChecks) widgetTotals[check.status] += 1;
|
||||
}
|
||||
|
||||
const ledger = {
|
||||
schemaVersion: 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
packetStatuses: Object.fromEntries(
|
||||
reports.map(({ packet, metadata }) => [packet.id, { status: metadata.status, cases: metadata.caseStatuses }]),
|
||||
),
|
||||
};
|
||||
const ledgerPath = join(qaRoot, "ledger.json");
|
||||
const masterReportPath = join(qaRoot, "master-report.md");
|
||||
await assertSafeReportPath(qaRoot, ledgerPath, "release-v2 QA ledger path");
|
||||
await assertSafeReportPath(qaRoot, masterReportPath, "release-v2 QA master report path");
|
||||
await writeSafeReportFile(qaRoot, ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, "release-v2 QA ledger path");
|
||||
|
||||
const waveRows = Object.entries(Object.groupBy(reports, ({ packet }) => packet.wave)).map(([wave, entries]) => {
|
||||
const waveReports = entries ?? [];
|
||||
const passed = waveReports.filter(({ metadata }) => metadata.status === "passed").length;
|
||||
return `| ${wave} | ${passed} | ${waveReports.length} |`;
|
||||
});
|
||||
const findingRows =
|
||||
findings.length === 0
|
||||
? "| — | — | No findings recorded | — | — |"
|
||||
: findings
|
||||
.map(
|
||||
(finding) =>
|
||||
`| ${finding.severity} | ${escapeCell(finding.area ?? "general")} | ${escapeCell(finding.title)} | ${list(finding.packets)} | ${list(finding.caseIds)} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const gapRows =
|
||||
criticalGaps.length === 0
|
||||
? "| — | — | — | — |"
|
||||
: criticalGaps
|
||||
.map(
|
||||
(gap) =>
|
||||
`| ${gap.packetId} | ${gap.kind} | ${escapeCell(gap.itemId)} | ${gap.status}${gap.detail ? ` — ${escapeCell(gap.detail)}` : ""} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const prRows = manifest.pullRequests
|
||||
.map(
|
||||
(pullRequest) =>
|
||||
`| #${pullRequest.number} | ${pullRequest.label} | ${manifest.packets.filter((packet) => packet.prRefs.includes(pullRequest.number)).length} |`,
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
const prCoverage = coverageTable(
|
||||
manifest.pullRequests.flatMap((pullRequest) =>
|
||||
reports
|
||||
.filter(({ packet }) => packet.prRefs.includes(pullRequest.number))
|
||||
.map(({ metadata }) => ({
|
||||
label: `#${pullRequest.number}`,
|
||||
detail: pullRequest.label,
|
||||
status: metadata.status,
|
||||
})),
|
||||
),
|
||||
"PR",
|
||||
);
|
||||
const featureCoverage = coverageTable(
|
||||
reports.map(({ packet, metadata }) => ({ label: packet.id, detail: packet.title, status: metadata.status })),
|
||||
"Feature / agent",
|
||||
);
|
||||
const widgetAssignments = reports.flatMap(({ packet, metadata }) =>
|
||||
(metadata.widgetChecks ?? widgetChecks(packet)).map((check) => ({
|
||||
label: check.widgetKind,
|
||||
detail: packet.id,
|
||||
status: check.status,
|
||||
})),
|
||||
);
|
||||
const widgetCoverage = coverageTable(widgetAssignments, "Widget kind");
|
||||
const sizeCoverage = coverageTable(
|
||||
reports.flatMap(({ packet, metadata }) =>
|
||||
(metadata.widgetChecks ?? widgetChecks(packet)).map((check) => ({
|
||||
label: check.widgetKind,
|
||||
detail: check.sizeRequirement,
|
||||
status: check.status,
|
||||
})),
|
||||
),
|
||||
"Widget size threshold",
|
||||
);
|
||||
const viewportCoverage = coverageTable(
|
||||
reports.flatMap(({ packet, metadata }) =>
|
||||
packet.viewports.map((viewport) => ({ label: viewport, detail: packet.id, status: metadata.status })),
|
||||
),
|
||||
"Viewport",
|
||||
);
|
||||
const personaCoverage = coverageTable(
|
||||
reports.flatMap(({ packet, metadata }) =>
|
||||
packet.personas.map((persona) => ({ label: persona, detail: packet.id, status: metadata.status })),
|
||||
),
|
||||
"Persona",
|
||||
);
|
||||
const permissionCoverage = coverageTable(
|
||||
reports.flatMap(({ packet, metadata }) => {
|
||||
const status = matchingCaseStatus(metadata, ["ACCESS", "PERMISSION", "READONLY"]);
|
||||
return [
|
||||
...packet.personas.map((persona) => ({ label: persona, detail: packet.id, status })),
|
||||
...packet.profiles.map((profile) => ({ label: `profile:${profile}`, detail: packet.id, status })),
|
||||
];
|
||||
}),
|
||||
"Permission boundary",
|
||||
);
|
||||
const mutationCoverage = coverageTable(
|
||||
reports.map(({ packet, metadata }) => ({
|
||||
label: packet.id,
|
||||
detail: packet.title,
|
||||
status: matchingCaseStatus(metadata, ["MUTATION", "PERSISTENCE", "OPTIONS"]),
|
||||
})),
|
||||
"Mutation area",
|
||||
);
|
||||
const stateCoverage = coverageTable(
|
||||
reports.map(({ packet, metadata }) => ({
|
||||
label: packet.id,
|
||||
detail: packet.title,
|
||||
status: matchingCaseStatus(metadata, ["STATE", "DEGRADED", "RECOVERY", "FAILURE", "EVIDENCE"]),
|
||||
})),
|
||||
"State / recovery area",
|
||||
);
|
||||
const performanceMeasurements = reports.flatMap(({ packet, metadata }) =>
|
||||
(metadata.performance?.measurements ?? []).map((measurement) => ({ packet, measurement })),
|
||||
);
|
||||
const performanceRows =
|
||||
performanceMeasurements.length === 0
|
||||
? "| — | No measurement recorded | — | — | not-reached | — |"
|
||||
: performanceMeasurements
|
||||
.map(({ packet, measurement }) => {
|
||||
const value = measurement.value === null ? "not-recorded" : `${measurement.value} ${measurement.unit}`;
|
||||
return `| ${packet.id} | ${escapeCell(measurement.name)} | ${escapeCell(value)} | ${escapeCell(measurement.threshold)} | ${measurement.status} | ${list(measurement.evidence)} |`;
|
||||
})
|
||||
.join("\n");
|
||||
const limitationRows = [
|
||||
`| host-runtime | ${escapeCell(hostRuntimeLimitation)} |`,
|
||||
...reports.flatMap(({ packet, metadata }) =>
|
||||
(metadata.performance?.limitations ?? []).map((limitation) => `| ${packet.id} | ${escapeCell(limitation)} |`),
|
||||
),
|
||||
];
|
||||
const reproductionRows = findings.flatMap((finding) =>
|
||||
finding.reproductions.map(
|
||||
(reproduction) =>
|
||||
`| ${list(finding.packets)} | ${escapeCell(reproduction.findingFingerprint)} | ${escapeCell(reproduction.agentId)} | ${reproduction.outcome} | ${list(reproduction.evidence)} | ${escapeCell(reproduction.notes)} |`,
|
||||
),
|
||||
);
|
||||
|
||||
const coverageAppendix = `## Coverage by required axis\n\n### Pull request\n\n${prCoverage}\n\n### Feature\n\n${featureCoverage}\n\n### Widget\n\n${widgetCoverage}\n\n### Size threshold\n\nHigh-risk widgets require every width 1–24 × every height 1–6 at all assigned mobile, breakpoint-edge, and desktop viewports. Other widgets require minimum, canonical, wide, tall, maximum, overflow, and behavior-changing threshold checks.\n\n${sizeCoverage}\n\n### Viewport\n\n${viewportCoverage}\n\n### Persona\n\n${personaCoverage}\n\n### Permission\n\n${permissionCoverage}\n\n### Mutation\n\n${mutationCoverage}\n\n### State and recovery\n\n${stateCoverage}\n\n## Performance measurements\n\n| Agent | Measurement | Value | Threshold | Status | Evidence |\n| --- | --- | --- | --- | --- | --- |\n${performanceRows}\n\n### Performance limitations\n\n| Agent | Limitation |\n| --- | --- |\n${limitationRows.length > 0 ? limitationRows.join("\n") : "| — | Not recorded |"}\n\n## Independent reproduction results\n\n| Agent | Finding fingerprint | Reproducing agent | Outcome | Evidence | Notes |\n| --- | --- | --- | --- | --- | --- |\n${reproductionRows.length > 0 ? reproductionRows.join("\n") : "| — | — | — | not-reached | — | Not recorded |"}\n\n`;
|
||||
|
||||
const master = `# Release-v2 QA master report\n\nGenerated: ${new Date().toISOString()}\n\n## Decision: ${decision}\n\nGO requires zero P0/P1 findings and zero critical gaps across assigned cases, widget checks, and report metadata. Current critical gaps: **${criticalGaps.length}**.\n\n## Coverage\n\n| Measure | Passed | Failed | Blocked | Not reached | Total |\n| --- | ---: | ---: | ---: | ---: | ---: |\n| Packets | ${packetTotals.passed} | ${packetTotals.failed} | ${packetTotals.blocked} | ${packetTotals["not-reached"]} | ${reports.length} |\n| Cases | ${caseTotals.passed} | ${caseTotals.failed} | ${caseTotals.blocked} | ${caseTotals["not-reached"]} | ${Object.values(caseTotals).reduce((sum, count) => sum + count, 0)} |\n| Widget checks | ${widgetTotals.passed} | ${widgetTotals.failed} | ${widgetTotals.blocked} | ${widgetTotals["not-reached"]} | ${Object.values(widgetTotals).reduce((sum, count) => sum + count, 0)} |\n\n| Wave | Passed | Total |\n| --- | ---: | ---: |\n${waveRows.join("\n")}\n\n## Severity totals\n\n| P0 | P1 | P2 | P3 |\n| ---: | ---: | ---: | ---: |\n| ${severityTotals.P0} | ${severityTotals.P1} | ${severityTotals.P2} | ${severityTotals.P3} |\n\nDeduplicated by explicit fingerprint, or by severity + area + title when no fingerprint is supplied.\n\n| Severity | Area | Finding | Packets | Cases |\n| --- | --- | --- | --- | --- |\n${findingRows}\n\n## Critical gaps\n\n| Packet | Kind | Coverage item | Status / detail |\n| --- | --- | --- | --- |\n${gapRows}\n\n## PR coverage\n\n| PR | Area | Packets |\n| --- | --- | ---: |\n${prRows}\n\n## Packet reports\n\n${reports.map(({ packet, metadata }) => `- [${packet.id}: ${packet.title}](packets/${packet.id}/report.md) — ${metadata.status}`).join("\n")}\n`;
|
||||
const enrichedMaster = master
|
||||
.replace("## Severity totals", `${coverageAppendix}## Severity totals`)
|
||||
.replaceAll("(packets/", "(reports/")
|
||||
.replace("## Packet reports", "## Agent reports");
|
||||
await writeSafeReportFile(qaRoot, masterReportPath, enrichedMaster, "release-v2 QA master report path");
|
||||
|
||||
console.log(
|
||||
`release-v2 QA report: ${decision}; ${reports.length} packets; ${findings.length} deduplicated findings; ${criticalGaps.length} critical gaps`,
|
||||
);
|
||||
const metadataErrors = reports.flatMap((report) => report.metadataErrors);
|
||||
if (metadataErrors.length > 0) {
|
||||
console.error(`release-v2 QA report found ${metadataErrors.length} metadata error(s):`);
|
||||
for (const error of metadataErrors) console.error(`- ${error}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
|
||||
await main();
|
||||
@@ -0,0 +1,739 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { closeSync, constants, existsSync, openSync } from "node:fs";
|
||||
import { chmod, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises";
|
||||
import { createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
import {
|
||||
acquireSlotLease,
|
||||
assertSafeContainedPath,
|
||||
assertSafeRunRoot,
|
||||
assertSupportedRuntimePlatform,
|
||||
createFixtureEnvironment,
|
||||
sanitizeAppEnvironment,
|
||||
} from "./safety.mts";
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, "../..");
|
||||
const defaultRunRoot = path.join(tmpdir(), "homarr-release-v2-qa");
|
||||
const markerName = ".homarr-release-v2-qa-root.json";
|
||||
const watchpackPollingIntervalMs = 1_000;
|
||||
const profiles = ["main-writable", "main-readonly", "onboarding-fresh", "degraded"] as const;
|
||||
|
||||
type Profile = (typeof profiles)[number];
|
||||
export type RuntimeCommand = "start" | "status" | "stop";
|
||||
|
||||
export const assertRuntimeCommandPlatform = (command: RuntimeCommand, platform: NodeJS.Platform = process.platform) => {
|
||||
if (command === "status") return;
|
||||
assertSupportedRuntimePlatform(platform);
|
||||
};
|
||||
|
||||
interface Options {
|
||||
command: RuntimeCommand;
|
||||
slot: number;
|
||||
profile: Profile;
|
||||
runRoot: string;
|
||||
candidate: string;
|
||||
reset: boolean;
|
||||
}
|
||||
|
||||
interface RuntimeManifest {
|
||||
schemaVersion: 1;
|
||||
runId: string;
|
||||
status: "running" | "stopped";
|
||||
candidateSha: string;
|
||||
branch: string | null;
|
||||
slot: number;
|
||||
profile: Profile;
|
||||
startedAt: string;
|
||||
stoppedAt?: string;
|
||||
repoRoot: string;
|
||||
runRoot: string;
|
||||
slotDir: string;
|
||||
dbPath: string;
|
||||
fixtureManifestPath: string;
|
||||
nextDistDir: string;
|
||||
url: string;
|
||||
fixtureUrl: string;
|
||||
ports: { app: number; fixture: number; redis: number };
|
||||
flags: { demoMode: boolean; demoReadOnly: boolean; unsafeMockIntegration: boolean };
|
||||
watcher: { watchpackPollingIntervalMs: number };
|
||||
processes: {
|
||||
app: { pid: number; logPath: string };
|
||||
fixture: { pid: number; logPath: string };
|
||||
};
|
||||
docker: { redisContainer: string };
|
||||
}
|
||||
|
||||
const usage = () => {
|
||||
console.log(
|
||||
"Usage: runtime.mts <start|status|stop> --slot <1|2|3> [--profile <main-writable|main-readonly|onboarding-fresh|degraded>] [--run-root <path>] [--candidate <ref>] [--reset]",
|
||||
);
|
||||
};
|
||||
|
||||
const parseArgs = (): Options => {
|
||||
const args = process.argv.slice(2).filter((argument) => argument !== "--");
|
||||
if (args[0] === "--help" || args[0] === "-h") {
|
||||
usage();
|
||||
process.exit(0);
|
||||
}
|
||||
const command = args.shift();
|
||||
if (command !== "start" && command !== "status" && command !== "stop") {
|
||||
usage();
|
||||
throw new Error("A runtime command is required");
|
||||
}
|
||||
|
||||
let slot = 0;
|
||||
let profile: Profile = "main-writable";
|
||||
let runRoot = defaultRunRoot;
|
||||
let candidate = "HEAD";
|
||||
let reset = false;
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
const value = args[index + 1];
|
||||
if (argument === "--slot" && value) {
|
||||
slot = Number(value);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--profile" && value && profiles.includes(value as Profile)) {
|
||||
profile = value as Profile;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--run-root" && value) {
|
||||
runRoot = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--candidate" && value) {
|
||||
candidate = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--reset") {
|
||||
reset = true;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--help" || argument === "-h") {
|
||||
usage();
|
||||
process.exit(0);
|
||||
}
|
||||
throw new Error(`Unknown or incomplete argument: ${argument ?? "<missing>"}`);
|
||||
}
|
||||
|
||||
if (![1, 2, 3].includes(slot)) throw new Error("--slot must be 1, 2, or 3");
|
||||
return { command, slot, profile, runRoot: path.resolve(runRoot), candidate, reset };
|
||||
};
|
||||
|
||||
const commandOutput = (command: string, args: string[]) => {
|
||||
const result = spawnSync(command, args, { cwd: repoRoot, encoding: "utf8" });
|
||||
if (result.status !== 0) {
|
||||
const detail = result.stderr.trim() || result.stdout.trim() || `exit ${result.status ?? "unknown"}`;
|
||||
throw new Error(`${command} ${args.join(" ")} failed: ${detail}`);
|
||||
}
|
||||
return result.stdout.trim();
|
||||
};
|
||||
|
||||
const resolveCandidate = (candidate: string) =>
|
||||
commandOutput("git", ["rev-parse", "--verify", `${candidate}^{commit}`]).toLowerCase();
|
||||
|
||||
const currentBranch = () => {
|
||||
const result = spawnSync("git", ["branch", "--show-current"], { cwd: repoRoot, encoding: "utf8" });
|
||||
const branch = result.stdout.trim();
|
||||
return branch.length > 0 ? branch : null;
|
||||
};
|
||||
|
||||
const writeJsonAtomic = async (filePath: string, value: unknown) => {
|
||||
await mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
||||
const temporaryPath = `${filePath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`;
|
||||
const handle = await open(
|
||||
temporaryPath,
|
||||
constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW,
|
||||
0o600,
|
||||
);
|
||||
try {
|
||||
await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
try {
|
||||
await rename(temporaryPath, filePath);
|
||||
} catch (error) {
|
||||
await rm(temporaryPath, { force: true });
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const assertRunRootMarker = async (runRoot: string) => {
|
||||
const markerPath = path.join(runRoot, markerName);
|
||||
await assertSafeContainedPath(runRoot, markerPath, "QA run-root marker");
|
||||
const marker = JSON.parse(await readFile(markerPath, "utf8")) as {
|
||||
schemaVersion?: number;
|
||||
repoRoot?: string;
|
||||
runRoot?: string;
|
||||
};
|
||||
if (marker.schemaVersion !== 1 || path.resolve(marker.repoRoot ?? "") !== repoRoot) {
|
||||
throw new Error(`QA run-root marker does not belong to this checkout: ${markerPath}`);
|
||||
}
|
||||
if (marker.runRoot && path.resolve(marker.runRoot) !== runRoot) {
|
||||
throw new Error(`QA run-root marker records a different root: ${markerPath}`);
|
||||
}
|
||||
};
|
||||
|
||||
const readManifest = async (manifestPath: string) => {
|
||||
const value = JSON.parse(await readFile(manifestPath, "utf8")) as RuntimeManifest;
|
||||
if (value.schemaVersion !== 1) throw new Error(`Unsupported runtime manifest: ${manifestPath}`);
|
||||
return value;
|
||||
};
|
||||
|
||||
export interface PortReservation {
|
||||
port: number;
|
||||
release: () => Promise<void>;
|
||||
}
|
||||
|
||||
export const reserveLoopbackPort = async (): Promise<PortReservation> =>
|
||||
await new Promise<PortReservation>((resolve, reject) => {
|
||||
const server = createServer();
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
server.close();
|
||||
reject(new Error("Unable to allocate a TCP port"));
|
||||
return;
|
||||
}
|
||||
let released = false;
|
||||
resolve({
|
||||
port: address.port,
|
||||
release: async () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
await new Promise<void>((closeResolve, closeReject) => {
|
||||
server.close((error) => {
|
||||
if (error) closeReject(error);
|
||||
else closeResolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
type PortLaunchResult<T> = { status: "ready"; value: T } | { status: "address-in-use" };
|
||||
|
||||
export const launchWithPortRetry = async <T,>(
|
||||
launch: (port: number) => Promise<PortLaunchResult<T>>,
|
||||
options: { attempts?: number; initialReservation?: PortReservation } = {},
|
||||
): Promise<T> => {
|
||||
const attempts = options.attempts ?? 3;
|
||||
let reservation = options.initialReservation;
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
reservation ??= await reserveLoopbackPort();
|
||||
const port = reservation.port;
|
||||
await reservation.release();
|
||||
reservation = undefined;
|
||||
const result = await launch(port);
|
||||
if (result.status === "ready") return result.value;
|
||||
}
|
||||
throw new Error(`Unable to launch the QA app after ${attempts} address-in-use retries`);
|
||||
};
|
||||
|
||||
const waitForJsonFile = async <T,>(filePath: string, timeoutMs: number) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
return JSON.parse(await readFile(filePath, "utf8")) as T;
|
||||
} catch {
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${filePath}`);
|
||||
};
|
||||
|
||||
const waitForUrl = async (url: string, timeoutMs: number) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastStatus: number | undefined;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const response = await fetch(url, { redirect: "manual", signal: AbortSignal.timeout(5_000) });
|
||||
lastStatus = response.status;
|
||||
if (response.ok) return;
|
||||
} catch {
|
||||
// Startup failures are reported from the process log after the deadline.
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${url}; last status=${lastStatus ?? "unreachable"}`);
|
||||
};
|
||||
|
||||
interface SpawnedProcess {
|
||||
pid: number;
|
||||
exited: Promise<number | null>;
|
||||
}
|
||||
|
||||
const spawnLogged = (command: string, args: string[], logPath: string, env: NodeJS.ProcessEnv): SpawnedProcess => {
|
||||
const output = openSync(
|
||||
logPath,
|
||||
constants.O_CREAT | constants.O_APPEND | constants.O_WRONLY | constants.O_NOFOLLOW,
|
||||
0o600,
|
||||
);
|
||||
const child = spawn(command, args, {
|
||||
cwd: repoRoot,
|
||||
detached: true,
|
||||
env,
|
||||
stdio: ["ignore", output, output],
|
||||
});
|
||||
closeSync(output);
|
||||
if (!child.pid) throw new Error(`Failed to launch ${command}`);
|
||||
const exited = new Promise<number | null>((resolve) => {
|
||||
child.once("exit", (code) => resolve(code));
|
||||
});
|
||||
child.unref();
|
||||
return { pid: child.pid, exited };
|
||||
};
|
||||
|
||||
const runForeground = (command: string, args: string[], env: NodeJS.ProcessEnv) => {
|
||||
const result = spawnSync(command, args, { cwd: repoRoot, env, stdio: "inherit" });
|
||||
if (result.status !== 0)
|
||||
throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status ?? "unknown"}`);
|
||||
};
|
||||
|
||||
const isProcessRunning = (pid: number) => {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const waitForAppAttempt = async (
|
||||
spawned: SpawnedProcess,
|
||||
url: string,
|
||||
logPath: string,
|
||||
logOffset: number,
|
||||
timeoutMs: number,
|
||||
): Promise<"ready" | "address-in-use"> => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const exitCode = await Promise.race([
|
||||
spawned.exited,
|
||||
new Promise<undefined>((resolve) => setTimeout(() => resolve(undefined), 250)),
|
||||
]);
|
||||
if (exitCode !== undefined) {
|
||||
const log = await readFile(logPath, "utf8").catch(() => "");
|
||||
const attemptLog = log.slice(logOffset);
|
||||
if (/EADDRINUSE|address already in use/iu.test(attemptLog)) return "address-in-use";
|
||||
throw new Error(`QA app process exited during startup; inspect ${logPath}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${url}/api/health/ready`, {
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
if (response.ok && isProcessRunning(spawned.pid)) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
if (isProcessRunning(spawned.pid)) return "ready";
|
||||
}
|
||||
} catch {
|
||||
// Continue while the selected process owns its startup attempt.
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${url}; inspect ${logPath}`);
|
||||
};
|
||||
|
||||
const processBelongsToRun = async (pid: number, runId: string) => {
|
||||
if (!isProcessRunning(pid)) return false;
|
||||
try {
|
||||
const environment = await readFile(`/proc/${pid}/environ`);
|
||||
return environment.includes(Buffer.from(`QA_RUN_ID=${runId}\0`));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const stopProcess = async (pid: number, runId: string) => {
|
||||
if (!isProcessRunning(pid)) return;
|
||||
if (!(await processBelongsToRun(pid, runId))) {
|
||||
throw new Error(`Refusing to signal PID ${pid}: it is not owned by QA run ${runId}`);
|
||||
}
|
||||
try {
|
||||
process.kill(-pid, "SIGTERM");
|
||||
} catch {
|
||||
process.kill(pid, "SIGTERM");
|
||||
}
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline && isProcessRunning(pid)) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 200));
|
||||
}
|
||||
if (isProcessRunning(pid)) {
|
||||
try {
|
||||
process.kill(-pid, "SIGKILL");
|
||||
} catch {
|
||||
process.kill(pid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const removeRedisContainer = (container: string, runId: string) => {
|
||||
const inspect = spawnSync(
|
||||
"docker",
|
||||
["inspect", "--format", '{{ index .Config.Labels "com.homarr.release-v2-qa.run" }}', container],
|
||||
{
|
||||
encoding: "utf8",
|
||||
},
|
||||
);
|
||||
if (inspect.status !== 0) return;
|
||||
if (inspect.stdout.trim() !== runId)
|
||||
throw new Error(`Refusing to remove Docker container without matching QA run label: ${container}`);
|
||||
const result = spawnSync("docker", ["rm", "--force", container], { stdio: "inherit" });
|
||||
if (result.status !== 0) throw new Error(`Failed to remove QA Redis container ${container}`);
|
||||
};
|
||||
|
||||
const stopFromManifest = async (
|
||||
manifestPath: string,
|
||||
manifest: RuntimeManifest,
|
||||
expectedRunRoot: string,
|
||||
expectedSlot: number,
|
||||
) => {
|
||||
if (manifest.status === "stopped") return;
|
||||
const expectedSlotDir = path.join(expectedRunRoot, "slots", String(expectedSlot));
|
||||
const expectedManifestPath = path.join(expectedSlotDir, "runtime-manifest.json");
|
||||
if (path.resolve(manifest.runRoot) !== expectedRunRoot || manifest.slot !== expectedSlot) {
|
||||
throw new Error(`Refusing runtime manifest for a different QA run-root or slot: ${manifestPath}`);
|
||||
}
|
||||
if (path.resolve(manifestPath) !== expectedManifestPath) {
|
||||
throw new Error(`Refusing runtime manifest outside the selected slot: ${manifestPath}`);
|
||||
}
|
||||
if (path.resolve(manifest.slotDir) !== path.resolve(expectedSlotDir)) {
|
||||
throw new Error(`Refusing runtime manifest with an unsafe slot directory: ${manifest.slotDir}`);
|
||||
}
|
||||
await assertSafeContainedPath(expectedRunRoot, expectedSlotDir, "QA slot directory");
|
||||
await stopProcess(manifest.processes.app.pid, manifest.runId);
|
||||
await stopProcess(manifest.processes.fixture.pid, manifest.runId);
|
||||
removeRedisContainer(manifest.docker.redisContainer, manifest.runId);
|
||||
const stopped: RuntimeManifest = { ...manifest, status: "stopped", stoppedAt: new Date().toISOString() };
|
||||
await assertSafeContainedPath(expectedRunRoot, manifestPath, "QA runtime manifest");
|
||||
await writeJsonAtomic(manifestPath, stopped);
|
||||
};
|
||||
|
||||
const profileFlags = (profile: Profile) => ({
|
||||
demoMode: profile !== "onboarding-fresh",
|
||||
demoReadOnly: profile === "main-readonly",
|
||||
unsafeMockIntegration: true,
|
||||
});
|
||||
|
||||
const startWithLease = async (options: Options) => {
|
||||
const candidateSha = resolveCandidate(options.candidate);
|
||||
const checkoutSha = resolveCandidate("HEAD");
|
||||
if (checkoutSha !== candidateSha) {
|
||||
throw new Error(`The requested candidate ${candidateSha} does not match the checked-out HEAD ${checkoutSha}`);
|
||||
}
|
||||
const branch = currentBranch();
|
||||
|
||||
await mkdir(options.runRoot, { recursive: true, mode: 0o700 });
|
||||
await assertSafeRunRoot(options.runRoot);
|
||||
await chmod(options.runRoot, 0o700);
|
||||
await assertSafeRunRoot(options.runRoot);
|
||||
const markerPath = path.join(options.runRoot, markerName);
|
||||
await assertSafeContainedPath(options.runRoot, markerPath, "QA run-root marker");
|
||||
if (!existsSync(markerPath)) {
|
||||
await assertSafeContainedPath(options.runRoot, markerPath, "QA run-root marker");
|
||||
await writeJsonAtomic(markerPath, {
|
||||
schemaVersion: 1,
|
||||
repoRoot,
|
||||
runRoot: options.runRoot,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
} else {
|
||||
await assertRunRootMarker(options.runRoot);
|
||||
}
|
||||
|
||||
const slotsRoot = path.join(options.runRoot, "slots");
|
||||
await assertSafeContainedPath(options.runRoot, slotsRoot, "QA slots directory");
|
||||
await mkdir(slotsRoot, { recursive: true, mode: 0o700 });
|
||||
await assertSafeContainedPath(options.runRoot, slotsRoot, "QA slots directory");
|
||||
const slotDir = path.join(slotsRoot, String(options.slot));
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot directory");
|
||||
const manifestPath = path.join(slotDir, "runtime-manifest.json");
|
||||
if (existsSync(manifestPath)) {
|
||||
await assertSafeContainedPath(options.runRoot, manifestPath, "QA runtime manifest");
|
||||
const existing = await readManifest(manifestPath);
|
||||
if (!options.reset) throw new Error(`Slot ${options.slot} already has a manifest; pass --reset to replace it`);
|
||||
await stopFromManifest(manifestPath, existing, options.runRoot, options.slot);
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot reset target");
|
||||
await rm(slotDir, { recursive: true });
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot reset target");
|
||||
} else if (existsSync(slotDir)) {
|
||||
if (!options.reset) throw new Error(`Slot ${options.slot} has incomplete state; pass --reset to replace it`);
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot reset target");
|
||||
await rm(slotDir, { recursive: true });
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot reset target");
|
||||
}
|
||||
const nextDistDir = path.join(repoRoot, "apps/nextjs/.next-qa", `slot-${options.slot}`);
|
||||
if (options.reset) {
|
||||
await assertSafeContainedPath(path.join(repoRoot, "apps/nextjs"), nextDistDir, "QA Next cache reset target");
|
||||
await rm(nextDistDir, { recursive: true, force: true });
|
||||
await assertSafeContainedPath(path.join(repoRoot, "apps/nextjs"), nextDistDir, "QA Next cache reset target");
|
||||
}
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot creation target");
|
||||
await mkdir(slotDir, { mode: 0o700 });
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot directory");
|
||||
await chmod(slotDir, 0o700);
|
||||
await assertSafeContainedPath(options.runRoot, slotDir, "QA slot directory");
|
||||
|
||||
const runId = `release-v2-${candidateSha.slice(0, 8)}-s${options.slot}-${Date.now()}`;
|
||||
const redisContainer = `homarr-release-v2-qa-redis-s${options.slot}-${candidateSha.slice(0, 8)}-${Date.now().toString(36)}`;
|
||||
const dbPath = path.join(slotDir, "db.sqlite");
|
||||
const fixtureReadyPath = path.join(slotDir, "fixture-ready.json");
|
||||
const fixtureManifestPath = path.join(slotDir, "fixture-manifest.json");
|
||||
const fixtureLogPath = path.join(slotDir, "fixture.log");
|
||||
const appLogPath = path.join(slotDir, "app.log");
|
||||
const flags = profileFlags(options.profile);
|
||||
let fixturePid: number | undefined;
|
||||
let appPid: number | undefined;
|
||||
let appPortReservation: PortReservation | undefined;
|
||||
let redisCreated = false;
|
||||
|
||||
try {
|
||||
const redisRun = spawnSync(
|
||||
"docker",
|
||||
[
|
||||
"run",
|
||||
"--detach",
|
||||
"--name",
|
||||
redisContainer,
|
||||
"--label",
|
||||
`com.homarr.release-v2-qa.run=${runId}`,
|
||||
"--publish",
|
||||
"127.0.0.1::6379",
|
||||
"redis:latest",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (redisRun.status !== 0) throw new Error(`Unable to start QA Redis: ${redisRun.stderr.trim()}`);
|
||||
redisCreated = true;
|
||||
const redisPortOutput = commandOutput("docker", ["port", redisContainer, "6379/tcp"]);
|
||||
const redisPort = Number(redisPortOutput.split(":").at(-1));
|
||||
if (!Number.isInteger(redisPort)) throw new Error(`Unable to parse QA Redis port: ${redisPortOutput}`);
|
||||
|
||||
await assertSafeContainedPath(options.runRoot, fixtureReadyPath, "QA fixture readiness file");
|
||||
await assertSafeContainedPath(options.runRoot, fixtureLogPath, "QA fixture log");
|
||||
const fixtureProcess = spawnLogged(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/release-v2-qa/fixture-server.mts", "--port", "0", "--ready-file", fixtureReadyPath],
|
||||
fixtureLogPath,
|
||||
createFixtureEnvironment(runId),
|
||||
);
|
||||
fixturePid = fixtureProcess.pid;
|
||||
const fixtureReady = await waitForJsonFile<{ url: string; port: number }>(fixtureReadyPath, 30_000);
|
||||
await waitForUrl(`${fixtureReady.url}/health`, 30_000);
|
||||
appPortReservation = await reserveLoopbackPort();
|
||||
let appPort = appPortReservation.port;
|
||||
let url = `http://127.0.0.1:${appPort}`;
|
||||
let qaPassword = process.env.QA_PASSWORD;
|
||||
if (!qaPassword && process.env.QA_PASSWORD_FILE) {
|
||||
qaPassword = (await readFile(path.resolve(process.env.QA_PASSWORD_FILE), "utf8")).trim();
|
||||
}
|
||||
if (flags.demoMode && !qaPassword) {
|
||||
throw new Error("QA_PASSWORD or QA_PASSWORD_FILE is required for populated QA profiles");
|
||||
}
|
||||
|
||||
const environment: NodeJS.ProcessEnv = {
|
||||
...process.env,
|
||||
AUTH_SECRET: randomBytes(32).toString("hex"),
|
||||
AUTH_PROVIDERS: "credentials",
|
||||
AUTH_COOKIE_PREFIX: `homarr_qa_s${options.slot}`,
|
||||
DB_DRIVER: "better-sqlite3",
|
||||
DB_URL: dbPath,
|
||||
DEMO_MODE: String(flags.demoMode),
|
||||
DEMO_READ_ONLY: String(flags.demoReadOnly),
|
||||
UNSAFE_ENABLE_MOCK_INTEGRATION: String(flags.unsafeMockIntegration),
|
||||
SECRET_ENCRYPTION_KEY: randomBytes(32).toString("hex"),
|
||||
REDIS_IS_EXTERNAL: "true",
|
||||
REDIS_HOST: "127.0.0.1",
|
||||
REDIS_PORT: String(redisPort),
|
||||
REDIS_DATABASE_INDEX: String(options.slot),
|
||||
QA_CANDIDATE_SHA: candidateSha,
|
||||
QA_FIXTURE_URL: fixtureReady.url,
|
||||
QA_PROFILE: options.profile,
|
||||
QA_RUN_ID: runId,
|
||||
QA_RUN_ROOT: options.runRoot,
|
||||
QA_SLOT: String(options.slot),
|
||||
QA_PASSWORD: qaPassword,
|
||||
HOMARR_WEBSITE_URL: url,
|
||||
WORKSHOP_API_URL: fixtureReady.url,
|
||||
WORKSHOP_WEB_URL: `${fixtureReady.url}/iframe`,
|
||||
HOMARR_QA_NEXT_DIST_DIR: `.next-qa/slot-${options.slot}`,
|
||||
HOMARR_VERSION: candidateSha.slice(0, 12),
|
||||
NEXT_TELEMETRY_DISABLED: "1",
|
||||
};
|
||||
|
||||
await assertSafeContainedPath(options.runRoot, dbPath, "QA SQLite database");
|
||||
runForeground("pnpm", ["db:migration:sqlite:run"], environment);
|
||||
await assertSafeContainedPath(options.runRoot, dbPath, "QA SQLite database");
|
||||
await assertSafeContainedPath(options.runRoot, fixtureManifestPath, "QA fixture manifest");
|
||||
runForeground("pnpm", ["qa:release-v2:seed", "--", "--profile", options.profile, "--output", slotDir], environment);
|
||||
await assertSafeContainedPath(options.runRoot, fixtureManifestPath, "QA fixture manifest");
|
||||
|
||||
let appCommand = "pnpm";
|
||||
const appEnvironment = {
|
||||
...sanitizeAppEnvironment(environment),
|
||||
WATCHPACK_POLLING: String(watchpackPollingIntervalMs),
|
||||
};
|
||||
await assertSafeContainedPath(options.runRoot, appLogPath, "QA app log");
|
||||
const launchedApp = await launchWithPortRetry<{ pid: number; port: number; url: string }>(
|
||||
async (attemptPort) => {
|
||||
appPort = attemptPort;
|
||||
url = `http://127.0.0.1:${appPort}`;
|
||||
let appArguments = ["--filter", "@homarr/nextjs", "dev", "--hostname", "127.0.0.1", "--port", String(appPort)];
|
||||
if (process.platform === "linux" && existsSync("/usr/bin/prlimit")) {
|
||||
appCommand = "/usr/bin/prlimit";
|
||||
appArguments = ["--nofile=65536:1048576", "--", "pnpm", ...appArguments];
|
||||
} else {
|
||||
appCommand = "pnpm";
|
||||
}
|
||||
const logOffset = await stat(appLogPath)
|
||||
.then((stats) => stats.size)
|
||||
.catch(() => 0);
|
||||
await assertSafeContainedPath(options.runRoot, appLogPath, "QA app log");
|
||||
const spawned = spawnLogged(appCommand, appArguments, appLogPath, {
|
||||
...appEnvironment,
|
||||
HOMARR_WEBSITE_URL: url,
|
||||
});
|
||||
appPid = spawned.pid;
|
||||
const outcome = await waitForAppAttempt(spawned, url, appLogPath, logOffset, 300_000);
|
||||
if (outcome === "address-in-use") {
|
||||
appPid = undefined;
|
||||
return { status: "address-in-use" } as const;
|
||||
}
|
||||
return { status: "ready", value: { pid: spawned.pid, port: appPort, url } } as const;
|
||||
},
|
||||
{ initialReservation: appPortReservation },
|
||||
);
|
||||
appPortReservation = undefined;
|
||||
appPid = launchedApp.pid;
|
||||
appPort = launchedApp.port;
|
||||
url = launchedApp.url;
|
||||
|
||||
const manifest: RuntimeManifest = {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
status: "running",
|
||||
candidateSha,
|
||||
branch,
|
||||
slot: options.slot,
|
||||
profile: options.profile,
|
||||
startedAt: new Date().toISOString(),
|
||||
repoRoot,
|
||||
runRoot: options.runRoot,
|
||||
slotDir,
|
||||
dbPath,
|
||||
fixtureManifestPath,
|
||||
nextDistDir,
|
||||
url,
|
||||
fixtureUrl: fixtureReady.url,
|
||||
ports: { app: appPort, fixture: fixtureReady.port, redis: redisPort },
|
||||
flags,
|
||||
watcher: { watchpackPollingIntervalMs },
|
||||
processes: {
|
||||
app: { pid: appPid, logPath: appLogPath },
|
||||
fixture: { pid: fixturePid, logPath: fixtureLogPath },
|
||||
},
|
||||
docker: { redisContainer },
|
||||
};
|
||||
await writeJsonAtomic(manifestPath, manifest);
|
||||
console.log(JSON.stringify(manifest, null, 2));
|
||||
} catch (error) {
|
||||
if (appPortReservation) await appPortReservation.release();
|
||||
if (appPid) await stopProcess(appPid, runId);
|
||||
if (fixturePid) await stopProcess(fixturePid, runId);
|
||||
if (redisCreated) removeRedisContainer(redisContainer, runId);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const start = async (options: Options) => {
|
||||
options.runRoot = await assertSafeRunRoot(options.runRoot, { allowMissing: true });
|
||||
const lease = await acquireSlotLease(options.runRoot, options.slot, `start:${options.profile}`);
|
||||
try {
|
||||
await startWithLease(options);
|
||||
} finally {
|
||||
await lease.release();
|
||||
}
|
||||
};
|
||||
|
||||
const status = async (options: Options) => {
|
||||
options.runRoot = await assertSafeRunRoot(options.runRoot);
|
||||
await assertRunRootMarker(options.runRoot);
|
||||
const manifestPath = path.join(options.runRoot, "slots", String(options.slot), "runtime-manifest.json");
|
||||
await assertSafeContainedPath(options.runRoot, manifestPath, "QA runtime manifest");
|
||||
const manifest = await readManifest(manifestPath);
|
||||
const appReachable = await fetch(`${manifest.url}/api/health/live`, { signal: AbortSignal.timeout(5_000) })
|
||||
.then((response) => response.ok)
|
||||
.catch(() => false);
|
||||
const fixtureReachable = await fetch(`${manifest.fixtureUrl}/health`, { signal: AbortSignal.timeout(5_000) })
|
||||
.then((response) => response.ok)
|
||||
.catch(() => false);
|
||||
const redisRunning =
|
||||
spawnSync("docker", ["inspect", "--format", "{{.State.Running}}", manifest.docker.redisContainer], {
|
||||
encoding: "utf8",
|
||||
}).stdout.trim() === "true";
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
...manifest,
|
||||
observed: {
|
||||
appProcess: await processBelongsToRun(manifest.processes.app.pid, manifest.runId),
|
||||
fixtureProcess: await processBelongsToRun(manifest.processes.fixture.pid, manifest.runId),
|
||||
appReachable,
|
||||
fixtureReachable,
|
||||
redisRunning,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
const stopWithLease = async (options: Options) => {
|
||||
const markerPath = path.join(options.runRoot, markerName);
|
||||
await assertSafeContainedPath(options.runRoot, markerPath, "QA run-root marker");
|
||||
if (!existsSync(markerPath)) throw new Error(`QA run-root marker is missing: ${markerPath}`);
|
||||
await assertRunRootMarker(options.runRoot);
|
||||
const manifestPath = path.join(options.runRoot, "slots", String(options.slot), "runtime-manifest.json");
|
||||
await assertSafeContainedPath(options.runRoot, manifestPath, "QA runtime manifest");
|
||||
const manifest = await readManifest(manifestPath);
|
||||
await stopFromManifest(manifestPath, manifest, options.runRoot, options.slot);
|
||||
console.log(`Stopped release-v2 QA slot ${options.slot}`);
|
||||
};
|
||||
|
||||
const stop = async (options: Options) => {
|
||||
options.runRoot = await assertSafeRunRoot(options.runRoot);
|
||||
const lease = await acquireSlotLease(options.runRoot, options.slot, "stop");
|
||||
try {
|
||||
await stopWithLease(options);
|
||||
} finally {
|
||||
await lease.release();
|
||||
}
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const options = parseArgs();
|
||||
assertRuntimeCommandPlatform(options.command);
|
||||
if (options.command === "start") await start(options);
|
||||
else if (options.command === "status") await status(options);
|
||||
else await stop(options);
|
||||
};
|
||||
|
||||
const isDirectExecution = process.argv[1]
|
||||
? pathToFileURL(path.resolve(process.argv[1])).href === pathToFileURL(fileURLToPath(import.meta.url)).href
|
||||
: false;
|
||||
if (isDirectExecution) await main();
|
||||
@@ -0,0 +1,81 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { createServer } from "node:net";
|
||||
import type { Server } from "node:net";
|
||||
import { afterEach, describe, expect, test } from "vitest";
|
||||
|
||||
import { assertRuntimeCommandPlatform, launchWithPortRetry, reserveLoopbackPort } from "./runtime.mts";
|
||||
|
||||
const openServers: Server[] = [];
|
||||
|
||||
const listen = async (server: Server, port: number) =>
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(port, "127.0.0.1", resolve);
|
||||
});
|
||||
|
||||
const close = async (server: Server) =>
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) reject(error);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
openServers.splice(0).map(async (server) => {
|
||||
if (server.listening) await close(server);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
describe("release-v2 QA runtime port startup", () => {
|
||||
test("keeps an allocated loopback port reserved until startup is ready to bind", async () => {
|
||||
const reservation = await reserveLoopbackPort();
|
||||
const contender = createServer();
|
||||
openServers.push(contender);
|
||||
|
||||
await expect(listen(contender, reservation.port)).rejects.toMatchObject({ code: "EADDRINUSE" });
|
||||
await reservation.release();
|
||||
await listen(contender, reservation.port);
|
||||
});
|
||||
|
||||
test("retries with a newly reserved port after an address-in-use collision", async () => {
|
||||
let attempts = 0;
|
||||
const selectedPort = await launchWithPortRetry(async (port) => {
|
||||
attempts += 1;
|
||||
const server = createServer();
|
||||
openServers.push(server);
|
||||
await listen(server, port);
|
||||
|
||||
if (attempts === 1) {
|
||||
const collidedProcess = createServer();
|
||||
openServers.push(collidedProcess);
|
||||
await expect(listen(collidedProcess, port)).rejects.toMatchObject({ code: "EADDRINUSE" });
|
||||
await close(server);
|
||||
return { status: "address-in-use" } as const;
|
||||
}
|
||||
|
||||
await close(server);
|
||||
return { status: "ready", value: port } as const;
|
||||
});
|
||||
|
||||
expect(attempts).toBe(2);
|
||||
expect(selectedPort).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("release-v2 QA runtime platform boundary", () => {
|
||||
test.each(["start", "stop"] as const)("rejects %s before dispatch on an unsupported platform", (command) => {
|
||||
expect(() => assertRuntimeCommandPlatform(command, "darwin")).toThrow(/Linux/u);
|
||||
});
|
||||
|
||||
test("allows read-only status on an unsupported platform", () => {
|
||||
expect(() => assertRuntimeCommandPlatform("status", "win32")).not.toThrow();
|
||||
});
|
||||
|
||||
test.each(["start", "status", "stop"] as const)("allows %s on Linux", (command) => {
|
||||
expect(() => assertRuntimeCommandPlatform(command, "linux")).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,349 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { lstat, open, readFile, realpath, unlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const runRootNamePattern = /^homarr-release-v2-qa(?:$|-[A-Za-z0-9][A-Za-z0-9._-]*)$/u;
|
||||
const zoomOutputNamePattern = /^homarr-release-v2-qa-zoom-[A-Za-z0-9][A-Za-z0-9._-]*$/u;
|
||||
const ipv6LoopbackHosts = new Set(["::1", "0:0:0:0:0:0:0:1"]);
|
||||
const leaseSchemaVersion = 1;
|
||||
const incompleteLeaseGraceMs = 30_000;
|
||||
|
||||
const missingPath = (error: unknown) => (error as NodeJS.ErrnoException).code === "ENOENT";
|
||||
const existingPath = (error: unknown) => (error as NodeJS.ErrnoException).code === "EEXIST";
|
||||
|
||||
export const assertSupportedRuntimePlatform = (platform: NodeJS.Platform = process.platform) => {
|
||||
if (platform !== "linux") {
|
||||
throw new Error(
|
||||
"Release-v2 QA runtime mutations are supported only on Linux because safe process ownership and cleanup require procfs and POSIX process groups",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
export const isPathWithin = (parent: string, child: string) => {
|
||||
if (!path.isAbsolute(parent) || !path.isAbsolute(child)) return false;
|
||||
const relativePath = path.relative(path.resolve(parent), path.resolve(child));
|
||||
return relativePath.length === 0 || (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
|
||||
};
|
||||
|
||||
const canonicalTemporaryParent = async () => await realpath(path.resolve(tmpdir()));
|
||||
|
||||
const assertSafeTemporaryChild = async (
|
||||
target: string,
|
||||
options: {
|
||||
allowMissing?: boolean;
|
||||
label: string;
|
||||
namePattern: RegExp;
|
||||
nameRequirement: string;
|
||||
},
|
||||
) => {
|
||||
const resolvedTarget = path.resolve(target);
|
||||
const canonicalParent = await canonicalTemporaryParent();
|
||||
const targetParent = path.dirname(resolvedTarget);
|
||||
let realTargetParent: string;
|
||||
try {
|
||||
realTargetParent = await realpath(targetParent);
|
||||
} catch (error) {
|
||||
if (missingPath(error)) {
|
||||
throw new Error(`${options.label} must be a direct child of the operating system temporary directory`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (realTargetParent !== canonicalParent) {
|
||||
throw new Error(`${options.label} must be a direct child of the operating system temporary directory`);
|
||||
}
|
||||
|
||||
const targetName = path.basename(resolvedTarget);
|
||||
if (!options.namePattern.test(targetName)) {
|
||||
throw new Error(options.nameRequirement);
|
||||
}
|
||||
|
||||
const canonicalTarget = path.join(canonicalParent, targetName);
|
||||
try {
|
||||
const stats = await lstat(canonicalTarget);
|
||||
if (stats.isSymbolicLink()) throw new Error(`${options.label} must not be a symlink`);
|
||||
if (!stats.isDirectory()) throw new Error(`${options.label} must be a directory`);
|
||||
if ((await realpath(canonicalTarget)) !== canonicalTarget) {
|
||||
throw new Error(`${options.label} must not contain symlink components`);
|
||||
}
|
||||
} catch (error) {
|
||||
if (missingPath(error) && options.allowMissing) return canonicalTarget;
|
||||
throw error;
|
||||
}
|
||||
|
||||
return canonicalTarget;
|
||||
};
|
||||
|
||||
export const assertSafeContainedPath = async (parent: string, child: string, label: string) => {
|
||||
const resolvedParent = path.resolve(parent);
|
||||
const resolvedChild = path.resolve(child);
|
||||
if (!isPathWithin(resolvedParent, resolvedChild)) {
|
||||
throw new Error(`${label} escapes its approved parent`);
|
||||
}
|
||||
|
||||
const realParent = await realpath(resolvedParent);
|
||||
if (realParent !== resolvedParent) {
|
||||
throw new Error(`${label} parent contains a symlink`);
|
||||
}
|
||||
|
||||
const relativePath = path.relative(resolvedParent, resolvedChild);
|
||||
let currentPath = resolvedParent;
|
||||
for (const segment of relativePath.split(path.sep).filter(Boolean)) {
|
||||
currentPath = path.join(currentPath, segment);
|
||||
try {
|
||||
const stats = await lstat(currentPath);
|
||||
if (stats.isSymbolicLink()) throw new Error(`${label} contains a symlink`);
|
||||
} catch (error) {
|
||||
if (missingPath(error)) break;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return resolvedChild;
|
||||
};
|
||||
|
||||
export const assertSafeRunRoot = async (runRoot: string, options: { allowMissing?: boolean } = {}) =>
|
||||
await assertSafeTemporaryChild(runRoot, {
|
||||
allowMissing: options.allowMissing,
|
||||
label: "The QA run-root",
|
||||
namePattern: runRootNamePattern,
|
||||
nameRequirement:
|
||||
'The QA run-root basename must start with "homarr-release-v2-qa" and use only safe suffix characters',
|
||||
});
|
||||
|
||||
export const assertSafeZoomOutput = async (outputDirectory: string, options: { allowMissing?: boolean } = {}) =>
|
||||
await assertSafeTemporaryChild(outputDirectory, {
|
||||
allowMissing: options.allowMissing,
|
||||
label: "The zoom extension output",
|
||||
namePattern: zoomOutputNamePattern,
|
||||
nameRequirement:
|
||||
'The zoom extension output basename must start with "homarr-release-v2-qa-zoom-" and use only safe suffix characters',
|
||||
});
|
||||
|
||||
const readLinuxProcessStartMarker = async (pid: number) => {
|
||||
if (process.platform !== "linux") return null;
|
||||
try {
|
||||
const processStat = await readFile(`/proc/${pid}/stat`, "utf8");
|
||||
const commandEnd = processStat.lastIndexOf(")");
|
||||
if (commandEnd < 0) return null;
|
||||
const fieldsAfterCommand = processStat
|
||||
.slice(commandEnd + 2)
|
||||
.trim()
|
||||
.split(/\s+/u);
|
||||
const startTime = fieldsAfterCommand[19];
|
||||
return startTime ? `linux:${startTime}` : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const isProcessRunning = (pid: number) => {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
interface SlotLeaseOwner {
|
||||
schemaVersion: 1;
|
||||
pid: number;
|
||||
processStartMarker: string | null;
|
||||
token: string;
|
||||
runRoot: string;
|
||||
slot: number;
|
||||
operation: string;
|
||||
acquiredAt: string;
|
||||
}
|
||||
|
||||
interface LeaseSnapshot {
|
||||
device: bigint;
|
||||
inode: bigint;
|
||||
modifiedMs: number;
|
||||
content: string;
|
||||
owner: SlotLeaseOwner | null;
|
||||
}
|
||||
|
||||
export interface SlotLease {
|
||||
path: string;
|
||||
owner: SlotLeaseOwner;
|
||||
release: () => Promise<void>;
|
||||
}
|
||||
|
||||
export const getSlotLeasePath = async (runRoot: string, slot: number) => {
|
||||
const canonicalRunRoot = await assertSafeRunRoot(runRoot, { allowMissing: true });
|
||||
const canonicalParent = await canonicalTemporaryParent();
|
||||
return path.join(canonicalParent, `.${path.basename(canonicalRunRoot)}.slot-${slot}.lease`);
|
||||
};
|
||||
|
||||
const readLeaseSnapshot = async (leasePath: string): Promise<LeaseSnapshot> => {
|
||||
const stats = await lstat(leasePath, { bigint: true });
|
||||
if (stats.isSymbolicLink() || !stats.isFile()) {
|
||||
throw new Error(`Refusing unsafe QA slot lease path: ${leasePath}`);
|
||||
}
|
||||
const content = await readFile(leasePath, "utf8");
|
||||
let owner: SlotLeaseOwner | null = null;
|
||||
try {
|
||||
owner = JSON.parse(content) as SlotLeaseOwner;
|
||||
} catch {
|
||||
// An acquiring process may have created the file but not completed its marker write yet.
|
||||
}
|
||||
return {
|
||||
device: stats.dev,
|
||||
inode: stats.ino,
|
||||
modifiedMs: Number(stats.mtimeMs),
|
||||
content,
|
||||
owner,
|
||||
};
|
||||
};
|
||||
|
||||
const isValidLeaseOwner = (owner: SlotLeaseOwner | null, runRoot: string, slot: number): owner is SlotLeaseOwner =>
|
||||
owner?.schemaVersion === leaseSchemaVersion &&
|
||||
Number.isInteger(owner.pid) &&
|
||||
owner.pid > 0 &&
|
||||
typeof owner.token === "string" &&
|
||||
owner.token.length > 0 &&
|
||||
path.resolve(owner.runRoot) === runRoot &&
|
||||
owner.slot === slot;
|
||||
|
||||
const processMatchesLeaseOwner = async (owner: SlotLeaseOwner) => {
|
||||
if (!isProcessRunning(owner.pid)) return false;
|
||||
const observedStartMarker = await readLinuxProcessStartMarker(owner.pid);
|
||||
if (owner.processStartMarker && observedStartMarker) {
|
||||
return owner.processStartMarker === observedStartMarker;
|
||||
}
|
||||
// If the platform cannot verify process start identity, preserve the live lock rather than
|
||||
// risking recovery over an active owner.
|
||||
return true;
|
||||
};
|
||||
|
||||
const sameLeaseSnapshot = (left: LeaseSnapshot, right: LeaseSnapshot) =>
|
||||
left.device === right.device &&
|
||||
left.inode === right.inode &&
|
||||
left.modifiedMs === right.modifiedMs &&
|
||||
left.content === right.content;
|
||||
|
||||
const recoverStaleLease = async (leasePath: string, snapshot: LeaseSnapshot) => {
|
||||
const confirmation = await readLeaseSnapshot(leasePath);
|
||||
if (!sameLeaseSnapshot(snapshot, confirmation)) {
|
||||
throw new Error(`QA slot lease changed during stale recovery: ${leasePath}`);
|
||||
}
|
||||
await unlink(leasePath);
|
||||
};
|
||||
|
||||
export const acquireSlotLease = async (
|
||||
runRoot: string,
|
||||
slot: number,
|
||||
operation: string,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): Promise<SlotLease> => {
|
||||
assertSupportedRuntimePlatform(platform);
|
||||
if (![1, 2, 3].includes(slot)) throw new Error("QA slot lease requires slot 1, 2, or 3");
|
||||
const canonicalRunRoot = await assertSafeRunRoot(runRoot, { allowMissing: true });
|
||||
const leasePath = await getSlotLeasePath(canonicalRunRoot, slot);
|
||||
const owner: SlotLeaseOwner = {
|
||||
schemaVersion: leaseSchemaVersion,
|
||||
pid: process.pid,
|
||||
processStartMarker: await readLinuxProcessStartMarker(process.pid),
|
||||
token: randomUUID(),
|
||||
runRoot: canonicalRunRoot,
|
||||
slot,
|
||||
operation,
|
||||
acquiredAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
try {
|
||||
const handle = await open(leasePath, "wx", 0o600);
|
||||
try {
|
||||
await handle.writeFile(`${JSON.stringify(owner)}\n`, "utf8");
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
|
||||
return {
|
||||
path: leasePath,
|
||||
owner,
|
||||
release: async () => {
|
||||
const snapshot = await readLeaseSnapshot(leasePath);
|
||||
if (!isValidLeaseOwner(snapshot.owner, canonicalRunRoot, slot) || snapshot.owner?.token !== owner.token) {
|
||||
throw new Error(`Refusing to release a QA slot lease owned by another process: ${leasePath}`);
|
||||
}
|
||||
await recoverStaleLease(leasePath, snapshot);
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (!existingPath(error)) throw error;
|
||||
const snapshot = await readLeaseSnapshot(leasePath);
|
||||
const existingOwner = snapshot.owner;
|
||||
if (!isValidLeaseOwner(existingOwner, canonicalRunRoot, slot)) {
|
||||
if (Date.now() - snapshot.modifiedMs < incompleteLeaseGraceMs) {
|
||||
throw new Error(`QA slot ${slot} lease is being initialized by another operation`, { cause: error });
|
||||
}
|
||||
await recoverStaleLease(leasePath, snapshot);
|
||||
continue;
|
||||
}
|
||||
if (await processMatchesLeaseOwner(existingOwner)) {
|
||||
throw new Error(`QA slot ${slot} is busy with ${existingOwner.operation} owned by PID ${existingOwner.pid}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
await recoverStaleLease(leasePath, snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Unable to acquire QA slot ${slot} lease after stale recovery`);
|
||||
};
|
||||
|
||||
const normalizedHost = (host: string) => {
|
||||
const lowerHost = host.toLowerCase();
|
||||
if (lowerHost.startsWith("[") && lowerHost.endsWith("]")) return lowerHost.slice(1, -1);
|
||||
return lowerHost;
|
||||
};
|
||||
|
||||
export const validateLoopbackHost = (host: string) => {
|
||||
const normalized = normalizedHost(host);
|
||||
if (normalized !== "127.0.0.1" && normalized !== "localhost" && !ipv6LoopbackHosts.has(normalized)) {
|
||||
throw new Error("The fixture host must be an explicit loopback address");
|
||||
}
|
||||
return normalized;
|
||||
};
|
||||
|
||||
export const validateFixtureUrl = (value: string) => {
|
||||
let fixtureUrl: URL;
|
||||
try {
|
||||
fixtureUrl = new URL(value);
|
||||
} catch {
|
||||
throw new Error("The fixture URL must be a valid loopback HTTP origin");
|
||||
}
|
||||
|
||||
if (
|
||||
fixtureUrl.protocol !== "http:" ||
|
||||
fixtureUrl.username ||
|
||||
fixtureUrl.password ||
|
||||
fixtureUrl.pathname !== "/" ||
|
||||
fixtureUrl.search ||
|
||||
fixtureUrl.hash
|
||||
) {
|
||||
throw new Error("The fixture URL must be a credential-free loopback HTTP origin");
|
||||
}
|
||||
try {
|
||||
validateLoopbackHost(fixtureUrl.hostname);
|
||||
} catch {
|
||||
throw new Error("The fixture URL must be a credential-free loopback HTTP origin");
|
||||
}
|
||||
return fixtureUrl.origin;
|
||||
};
|
||||
|
||||
export const sanitizeAppEnvironment = (environment: NodeJS.ProcessEnv) => {
|
||||
const sanitized = { ...environment };
|
||||
delete sanitized.QA_PASSWORD;
|
||||
delete sanitized.QA_PASSWORD_FILE;
|
||||
return sanitized;
|
||||
};
|
||||
|
||||
export const createFixtureEnvironment = (runId: string): NodeJS.ProcessEnv => ({ QA_RUN_ID: runId });
|
||||
@@ -0,0 +1,233 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { mkdir, realpath, rm, stat, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, test } from "vitest";
|
||||
|
||||
import {
|
||||
acquireSlotLease,
|
||||
assertSafeContainedPath,
|
||||
assertSafeRunRoot,
|
||||
assertSupportedRuntimePlatform,
|
||||
createFixtureEnvironment,
|
||||
getSlotLeasePath,
|
||||
isPathWithin,
|
||||
sanitizeAppEnvironment,
|
||||
validateFixtureUrl,
|
||||
validateLoopbackHost,
|
||||
} from "./safety.mts";
|
||||
|
||||
const temporaryPaths: string[] = [];
|
||||
const uniqueName = (suffix: string) => `homarr-release-v2-qa-test-${process.pid}-${Date.now()}-${suffix}`;
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temporaryPaths.splice(0).map((target) => rm(target, { force: true, recursive: true })));
|
||||
});
|
||||
|
||||
describe("release-v2 QA harness safety", () => {
|
||||
test("supports runtime mutations only on Linux", () => {
|
||||
expect(() => assertSupportedRuntimePlatform("linux")).not.toThrow();
|
||||
expect(() => assertSupportedRuntimePlatform("darwin")).toThrow(/Linux/u);
|
||||
expect(() => assertSupportedRuntimePlatform("win32")).toThrow(/Linux/u);
|
||||
});
|
||||
|
||||
test("rejects an unsupported platform before creating a slot lease", async () => {
|
||||
const runRoot = path.join(tmpdir(), uniqueName("unsupported-platform"));
|
||||
const leasePath = await getSlotLeasePath(runRoot, 1);
|
||||
temporaryPaths.push(leasePath);
|
||||
|
||||
await expect(acquireSlotLease(runRoot, 1, "unsupported-start", "darwin")).rejects.toThrow(/Linux/u);
|
||||
await expect(stat(leasePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
test("accepts only direct, predictably named children of the temporary directory", async () => {
|
||||
const safeRoot = path.join(tmpdir(), uniqueName("safe"));
|
||||
temporaryPaths.push(safeRoot);
|
||||
|
||||
await expect(assertSafeRunRoot(safeRoot, { allowMissing: true })).resolves.toBe(path.resolve(safeRoot));
|
||||
await expect(
|
||||
assertSafeRunRoot(path.join(tmpdir(), "unrelated-homarr-release-v2-qa"), { allowMissing: true }),
|
||||
).rejects.toThrow("must start with");
|
||||
await expect(
|
||||
assertSafeRunRoot(path.join(tmpdir(), "homarr-release-v2-qaevil"), { allowMissing: true }),
|
||||
).rejects.toThrow("must start with");
|
||||
await expect(
|
||||
assertSafeRunRoot(path.join(tmpdir(), "nested", uniqueName("nested")), { allowMissing: true }),
|
||||
).rejects.toThrow("direct child");
|
||||
await expect(
|
||||
assertSafeRunRoot(path.join(tmpdir(), "..", "var", "tmp", uniqueName("traversal")), { allowMissing: true }),
|
||||
).rejects.toThrow("direct child");
|
||||
});
|
||||
|
||||
test("canonicalizes a temporary-directory alias without widening containment", async () => {
|
||||
const temporaryAlias = path.join(tmpdir(), uniqueName("temporary-alias"));
|
||||
const runRootName = uniqueName("canonical-root");
|
||||
temporaryPaths.push(temporaryAlias);
|
||||
await symlink(tmpdir(), temporaryAlias, "dir");
|
||||
|
||||
await expect(assertSafeRunRoot(path.join(temporaryAlias, runRootName), { allowMissing: true })).resolves.toBe(
|
||||
path.join(await realpath(tmpdir()), runRootName),
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects a symlink run-root", async () => {
|
||||
const target = path.join(tmpdir(), uniqueName("target"));
|
||||
const linkedRoot = path.join(tmpdir(), uniqueName("linked"));
|
||||
temporaryPaths.push(linkedRoot, target);
|
||||
await mkdir(target);
|
||||
await symlink(target, linkedRoot);
|
||||
|
||||
await expect(assertSafeRunRoot(linkedRoot)).rejects.toThrow("symlink");
|
||||
});
|
||||
|
||||
test("rejects symlink components in a contained reset target", async () => {
|
||||
const runRoot = path.join(tmpdir(), uniqueName("component-root"));
|
||||
const externalTarget = path.join(tmpdir(), uniqueName("component-target"));
|
||||
temporaryPaths.push(runRoot, externalTarget);
|
||||
await mkdir(runRoot);
|
||||
await mkdir(externalTarget);
|
||||
await symlink(externalTarget, path.join(runRoot, "slots"));
|
||||
|
||||
await expect(
|
||||
assertSafeContainedPath(runRoot, path.join(runRoot, "slots", "1"), "QA slot reset target"),
|
||||
).rejects.toThrow("symlink");
|
||||
});
|
||||
|
||||
test("catches a symlink swapped in after an earlier missing-target check", async () => {
|
||||
const runRoot = path.join(tmpdir(), uniqueName("swap-root"));
|
||||
const externalTarget = path.join(tmpdir(), uniqueName("swap-target"));
|
||||
const slotPath = path.join(runRoot, "slots", "1");
|
||||
temporaryPaths.push(runRoot, externalTarget);
|
||||
await mkdir(path.dirname(slotPath), { recursive: true });
|
||||
await mkdir(externalTarget);
|
||||
|
||||
await expect(assertSafeContainedPath(runRoot, slotPath, "QA slot mutation target")).resolves.toBe(slotPath);
|
||||
await symlink(externalTarget, slotPath);
|
||||
await expect(assertSafeContainedPath(runRoot, slotPath, "QA slot mutation target")).rejects.toThrow("symlink");
|
||||
});
|
||||
|
||||
test("rejects a concurrent operation while a live slot lease is held", async () => {
|
||||
const runRoot = path.join(tmpdir(), uniqueName("live-lease"));
|
||||
const lease = await acquireSlotLease(runRoot, 1, "first-start");
|
||||
temporaryPaths.push(lease.path);
|
||||
|
||||
await expect(acquireSlotLease(runRoot, 1, "second-start")).rejects.toThrow(/slot 1 is busy.*first-start/u);
|
||||
await lease.release();
|
||||
});
|
||||
|
||||
test("recovers a stale lease with a dead owner PID", async () => {
|
||||
const runRoot = path.join(tmpdir(), uniqueName("stale-lease"));
|
||||
const leasePath = await getSlotLeasePath(runRoot, 2);
|
||||
temporaryPaths.push(leasePath);
|
||||
await writeFile(
|
||||
leasePath,
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
pid: 2_147_483_647,
|
||||
processStartMarker: "linux:stale",
|
||||
token: "stale-token",
|
||||
runRoot: path.resolve(runRoot),
|
||||
slot: 2,
|
||||
operation: "stale-start",
|
||||
acquiredAt: "2026-01-01T00:00:00.000Z",
|
||||
})}\n`,
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
|
||||
const lease = await acquireSlotLease(runRoot, 2, "replacement-start");
|
||||
expect(lease.owner.operation).toBe("replacement-start");
|
||||
await lease.release();
|
||||
});
|
||||
|
||||
test("recovers a PID-reuse lease marker without signaling the unrelated process", async () => {
|
||||
const runRoot = path.join(tmpdir(), uniqueName("pid-reuse-lease"));
|
||||
const leasePath = await getSlotLeasePath(runRoot, 3);
|
||||
temporaryPaths.push(leasePath);
|
||||
await writeFile(
|
||||
leasePath,
|
||||
`${JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
pid: process.pid,
|
||||
processStartMarker: "linux:not-this-process",
|
||||
token: "reused-pid-token",
|
||||
runRoot: path.resolve(runRoot),
|
||||
slot: 3,
|
||||
operation: "abandoned-start",
|
||||
acquiredAt: "2026-01-01T00:00:00.000Z",
|
||||
})}\n`,
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
|
||||
const lease = await acquireSlotLease(runRoot, 3, "safe-recovery");
|
||||
expect(process.pid).toBeGreaterThan(0);
|
||||
await lease.release();
|
||||
});
|
||||
|
||||
test("uses path segments rather than string prefixes for containment", () => {
|
||||
const parent = path.join(tmpdir(), "qa-artifacts", "packet-1");
|
||||
expect(isPathWithin(parent, path.join(parent, "screen.png"))).toBe(true);
|
||||
expect(isPathWithin(parent, `${parent}-spoof/screen.png`)).toBe(false);
|
||||
expect(isPathWithin(parent, path.join(parent, "..", "packet-2", "screen.png"))).toBe(false);
|
||||
expect(isPathWithin(parent, ".screenshots/release-v2/packet-1/screen.png")).toBe(false);
|
||||
});
|
||||
|
||||
test.each(["http://127.0.0.1:3210", "http://localhost:3210", "http://[::1]:3210", "http://[0:0:0:0:0:0:0:1]:3210"])(
|
||||
"accepts a credential-free loopback fixture origin: %s",
|
||||
(fixtureUrl) => {
|
||||
expect(validateFixtureUrl(fixtureUrl)).toBe(new URL(fixtureUrl).origin);
|
||||
},
|
||||
);
|
||||
|
||||
test.each([
|
||||
"https://127.0.0.1:3210",
|
||||
"http://user:secret@127.0.0.1:3210",
|
||||
"http://0.0.0.0:3210",
|
||||
"http://192.0.2.1:3210",
|
||||
"http://localhost.example:3210",
|
||||
"http://127.0.0.1:3210/path",
|
||||
])("rejects an unsafe fixture URL without echoing it: %s", (fixtureUrl) => {
|
||||
expect(() => validateFixtureUrl(fixtureUrl)).toThrow("fixture URL");
|
||||
});
|
||||
|
||||
test.each(["127.0.0.1", "localhost", "::1", "[::1]", "0:0:0:0:0:0:0:1"])(
|
||||
"accepts the loopback fixture bind host %s",
|
||||
(host) => {
|
||||
expect(validateLoopbackHost(host)).toBeTruthy();
|
||||
},
|
||||
);
|
||||
|
||||
test.each(["0.0.0.0", "::", "192.0.2.1", "example.test"])("rejects the non-loopback bind host %s", (host) => {
|
||||
expect(() => validateLoopbackHost(host)).toThrow("loopback");
|
||||
});
|
||||
|
||||
test("removes both QA password inputs from the spawned app environment", () => {
|
||||
const environment = sanitizeAppEnvironment({
|
||||
QA_PASSWORD: "do-not-forward",
|
||||
QA_PASSWORD_FILE: "/tmp/do-not-forward",
|
||||
QA_RUN_ID: "qa-run",
|
||||
});
|
||||
|
||||
expect(environment).toEqual({ QA_RUN_ID: "qa-run" });
|
||||
expect("QA_PASSWORD" in environment).toBe(false);
|
||||
expect("QA_PASSWORD_FILE" in environment).toBe(false);
|
||||
});
|
||||
|
||||
test("uses an explicit fixture environment allowlist", () => {
|
||||
const environment = createFixtureEnvironment("qa-run");
|
||||
|
||||
expect(environment).toEqual({ QA_RUN_ID: "qa-run" });
|
||||
for (const forbiddenName of [
|
||||
"QA_PASSWORD",
|
||||
"QA_PASSWORD_FILE",
|
||||
"DB_URL",
|
||||
"AUTH_SECRET",
|
||||
"API_KEY",
|
||||
"SECRET_ENCRYPTION_KEY",
|
||||
"HOME",
|
||||
"PATH",
|
||||
]) {
|
||||
expect(forbiddenName in environment).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,172 @@
|
||||
import { lstat, mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
import { assertSafeZoomOutput } from "./safety.mts";
|
||||
|
||||
const allowedFactorStrings = new Set(["0.8", "1", "1.25", "2"]);
|
||||
|
||||
interface ZoomOptions {
|
||||
origin: string;
|
||||
factor: number;
|
||||
outputDirectory: string;
|
||||
}
|
||||
|
||||
const usage = () => {
|
||||
console.log("Usage: zoom.mts --origin <http(s)-origin> --factor <0.8|1|1.25|2> --output <new-absolute-directory>");
|
||||
};
|
||||
|
||||
const validateOrigin = (value: string) => {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch {
|
||||
throw new Error("--origin must be a valid HTTP(S) origin");
|
||||
}
|
||||
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
||||
throw new Error("--origin must use http: or https:");
|
||||
}
|
||||
if (url.username || url.password) {
|
||||
throw new Error("--origin must not contain credentials");
|
||||
}
|
||||
if (url.pathname !== "/" || url.search || url.hash) {
|
||||
throw new Error("--origin must not contain a path, query, or fragment");
|
||||
}
|
||||
|
||||
return url.origin;
|
||||
};
|
||||
|
||||
const validateOutputDirectory = async (value: string) => {
|
||||
if (!path.isAbsolute(value)) {
|
||||
throw new Error("--output must be an absolute path to a new directory");
|
||||
}
|
||||
|
||||
const outputDirectory = await assertSafeZoomOutput(value, { allowMissing: true });
|
||||
|
||||
let outputExists = true;
|
||||
try {
|
||||
await lstat(outputDirectory);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
outputExists = false;
|
||||
}
|
||||
if (outputExists) throw new Error(`--output must not already exist: ${outputDirectory}`);
|
||||
return outputDirectory;
|
||||
};
|
||||
|
||||
export const parseZoomOptions = async (rawArguments: string[]): Promise<ZoomOptions> => {
|
||||
const args = rawArguments.filter((argument) => argument !== "--");
|
||||
let origin: string | undefined;
|
||||
let factorString: string | undefined;
|
||||
let output: string | undefined;
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
const value = args[index + 1];
|
||||
if (argument === "--origin" && value) {
|
||||
origin = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--factor" && value) {
|
||||
factorString = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument === "--output" && value) {
|
||||
output = value;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown or incomplete argument: ${argument ?? "<missing>"}`);
|
||||
}
|
||||
|
||||
if (!origin) throw new Error("--origin is required");
|
||||
if (!factorString) throw new Error("--factor is required");
|
||||
if (!allowedFactorStrings.has(factorString)) {
|
||||
throw new Error("--factor must be one of 0.8, 1, 1.25, or 2");
|
||||
}
|
||||
if (!output) throw new Error("--output is required");
|
||||
|
||||
return {
|
||||
origin: validateOrigin(origin),
|
||||
factor: Number(factorString),
|
||||
outputDirectory: await validateOutputDirectory(output),
|
||||
};
|
||||
};
|
||||
|
||||
export const createZoomExtensionContent = (origin: string, factor: number) => {
|
||||
const manifest = {
|
||||
manifest_version: 3,
|
||||
name: "Homarr release v2 native zoom",
|
||||
version: "1.0.0",
|
||||
permissions: ["tabs"],
|
||||
host_permissions: [`${origin}/*`],
|
||||
background: { service_worker: "background.js" },
|
||||
};
|
||||
const background = `const TARGET_ORIGIN = ${JSON.stringify(origin)};
|
||||
const ZOOM_FACTOR = ${JSON.stringify(factor)};
|
||||
|
||||
function applyZoom(tab) {
|
||||
if (tab?.id == null || !tab.url) return;
|
||||
|
||||
let origin;
|
||||
try {
|
||||
origin = new URL(tab.url).origin;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (origin !== TARGET_ORIGIN) return;
|
||||
|
||||
chrome.tabs.setZoom(tab.id, ZOOM_FACTOR, () => {
|
||||
void chrome.runtime.lastError;
|
||||
});
|
||||
}
|
||||
|
||||
chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
|
||||
if (changeInfo.status !== "complete") return;
|
||||
applyZoom({ ...tab, id: tabId });
|
||||
});
|
||||
|
||||
chrome.tabs.query({}, (tabs) => {
|
||||
tabs.forEach(applyZoom);
|
||||
});
|
||||
`;
|
||||
|
||||
return {
|
||||
manifest: `${JSON.stringify(manifest, null, 2)}\n`,
|
||||
background,
|
||||
};
|
||||
};
|
||||
|
||||
export const generateZoomExtension = async (options: ZoomOptions) => {
|
||||
const content = createZoomExtensionContent(options.origin, options.factor);
|
||||
await assertSafeZoomOutput(options.outputDirectory, { allowMissing: true });
|
||||
await mkdir(options.outputDirectory, { mode: 0o700 });
|
||||
await assertSafeZoomOutput(options.outputDirectory);
|
||||
await Promise.all([
|
||||
writeFile(path.join(options.outputDirectory, "manifest.json"), content.manifest, { flag: "wx", mode: 0o600 }),
|
||||
writeFile(path.join(options.outputDirectory, "background.js"), content.background, { flag: "wx", mode: 0o600 }),
|
||||
]);
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const rawArguments = process.argv.slice(2);
|
||||
if (rawArguments.includes("--help") || rawArguments.includes("-h")) {
|
||||
usage();
|
||||
return;
|
||||
}
|
||||
|
||||
const options = await parseZoomOptions(rawArguments);
|
||||
await generateZoomExtension(options);
|
||||
console.log(`Generated release-v2 native zoom extension`);
|
||||
console.log(`Origin: ${options.origin}`);
|
||||
console.log(`Factor: ${options.factor}`);
|
||||
console.log(`Output: ${options.outputDirectory}`);
|
||||
};
|
||||
|
||||
const isDirectExecution = process.argv[1]
|
||||
? pathToFileURL(path.resolve(process.argv[1])).href === pathToFileURL(fileURLToPath(import.meta.url)).href
|
||||
: false;
|
||||
if (isDirectExecution) await main();
|
||||
@@ -0,0 +1,155 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { mkdir, readFile, rm, symlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, test } from "vitest";
|
||||
|
||||
import { createZoomExtensionContent, generateZoomExtension, parseZoomOptions } from "./zoom.mts";
|
||||
|
||||
const temporaryDirectories: string[] = [];
|
||||
|
||||
const createOutputPath = (suffix: string) => {
|
||||
const directory = path.join(
|
||||
tmpdir(),
|
||||
`homarr-release-v2-qa-zoom-${process.pid}-${Date.now()}-${suffix}-${temporaryDirectories.length}`,
|
||||
);
|
||||
temporaryDirectories.push(directory);
|
||||
return directory;
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })));
|
||||
});
|
||||
|
||||
describe("release-v2 native zoom helper", () => {
|
||||
test("generates deterministic, exact-origin MV3 extension files", async () => {
|
||||
const firstOutput = createOutputPath("first");
|
||||
const secondOutput = createOutputPath("second");
|
||||
const firstOptions = await parseZoomOptions([
|
||||
"--origin",
|
||||
"http://127.0.0.1:34401/",
|
||||
"--factor",
|
||||
"1.25",
|
||||
"--output",
|
||||
firstOutput,
|
||||
]);
|
||||
const secondOptions = await parseZoomOptions([
|
||||
"--origin",
|
||||
"http://127.0.0.1:34401",
|
||||
"--factor",
|
||||
"1.25",
|
||||
"--output",
|
||||
secondOutput,
|
||||
]);
|
||||
|
||||
await generateZoomExtension(firstOptions);
|
||||
await generateZoomExtension(secondOptions);
|
||||
|
||||
const firstManifest = await readFile(path.join(firstOutput, "manifest.json"), "utf8");
|
||||
const firstBackground = await readFile(path.join(firstOutput, "background.js"), "utf8");
|
||||
expect(firstManifest).toBe(await readFile(path.join(secondOutput, "manifest.json"), "utf8"));
|
||||
expect(firstBackground).toBe(await readFile(path.join(secondOutput, "background.js"), "utf8"));
|
||||
expect(JSON.parse(firstManifest)).toMatchObject({
|
||||
manifest_version: 3,
|
||||
permissions: ["tabs"],
|
||||
host_permissions: ["http://127.0.0.1:34401/*"],
|
||||
background: { service_worker: "background.js" },
|
||||
});
|
||||
expect(firstBackground).toContain('const TARGET_ORIGIN = "http://127.0.0.1:34401";');
|
||||
expect(firstBackground).toContain("const ZOOM_FACTOR = 1.25;");
|
||||
expect(firstBackground).toContain("origin !== TARGET_ORIGIN");
|
||||
expect(firstBackground).not.toContain("password");
|
||||
});
|
||||
|
||||
test.each(["0.8", "1", "1.25", "2"])("accepts the supported factor %s", async (factor) => {
|
||||
const output = createOutputPath(`factor-${factor.replace(".", "-")}`);
|
||||
const options = await parseZoomOptions([
|
||||
"--origin",
|
||||
"https://qa.example.test:8443",
|
||||
"--factor",
|
||||
factor,
|
||||
"--output",
|
||||
output,
|
||||
]);
|
||||
|
||||
expect(options.factor).toBe(Number(factor));
|
||||
});
|
||||
|
||||
test.each(["0", "0.9", "1.0", "1.5", "2.1", "not-a-number"])("rejects unsupported factor %s", async (factor) => {
|
||||
const output = createOutputPath(`invalid-factor-${factor.replaceAll(/[^A-Za-z0-9]/gu, "-")}`);
|
||||
await expect(
|
||||
parseZoomOptions(["--origin", "http://127.0.0.1:34401", "--factor", factor, "--output", output]),
|
||||
).rejects.toThrow("--factor must be one of 0.8, 1, 1.25, or 2");
|
||||
});
|
||||
|
||||
test.each([
|
||||
"ftp://127.0.0.1:34401",
|
||||
"http://user:secret@127.0.0.1:34401",
|
||||
"http://127.0.0.1:34401/boards/qa-grid-24",
|
||||
"http://127.0.0.1:34401/?mode=qa",
|
||||
"not-a-url",
|
||||
])("rejects a non-origin target without echoing it: %s", async (origin) => {
|
||||
const output = createOutputPath("invalid-origin");
|
||||
await expect(parseZoomOptions(["--origin", origin, "--factor", "1", "--output", output])).rejects.toThrow(
|
||||
/--origin/u,
|
||||
);
|
||||
});
|
||||
|
||||
test("requires a new, predictably named direct child of the temporary directory", async () => {
|
||||
const existingDirectory = createOutputPath("existing");
|
||||
const nestedParent = createOutputPath("nested-parent");
|
||||
await mkdir(existingDirectory);
|
||||
await mkdir(nestedParent);
|
||||
await expect(
|
||||
parseZoomOptions(["--origin", "http://127.0.0.1:34401", "--factor", "1", "--output", "relative-extension"]),
|
||||
).rejects.toThrow("--output must be an absolute path");
|
||||
await expect(
|
||||
parseZoomOptions(["--origin", "http://127.0.0.1:34401", "--factor", "1", "--output", existingDirectory]),
|
||||
).rejects.toThrow("--output must not already exist");
|
||||
await expect(
|
||||
parseZoomOptions([
|
||||
"--origin",
|
||||
"http://127.0.0.1:34401",
|
||||
"--factor",
|
||||
"1",
|
||||
"--output",
|
||||
path.join(nestedParent, "homarr-release-v2-qa-zoom-nested"),
|
||||
]),
|
||||
).rejects.toThrow("direct child");
|
||||
await expect(
|
||||
parseZoomOptions([
|
||||
"--origin",
|
||||
"http://127.0.0.1:34401",
|
||||
"--factor",
|
||||
"1",
|
||||
"--output",
|
||||
path.join(tmpdir(), "uncontrolled-extension"),
|
||||
]),
|
||||
).rejects.toThrow("basename must start");
|
||||
});
|
||||
|
||||
test("canonicalizes a temporary-directory alias for extension output", async () => {
|
||||
const temporaryAlias = createOutputPath("alias-parent");
|
||||
await symlink(tmpdir(), temporaryAlias, "dir");
|
||||
const outputName = `homarr-release-v2-qa-zoom-${process.pid}-${Date.now()}-through-alias`;
|
||||
const canonicalOutput = path.join(tmpdir(), outputName);
|
||||
temporaryDirectories.push(canonicalOutput);
|
||||
|
||||
const options = await parseZoomOptions([
|
||||
"--origin",
|
||||
"http://127.0.0.1:34401",
|
||||
"--factor",
|
||||
"1",
|
||||
"--output",
|
||||
path.join(temporaryAlias, outputName),
|
||||
]);
|
||||
|
||||
expect(options.outputDirectory).toBe(canonicalOutput);
|
||||
});
|
||||
|
||||
test("renders baseline factor 1 without changing the extension shape", () => {
|
||||
const content = createZoomExtensionContent("https://qa.example.test", 1);
|
||||
expect(content.background).toContain("const ZOOM_FACTOR = 1;");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user