fix(qa): validate dense fixture geometry

This commit is contained in:
ajnart committed 2026-09-01 13:54:30 +02:00
1 parent 358d6469cf
commit 2d5edf4651
8 files changed
+389 -39

No files matched your search

+1 -1
View File
@@ -80,7 +80,7 @@ Screenshots and traces stay untracked under `.screenshots/release-v2/<agent-id>/
The decision is `GO` only when there are no P0/P1 findings and no critical gaps; otherwise it is `NO-GO`. The report command itself treats missing, null, malformed, wrong-candidate, incomplete, assignment-inconsistent, or case-rollup-inconsistent metadata as an explicit critical metadata gap. It does not depend on a separate verification run to force `NO-GO`.
Aggregation refuses symlinked packet reports, `ledger.json`, and `master-report.md`. Artifact and evidence paths must exist, be absolute, and ultimately resolve inside the assigned `.screenshots/release-v2/<agent-id>/` directory; a lexically contained symlink that resolves outside is rejected.
Aggregation refuses symlinked packet reports, `ledger.json`, and `master-report.md`. Primary report artifacts plus finding and performance evidence must be existing regular files under the packet's assigned `.screenshots/release-v2/<agent-id>/` directory. Independent reproduction evidence may use an existing regular file from any agent directory under the canonical `.screenshots/release-v2/` root. Every artifact path must be absolute and symlink-free; paths that escape the applicable root are rejected.
## Native browser zoom
+40
View File
@@ -52,11 +52,31 @@ const placementIsValid = (placement: GeometryPlacement) =>
placement.width >= 1 &&
placement.height >= 1;
const placementsOverlap = (first: GeometryPlacement, second: GeometryPlacement) =>
first.xOffset < second.xOffset + second.width &&
second.xOffset < first.xOffset + first.width &&
first.yOffset < second.yOffset + second.height &&
second.yOffset < first.yOffset + first.height;
const compareItemLayouts = (first: GeometryItemLayout, second: GeometryItemLayout) => {
if (first.itemId < second.itemId) return -1;
if (first.itemId > second.itemId) return 1;
return 0;
};
export const findFixtureGeometryErrors = (geometry: FixtureGeometry) => {
const errors: string[] = [];
const layoutsById = new Map(geometry.layouts.map((layout) => [layout.id, layout]));
const sectionsById = new Map(geometry.sections.map((section) => [section.id, section]));
const sectionLayoutsByKey = new Map<string, GeometrySectionLayout>();
const itemsByDomain = new Map<string, GeometryItemLayout[]>();
const addItemLayout = (itemLayout: GeometryItemLayout) => {
const key = `${itemLayout.layoutId}\0${itemLayout.sectionId}`;
const itemLayouts = itemsByDomain.get(key) ?? [];
itemLayouts.push(itemLayout);
itemsByDomain.set(key, itemLayouts);
};
for (const sectionLayout of geometry.sectionLayouts) {
const key = `${sectionLayout.layoutId}\0${sectionLayout.sectionId}`;
@@ -145,6 +165,26 @@ export const findFixtureGeometryErrors = (geometry: FixtureGeometry) => {
errors.push(
`item ${itemLayout.itemId} in layout ${layout.id} exceeds its ${capacity.columns}-column parent ${capacity.kind}`,
);
continue;
}
addItemLayout(itemLayout);
}
const sortedDomains = [...itemsByDomain.entries()].toSorted(([first], [second]) => {
if (first < second) return -1;
if (first > second) return 1;
return 0;
});
for (const [domain, itemLayouts] of sortedDomains) {
const separatorIndex = domain.indexOf("\0");
const layoutId = domain.slice(0, separatorIndex);
const sectionId = domain.slice(separatorIndex + 1);
const sortedItemLayouts = itemLayouts.toSorted(compareItemLayouts);
for (const [index, first] of sortedItemLayouts.entries()) {
for (const second of sortedItemLayouts.slice(index + 1)) {
if (!placementsOverlap(first, second)) continue;
errors.push(`item ${first.itemId} overlaps item ${second.itemId} in layout ${layoutId} section ${sectionId}`);
}
}
}
+143
View File
@@ -66,6 +66,149 @@ const baseGeometry = (): FixtureGeometry => ({
});
describe("release-v2 QA fixture geometry", () => {
test("rejects exact duplicate item placements in the same section", () => {
const geometry = baseGeometry();
geometry.itemLayouts.push({
itemId: "duplicate-item",
boardId: "board",
sectionId: "level-3",
layoutId: "layout-mobile",
xOffset: 0,
yOffset: 0,
width: 1,
height: 2,
});
expect(findFixtureGeometryErrors(geometry)).toContain(
"item duplicate-item overlaps item nested-item in layout layout-mobile section level-3",
);
});
test("rejects partial item overlaps in the same section", () => {
const geometry = baseGeometry();
geometry.sections = [{ id: "root", boardId: "board", kind: "empty", xOffset: 0 }];
geometry.sectionLayouts = [];
geometry.itemLayouts = [
{
itemId: "first-item",
boardId: "board",
sectionId: "root",
layoutId: "layout-mobile",
xOffset: 0,
yOffset: 0,
width: 2,
height: 2,
},
{
itemId: "second-item",
boardId: "board",
sectionId: "root",
layoutId: "layout-mobile",
xOffset: 1,
yOffset: 1,
width: 2,
height: 2,
},
];
expect(findFixtureGeometryErrors(geometry)).toContain(
"item first-item overlaps item second-item in layout layout-mobile section root",
);
});
test("accepts tightly packed items that only share edges", () => {
const geometry = baseGeometry();
geometry.sections = [{ id: "root", boardId: "board", kind: "empty", xOffset: 0 }];
geometry.sectionLayouts = [];
geometry.itemLayouts = [
{
itemId: "top-left",
boardId: "board",
sectionId: "root",
layoutId: "layout-mobile",
xOffset: 0,
yOffset: 0,
width: 1,
height: 2,
},
{
itemId: "top-right",
boardId: "board",
sectionId: "root",
layoutId: "layout-mobile",
xOffset: 1,
yOffset: 0,
width: 2,
height: 2,
},
{
itemId: "bottom",
boardId: "board",
sectionId: "root",
layoutId: "layout-mobile",
xOffset: 0,
yOffset: 2,
width: 3,
height: 1,
},
];
expect(findFixtureGeometryErrors(geometry)).toEqual([]);
});
test("does not compare local coordinates from different parent sections", () => {
const geometry = baseGeometry();
geometry.sections = [
{ id: "root", boardId: "board", kind: "empty", xOffset: 0 },
{ id: "left-container", boardId: "board", kind: "container", xOffset: null },
{ id: "right-container", boardId: "board", kind: "container", xOffset: null },
];
geometry.sectionLayouts = [
{
sectionId: "left-container",
layoutId: "layout-mobile",
parentSectionId: "root",
xOffset: 0,
yOffset: 0,
width: 1,
height: 2,
},
{
sectionId: "right-container",
layoutId: "layout-mobile",
parentSectionId: "root",
xOffset: 1,
yOffset: 0,
width: 2,
height: 2,
},
];
geometry.itemLayouts = [
{
itemId: "left-item",
boardId: "board",
sectionId: "left-container",
layoutId: "layout-mobile",
xOffset: 0,
yOffset: 0,
width: 1,
height: 1,
},
{
itemId: "right-item",
boardId: "board",
sectionId: "right-container",
layoutId: "layout-mobile",
xOffset: 0,
yOffset: 0,
width: 1,
height: 1,
},
];
expect(findFixtureGeometryErrors(geometry)).toEqual([]);
});
test("rejects an item that fits total columns but overflows the gutter-reduced main lane", () => {
const geometry = baseGeometry();
geometry.layouts[0] = {
+47 -12
View File
@@ -35,23 +35,39 @@ export const writeSafeReportFile = async (
}
};
export const validateResolvedArtifactPath = async (
expectedPacketDirectory: string,
interface ArtifactDirectoryMessages {
absolutePathRequirement: string;
directoryName: string;
}
const packetArtifactDirectoryMessages: ArtifactDirectoryMessages = {
absolutePathRequirement: "the packet's absolute artifact directory",
directoryName: "packet artifact directory",
};
const reproductionArtifactDirectoryMessages: ArtifactDirectoryMessages = {
absolutePathRequirement: "the absolute release-v2 artifact directory",
directoryName: "release-v2 artifact directory",
};
const validateResolvedArtifactPathWithin = async (
expectedDirectory: string,
artifactPath: string,
label: string,
messages: ArtifactDirectoryMessages,
): Promise<string | null> => {
if (!path.isAbsolute(artifactPath) || !isPathWithin(expectedPacketDirectory, artifactPath)) {
return `${label} must use the packet's absolute artifact directory`;
if (!path.isAbsolute(artifactPath) || !isPathWithin(expectedDirectory, artifactPath)) {
return `${label} must use ${messages.absolutePathRequirement}`;
}
let resolvedPacketDirectory: string;
let resolvedExpectedDirectory: string;
try {
resolvedPacketDirectory = await realpath(expectedPacketDirectory);
resolvedExpectedDirectory = await realpath(expectedDirectory);
} catch (error) {
return `${label} packet artifact directory does not resolve (${errorCode(error) ?? "unknown error"})`;
return `${label} ${messages.directoryName} does not resolve (${errorCode(error) ?? "unknown error"})`;
}
if (resolvedPacketDirectory !== path.resolve(expectedPacketDirectory)) {
return `${label} packet artifact directory must not contain symlinks`;
if (resolvedExpectedDirectory !== path.resolve(expectedDirectory)) {
return `${label} ${messages.directoryName} must not contain symlinks`;
}
let resolvedArtifact: string;
@@ -60,11 +76,11 @@ export const validateResolvedArtifactPath = async (
} catch (error) {
return `${label} does not resolve (${errorCode(error) ?? "unknown error"})`;
}
if (!isPathWithin(resolvedPacketDirectory, resolvedArtifact)) {
return `${label} resolves outside the packet artifact directory`;
if (!isPathWithin(resolvedExpectedDirectory, resolvedArtifact)) {
return `${label} resolves outside the ${messages.directoryName}`;
}
try {
await assertSafeContainedPath(resolvedPacketDirectory, artifactPath, label);
await assertSafeContainedPath(resolvedExpectedDirectory, artifactPath, label);
const artifactStats = await lstat(artifactPath);
if (!artifactStats.isFile()) return `${label} must resolve to an existing regular file`;
} catch (error) {
@@ -72,3 +88,22 @@ export const validateResolvedArtifactPath = async (
}
return null;
};
export const validateResolvedArtifactPath = async (
expectedPacketDirectory: string,
artifactPath: string,
label: string,
): Promise<string | null> =>
validateResolvedArtifactPathWithin(expectedPacketDirectory, artifactPath, label, packetArtifactDirectoryMessages);
export const validateResolvedReproductionEvidencePath = async (
releaseArtifactDirectory: string,
artifactPath: string,
label: string,
): Promise<string | null> =>
validateResolvedArtifactPathWithin(
releaseArtifactDirectory,
artifactPath,
label,
reproductionArtifactDirectoryMessages,
);
@@ -11,6 +11,7 @@ import {
assertSafeReportPath,
readSafeReportFile,
validateResolvedArtifactPath,
validateResolvedReproductionEvidencePath,
writeSafeReportFile,
} from "./report-path-integrity.mts";
@@ -46,7 +47,8 @@ describe("release-v2 QA report path integrity", () => {
it("rejects an artifact symlink that ultimately escapes its packet directory", async () => {
const root = await temporaryDirectory("artifact-escape");
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
const releaseArtifactDirectory = path.join(root, ".screenshots", "release-v2");
const packetDirectory = path.join(releaseArtifactDirectory, "preflight-01");
const outsideArtifact = path.join(root, "outside.png");
const linkedArtifact = path.join(packetDirectory, "escape.png");
await mkdir(packetDirectory, { recursive: true });
@@ -56,6 +58,13 @@ describe("release-v2 QA report path integrity", () => {
await expect(
validateResolvedArtifactPath(packetDirectory, linkedArtifact, "preflight-01: artifact"),
).resolves.toContain("resolves outside");
await expect(
validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
linkedArtifact,
"preflight-01: reproduction evidence",
),
).resolves.toContain("resolves outside");
});
it("accepts a resolved regular artifact inside the packet directory", async () => {
@@ -68,9 +77,28 @@ describe("release-v2 QA report path integrity", () => {
await expect(validateResolvedArtifactPath(packetDirectory, artifact, "preflight-01: artifact")).resolves.toBeNull();
});
it("accepts independent reproduction evidence from another release-v2 agent directory", async () => {
const root = await temporaryDirectory("reproduction-artifact-valid");
const releaseArtifactDirectory = path.join(root, ".screenshots", "release-v2");
const packetDirectory = path.join(releaseArtifactDirectory, "board-03");
const reproductionDirectory = path.join(releaseArtifactDirectory, "repro-board03-remove-358d-b");
const artifact = path.join(reproductionDirectory, "remove-confirmation.png");
await mkdir(packetDirectory, { recursive: true });
await mkdir(reproductionDirectory, { recursive: true });
await writeFile(artifact, "valid");
await expect(
validateResolvedReproductionEvidencePath(releaseArtifactDirectory, artifact, "board-03: reproduction evidence"),
).resolves.toBeNull();
await expect(
validateResolvedArtifactPath(packetDirectory, artifact, "board-03: finding evidence"),
).resolves.toContain("packet's absolute artifact directory");
});
it("rejects an artifact directory and an in-directory artifact symlink", async () => {
const root = await temporaryDirectory("artifact-not-file");
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
const releaseArtifactDirectory = path.join(root, ".screenshots", "release-v2");
const packetDirectory = path.join(releaseArtifactDirectory, "preflight-01");
const artifactDirectory = path.join(packetDirectory, "looks-like-evidence.png");
const realArtifact = path.join(packetDirectory, "real.png");
const linkedArtifact = path.join(packetDirectory, "linked.png");
@@ -84,11 +112,46 @@ describe("release-v2 QA report path integrity", () => {
await expect(
validateResolvedArtifactPath(packetDirectory, linkedArtifact, "preflight-01: artifact"),
).resolves.toContain("symlink");
await expect(
validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
artifactDirectory,
"preflight-01: reproduction evidence",
),
).resolves.toContain("regular file");
await expect(
validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
linkedArtifact,
"preflight-01: reproduction evidence",
),
).resolves.toContain("symlink");
});
it("rejects relative, escaping, and missing independent reproduction evidence paths", async () => {
const root = await temporaryDirectory("reproduction-artifact-unsafe");
const releaseArtifactDirectory = path.join(root, ".screenshots", "release-v2");
const reproductionDirectory = path.join(releaseArtifactDirectory, "reproducer-01");
const outsideArtifact = path.join(root, "outside.png");
const missingArtifact = path.join(reproductionDirectory, "missing.png");
await mkdir(reproductionDirectory, { recursive: true });
await writeFile(outsideArtifact, "outside");
for (const artifact of ["relative.png", outsideArtifact, missingArtifact]) {
await expect(
validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
artifact,
"preflight-01: reproduction evidence",
),
).resolves.not.toBeNull();
}
});
it.runIf(process.platform !== "win32")("rejects a FIFO artifact", async () => {
const root = await temporaryDirectory("artifact-fifo");
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
const releaseArtifactDirectory = path.join(root, ".screenshots", "release-v2");
const packetDirectory = path.join(releaseArtifactDirectory, "preflight-01");
const fifoArtifact = path.join(packetDirectory, "capture.pipe");
await mkdir(packetDirectory, { recursive: true });
const result = spawnSync("mkfifo", [fifoArtifact]);
@@ -97,11 +160,19 @@ describe("release-v2 QA report path integrity", () => {
await expect(
validateResolvedArtifactPath(packetDirectory, fifoArtifact, "preflight-01: artifact"),
).resolves.toContain("regular file");
await expect(
validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
fifoArtifact,
"preflight-01: reproduction evidence",
),
).resolves.toContain("regular file");
});
it.runIf(process.platform !== "win32")("rejects a socket artifact", async () => {
const root = await temporaryDirectory("artifact-socket");
const packetDirectory = path.join(root, ".screenshots", "release-v2", "preflight-01");
const releaseArtifactDirectory = path.join(root, ".screenshots", "release-v2");
const packetDirectory = path.join(releaseArtifactDirectory, "preflight-01");
const socketArtifact = path.join(packetDirectory, "capture.sock");
await mkdir(packetDirectory, { recursive: true });
const server = createServer();
@@ -113,6 +184,13 @@ describe("release-v2 QA report path integrity", () => {
await expect(
validateResolvedArtifactPath(packetDirectory, socketArtifact, "preflight-01: artifact"),
).resolves.toContain("regular file");
await expect(
validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
socketArtifact,
"preflight-01: reproduction evidence",
),
).resolves.toContain("regular file");
} finally {
await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
}
+15 -9
View File
@@ -14,6 +14,7 @@ import {
assertSafeReportPath,
readSafeReportFile,
validateResolvedArtifactPath,
validateResolvedReproductionEvidencePath,
writeSafeReportFile,
} from "./report-path-integrity.mts";
import { resolveCheckoutCandidateSha } from "./provenance.mts";
@@ -436,8 +437,9 @@ const collectHumanReportErrors = (content: string, packet: Packet, metadata: Rep
};
const collectArtifactErrors = async (packet: Packet, metadata: ReportMetadata): Promise<string[]> => {
const packetArtifactDirectory = resolve(repoRoot, ".screenshots/release-v2", packet.id);
const artifacts = [
const releaseArtifactDirectory = resolve(repoRoot, ".screenshots/release-v2");
const packetArtifactDirectory = resolve(releaseArtifactDirectory, packet.id);
const packetArtifacts = [
...metadata.artifacts.map((artifact) => ({ artifact, label: `${packet.id}: artifact` })),
...metadata.findings.flatMap((finding, findingIndex) =>
(finding.evidence ?? []).map((artifact) => ({
@@ -451,19 +453,23 @@ const collectArtifactErrors = async (packet: Packet, metadata: ReportMetadata):
label: `${packet.id}: performance.measurements[${measurementIndex}] evidence`,
})),
),
...metadata.independentReproductions.flatMap((reproduction, reproductionIndex) =>
reproduction.evidence.map((artifact) => ({
artifact,
label: `${packet.id}: independentReproductions[${reproductionIndex}] evidence`,
})),
),
];
const errors: string[] = [];
for (const { artifact, label } of artifacts) {
for (const { artifact, label } of packetArtifacts) {
const error = await validateResolvedArtifactPath(packetArtifactDirectory, artifact, label);
if (error) errors.push(error);
}
for (const [reproductionIndex, reproduction] of metadata.independentReproductions.entries()) {
for (const artifact of reproduction.evidence) {
const error = await validateResolvedReproductionEvidencePath(
releaseArtifactDirectory,
artifact,
`${packet.id}: independentReproductions[${reproductionIndex}] evidence`,
);
if (error) errors.push(error);
}
}
return errors;
};
+53 -10
View File
@@ -670,6 +670,48 @@ const packPositions = (itemDefinitions: ItemDefinition[], board: BoardDefinition
});
};
const denseCollisionPlacementGeometry = (columnCount: number) => {
if (columnCount < 3) throw new Error("The dense collision fixture requires at least three columns");
if (columnCount === 3) {
return [
{ xOffset: 0, yOffset: 0, width: 1, height: 2 },
{ xOffset: 1, yOffset: 0, width: 2, height: 1 },
{ xOffset: 1, yOffset: 1, width: 1, height: 1 },
{ xOffset: 0, yOffset: 2, width: 2, height: 1 },
{ xOffset: 2, yOffset: 2, width: 1, height: 2 },
{ xOffset: 0, yOffset: 3, width: 1, height: 1 },
{ xOffset: 0, yOffset: 4, width: 3, height: 1 },
{ xOffset: 0, yOffset: 5, width: 1, height: 2 },
{ xOffset: 1, yOffset: 5, width: 1, height: 1 },
{ xOffset: 2, yOffset: 5, width: 1, height: 2 },
{ xOffset: 1, yOffset: 6, width: 1, height: 1 },
{ xOffset: 0, yOffset: 7, width: 2, height: 2 },
];
}
const laneBoundary = (lane: number) => Math.floor((columnCount * lane) / 4);
const fromLanes = (startLane: number, endLane: number, yOffset: number, height: number) => ({
xOffset: laneBoundary(startLane),
yOffset,
width: laneBoundary(endLane) - laneBoundary(startLane),
height,
});
return [
fromLanes(0, 1, 0, 2),
fromLanes(1, 3, 0, 1),
fromLanes(3, 4, 0, 2),
fromLanes(1, 2, 1, 1),
fromLanes(0, 2, 2, 1),
fromLanes(2, 3, 2, 2),
fromLanes(3, 4, 2, 1),
fromLanes(0, 1, 3, 2),
fromLanes(1, 2, 3, 1),
fromLanes(3, 4, 3, 2),
fromLanes(1, 3, 4, 1),
fromLanes(0, 3, 5, 2),
];
};
const itemForKind = (
boardHandle: string,
handle: string,
@@ -1062,22 +1104,23 @@ const seedBoardsAsync = async (optionsByKind: Map<WidgetKind, string>) => {
}
if (board.handle === "dense-collisions") {
positionRows.push(
...boardLayouts(board).flatMap((layout) =>
boardItems.map((item, index) => {
const columnCount = mainLaneColumnCount(layout);
const width = Math.min(columnCount, 2 + (index % 3));
...boardLayouts(board).flatMap((layout) => {
const placements = denseCollisionPlacementGeometry(mainLaneColumnCount(layout));
if (placements.length !== boardItems.length) {
throw new Error(`Dense collision placement count does not match the item count in layout ${layout.handle}`);
}
return boardItems.map((item, index) => {
const placement = placements.at(index);
if (!placement) throw new Error(`Missing dense collision placement ${index + 1}`);
return {
boardHandle: board.handle,
itemHandle: item.handle,
layoutHandle: layout.handle,
sectionHandle: "root",
xOffset: Math.min(index % 3, columnCount - width),
yOffset: index % 2,
width,
height: 2 + (index % 2),
...placement,
};
}),
),
});
}),
);
} else {
positionRows.push(...packPositions(boardItems, board));
+8 -3
View File
@@ -18,7 +18,12 @@ import type {
} from "./permissions.mts";
import { resolveCheckoutCandidateSha } from "./provenance.mts";
import { collectHumanWidgetStatusErrors, normalizeReportMetadata } from "./report-integrity.mts";
import { assertSafeReportPath, readSafeReportFile, validateResolvedArtifactPath } from "./report-path-integrity.mts";
import {
assertSafeReportPath,
readSafeReportFile,
validateResolvedArtifactPath,
validateResolvedReproductionEvidencePath,
} from "./report-path-integrity.mts";
import { createCandidateBuildPaths, validateRuntimeExecutionContract } from "./runtime.mts";
import { assertSafeRunRoot, isPathWithin, validateFixtureUrl } from "./safety.mts";
@@ -1152,8 +1157,8 @@ const main = async (): Promise<void> => {
errors.push(`${packet.id}: independent reproduction has invalid outcome`);
}
for (const evidence of reproduction.evidence ?? []) {
const artifactError = await validateResolvedArtifactPath(
resolve(repoRoot, ".screenshots/release-v2", packet.id),
const artifactError = await validateResolvedReproductionEvidencePath(
resolve(repoRoot, ".screenshots/release-v2"),
evidence,
`${packet.id}: reproduction evidence`,
);