import { describe, expect, it } from "vitest"; import type { BenchmarkPairInput, BuildResult, RuntimeResult } from "./benchmark-pair-lib.mts"; import { compareBenchmarkPair } from "./benchmark-pair-lib.mts"; const runWindow = (index: number) => { const startedAt = new Date(Date.UTC(2026, 0, 1, 0, 0, index * 2)).toISOString(); const finishedAt = new Date(Date.UTC(2026, 0, 1, 0, 0, index * 2 + 1)).toISOString(); return { startedAt, finishedAt }; }; const expectedCache = [{ logicalId: "pnpm-store", target: "/pnpm/store", mustIncreaseUsage: true }]; const cacheStability = { attempts: 6, pollIntervalMs: 2_000, stableForMs: 10_000, timeoutMs: 30_000 }; const cacheSnapshot = (side: "baseline" | "candidate", stage: "empty" | "warm" | "r1" | "r2") => ({ cacheMountCount: stage === "empty" ? 0 : 1, cacheMounts: stage === "empty" ? [] : [ { logicalId: "pnpm-store", recordId: `${side}-pnpm-record`, target: "/pnpm/store", usageCount: { warm: 1, r1: 2, r2: 3 }[stage], }, ], recordCount: stage === "empty" ? 0 : { warm: 10, r1: 12, r2: 14 }[stage], recordsFingerprint: `${side}-${stage}`, }); const build = (side: "baseline" | "candidate", repetition: number): BuildResult => ({ schemaVersion: 4, createdAt: runWindow(side === "baseline" ? (repetition === 1 ? 2 : 5) : repetition === 1 ? 3 : 4).finishedAt, run: runWindow(side === "baseline" ? (repetition === 1 ? 2 : 5) : repetition === 1 ? 3 : 4), claimEligible: true, claimIneligibleReasons: [], mode: "source-change", builder: `${side}-builder`, runLabel: side, declaredRevision: "head+worktree", series: { id: "series", cacheState: "warm", repetition, totalRepetitions: 2 }, source: { headSha: "head", measuredRevision: "head+worktree", fingerprint: `${side}-change-${repetition}`, baseFingerprint: `${side}-fingerprint`, syntheticSourceChange: { file: "layout.tsx", id: `change-${repetition}` }, }, host: { architecture: "arm64", dockerServer: { architecture: "arm64", operatingSystem: "linux", targetPlatform: "linux/arm64", version: "1" }, platform: "darwin", }, image: { architecture: "arm64", digest: null, id: `${side}-build-${repetition}`, revision: "head+worktree", sourceFingerprint: `${side}-change-${repetition}`, targetPlatform: "linux/arm64", }, builderCache: { expected: expectedCache, before: cacheSnapshot(side, repetition === 1 ? "warm" : "r1"), after: cacheSnapshot(side, repetition === 1 ? "r1" : "r2"), afterStability: cacheStability, }, build: { nextCompileMs: side === "baseline" ? 200 : 100, wallTimeMs: side === "baseline" ? 300 : 150 }, }); const warmup = (side: "baseline" | "candidate"): BuildResult => { const run = runWindow(side === "baseline" ? 0 : 1); return { schemaVersion: 4, createdAt: run.finishedAt, run, claimEligible: false, claimIneligibleReasons: ["cached mode does not force an application rebuild"], mode: "cached", builder: `${side}-builder`, runLabel: side, declaredRevision: "head+worktree", series: { id: "series", cacheState: "cold", repetition: 1, totalRepetitions: 2 }, source: { headSha: "head", measuredRevision: "head+worktree", fingerprint: `${side}-fingerprint`, baseFingerprint: `${side}-fingerprint`, syntheticSourceChange: null, }, host: { architecture: "arm64", dockerServer: { architecture: "arm64", operatingSystem: "linux", targetPlatform: "linux/arm64", version: "1", }, platform: "darwin", }, image: { architecture: "arm64", digest: null, id: `${side}-runtime-image`, revision: "head+worktree", sourceFingerprint: `${side}-fingerprint`, targetPlatform: "linux/arm64", }, builderCache: { expected: expectedCache, before: cacheSnapshot(side, "empty"), after: cacheSnapshot(side, "warm"), afterStability: cacheStability, }, build: { nextCompileMs: side === "baseline" ? 400 : 300, wallTimeMs: side === "baseline" ? 500 : 400 }, }; }; const runtimeSample = (value: number) => ({ boardMountedMs: value, fcpMs: value, lcpFinalizedByInputMs: 5_100, lcpMs: value, ttfbMs: value, widgetImplementationsMountedMs: value, widgetDataSettledMs: value, }); const runtime = (side: "baseline" | "candidate"): RuntimeResult => { return { schemaVersion: 7, claimEligible: true, claimIneligibleReasons: [], claimScope: ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"], metricEligibility: Object.fromEntries( ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"].map((key) => [ key, { claimEligible: true }, ]), ), image: { architecture: "arm64", digest: null, id: `${side}-runtime-image`, revision: "head+worktree", sourceFingerprint: `${side}-fingerprint`, }, host: { architecture: "arm64", node: { version: "24", v8Version: "13" }, platform: "darwin", release: "1" }, provenance: { browser: { engine: "chromium", executablePath: "/chromium", headless: true, httpCachePolicy: "default", launchArguments: [`--proxy-server=http://127.0.0.1:${side === "baseline" ? "31001" : "32002"}`], version: "1", viewport: { width: 1280, height: 720 }, }, container: { cpuLimit: 2, memoryLimitBytes: 1024, network: { internal: true }, node: { architecture: "arm64", platform: "linux", version: "24", v8Version: "13" }, }, docker: { engine: { version: "1" }, version: { version: "1" } }, ingressProxy: { excludedFromMemorySamples: true, implementation: "Node.js TCP proxy" }, }, browserNetworkIsolation: { active: true, canaries: { http: { attemptCount: 1, responseHeader: "true", status: 502 }, webSockets: [ { connectTarget: "browser-ws.invalid:80", url: "ws://browser-ws.invalid/canary" }, { connectTarget: "browser-wss.invalid:443", url: "wss://browser-wss.invalid/canary" }, ].map((canary) => ({ ...canary, attemptCount: 1, browserEventCount: 1, closeCount: 1, errorEvent: true, frameReceivedCount: 0, frameSentCount: 0, opened: false, socketErrorCount: 1, timedOut: false, })), }, externalBrowserWebSockets: [], unexpectedExternalResponses: [], }, serverNetworkIsolation: { active: true, canary: { directIp: { blocked: true }, domain: { blocked: true } }, network: { internal: true }, }, workload: { settleMs: 600_000, sampleIntervalMs: 30_000, settleSampleCount: 20, lcpObservationMs: 5_000, lcpStabilityMs: 500, widgetDataQuietMs: 500, spotlightIdleMs: 2_500, spotlightIdlePolicy: side === "baseline" ? "mounted" : "preload-only", interactionIterations: 7, widgetCount: 32, implementationMarkerPolicy: "true", widgetImplementationMarkerCount: 32, widgetErrorCount: 0, startToReadyMs: 1_000, coldSpotlightFeedbackMs: 20, coldSpotlightReadyMs: 200, postIdleWarmInteractionMs: 30, routeTimings: [{ path: "/manage" }], pageLoadSamples: Array.from({ length: 20 }, () => runtimeSample(side === "baseline" ? 200 : 100)), }, checkpoints: [ { capturedAt: new Date(0).toISOString(), name: "startup" }, { capturedAt: new Date(1_000).toISOString(), name: "workload" }, ...Array.from({ length: 20 }, (_, index) => ({ capturedAt: new Date(1_000 + (index + 1) * 30_000).toISOString(), name: `settle-${index + 1}`, })), ], summary: { startupBytes: 1, workloadBytes: 2, settledBytes: 2, peakBytes: 3 }, }; }; const validInput = (): BenchmarkPairInput => ({ baselineWarmup: warmup("baseline"), candidateWarmup: warmup("candidate"), baselineBuilds: [build("baseline", 1), build("baseline", 2)], candidateBuilds: [build("candidate", 1), build("candidate", 2)], baselineRuntime: runtime("baseline"), candidateRuntime: runtime("candidate"), }); describe("benchmark pair validation", () => { it("accepts a complete, isolated, provenance-matched comparison", () => { const result = compareBenchmarkPair(validInput()); expect(result.claimEligible).toBe(true); expect(result.claimIneligibleReasons).toEqual([]); expect(result.runtime.lcpMs.comparison).toMatchObject({ baseline: 200, candidate: 100, improvementPercent: 50 }); }); it("rejects an incomplete build series", () => { const input = validInput(); input.candidateBuilds.pop(); expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate build repetitions are incomplete"); }); it("rejects runtime source drift and uncontrolled server egress", () => { const input = validInput(); input.candidateRuntime.image.sourceFingerprint = "wrong"; input.candidateRuntime.serverNetworkIsolation.canary.directIp.blocked = false; const result = compareBenchmarkPair(input); expect(result.claimEligible).toBe(false); expect(result.claimIneligibleReasons).toContain( "candidate runtime image does not match the candidate build source", ); expect(result.claimIneligibleReasons).toContain("candidate server network is not isolated"); }); it("rejects runtime evidence without blocked HTTP, WS, and WSS browser canaries", () => { const input = validInput(); const wssCanary = input.candidateRuntime.browserNetworkIsolation.canaries.webSockets[1]; if (!wssCanary) throw new Error("Missing WSS canary fixture"); wssCanary.opened = true; expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate browser network is not isolated"); }); it("requires one WS and one WSS browser isolation canary", () => { const input = validInput(); const wssCanary = input.candidateRuntime.browserNetworkIsolation.canaries.webSockets[1]; if (!wssCanary) throw new Error("Missing WSS canary fixture"); wssCanary.url = "ws://second-browser-ws.invalid/canary"; wssCanary.connectTarget = "second-browser-ws.invalid:80"; expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate browser network is not isolated"); }); it("recomputes runtime claim minima instead of trusting embedded eligibility flags", () => { const input = validInput(); input.candidateRuntime.claimScope = []; input.candidateRuntime.workload.lcpObservationMs = 1_000; input.candidateRuntime.workload.widgetDataQuietMs = 100; input.candidateRuntime.workload.settleMs = 1_000; input.candidateRuntime.workload.settleSampleCount = 19; input.candidateRuntime.checkpoints.pop(); const firstSample = input.candidateRuntime.workload.pageLoadSamples[0]; if (!firstSample) throw new Error("Missing runtime sample fixture"); firstSample.lcpMs = Number.POSITIVE_INFINITY; const reasons = compareBenchmarkPair(input).claimIneligibleReasons; expect(reasons).toContain("candidate runtime claim scope is incomplete"); expect(reasons).toContain("candidate runtime LCP observation is below 5000ms"); expect(reasons).toContain("candidate runtime widget-data quiet window is below 500ms"); expect(reasons).toContain("candidate runtime settle duration is below 600000ms"); expect(reasons).toContain("candidate runtime has fewer than 20 settle samples"); expect(reasons).toContain("candidate runtime metric lcpMs has invalid or missing samples"); }); it("recomputes measured cache-mount use instead of trusting build eligibility flags", () => { const input = validInput(); const firstBuild = input.candidateBuilds[0]; const beforeMount = firstBuild?.builderCache.before?.cacheMounts[0]; const afterMount = firstBuild?.builderCache.after?.cacheMounts[0]; if (!beforeMount || !afterMount) throw new Error("Missing cache mount fixture"); afterMount.usageCount = beforeMount.usageCount; expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain( "candidate repetition 1 build cache mount pnpm-store usage did not increase", ); }); it("verifies actual settle chronology, LCP finalization, and immutable build fingerprints", () => { const input = validInput(); const finalSettle = input.candidateRuntime.checkpoints.at(-1); const firstSample = input.candidateRuntime.workload.pageLoadSamples[0]; const firstBuild = input.candidateBuilds[0]; if (!finalSettle || !firstSample || !firstBuild) throw new Error("Missing evidence fixture"); finalSettle.capturedAt = new Date(500_000).toISOString(); firstSample.lcpFinalizedByInputMs = 4_999; firstBuild.image.sourceFingerprint = "wrong-fingerprint"; const reasons = compareBenchmarkPair(input).claimIneligibleReasons; expect(reasons).toContain("candidate runtime checkpoint span is shorter than the declared settle duration"); expect(reasons).toContain("candidate runtime LCP samples were not observed and finalized for the declared window"); expect(reasons).toContain("candidate build image fingerprint does not match its immutable source snapshot"); }); it("rejects a contaminated warmup or cache use outside the measured chain", () => { const input = validInput(); if (input.candidateWarmup.builderCache.before) input.candidateWarmup.builderCache.before.recordCount = 1; if (input.baselineBuilds[0]?.builderCache.before) { input.baselineBuilds[0].builderCache.before.recordsFingerprint = "unrecorded-cache-use"; } const result = compareBenchmarkPair(input); expect(result.claimIneligibleReasons).toContain("candidate warmup builder did not begin empty"); expect(result.claimIneligibleReasons).toContain("baseline builder cache changed outside repetition 1"); }); it("rejects overlapping builds and a stale runtime image with copied labels", () => { const input = validInput(); const candidateFirst = input.candidateBuilds[0]; if (candidateFirst) candidateFirst.run.startedAt = input.baselineBuilds[0]?.run.startedAt ?? ""; input.candidateRuntime.image.id = "stale-image-with-matching-labels"; const result = compareBenchmarkPair(input); expect(result.claimIneligibleReasons).toContain("build runs overlap between baseline:1 and candidate:1"); expect(result.claimIneligibleReasons).toContain( "candidate runtime image identity does not match the candidate warmup image", ); }); });