perf: optimize dashboard loading and builds

scripts/benchmarks/benchmark-pair-lib.spec.ts

@@ -0,0 +1,360 @@
+import { describe, expect, it } from "vitest";
+
+import type { BenchmarkPairInput, BuildResult, RuntimeResult } from "./benchmark-pair-lib.mts";
+import { compareBenchmarkPair } from "./benchmark-pair-lib.mts";
+
+const runWindow = (index: number) => {
+  const startedAt = new Date(Date.UTC(2026, 0, 1, 0, 0, index * 2)).toISOString();
+  const finishedAt = new Date(Date.UTC(2026, 0, 1, 0, 0, index * 2 + 1)).toISOString();
+  return { startedAt, finishedAt };
+};
+
+const expectedCache = [{ logicalId: "pnpm-store", target: "/pnpm/store", mustIncreaseUsage: true }];
+const cacheStability = { attempts: 6, pollIntervalMs: 2_000, stableForMs: 10_000, timeoutMs: 30_000 };
+const cacheSnapshot = (side: "baseline" | "candidate", stage: "empty" | "warm" | "r1" | "r2") => ({
+  cacheMountCount: stage === "empty" ? 0 : 1,
+  cacheMounts:
+    stage === "empty"
+      ? []
+      : [
+          {
+            logicalId: "pnpm-store",
+            recordId: `${side}-pnpm-record`,
+            target: "/pnpm/store",
+            usageCount: { warm: 1, r1: 2, r2: 3 }[stage],
+          },
+        ],
+  recordCount: stage === "empty" ? 0 : { warm: 10, r1: 12, r2: 14 }[stage],
+  recordsFingerprint: `${side}-${stage}`,
+});
+
+const build = (side: "baseline" | "candidate", repetition: number): BuildResult => ({
+  schemaVersion: 4,
+  createdAt: runWindow(side === "baseline" ? (repetition === 1 ? 2 : 5) : repetition === 1 ? 3 : 4).finishedAt,
+  run: runWindow(side === "baseline" ? (repetition === 1 ? 2 : 5) : repetition === 1 ? 3 : 4),
+  claimEligible: true,
+  claimIneligibleReasons: [],
+  mode: "source-change",
+  builder: `${side}-builder`,
+  runLabel: side,
+  declaredRevision: "head+worktree",
+  series: { id: "series", cacheState: "warm", repetition, totalRepetitions: 2 },
+  source: {
+    headSha: "head",
+    measuredRevision: "head+worktree",
+    fingerprint: `${side}-change-${repetition}`,
+    baseFingerprint: `${side}-fingerprint`,
+    syntheticSourceChange: { file: "layout.tsx", id: `change-${repetition}` },
+  },
+  host: {
+    architecture: "arm64",
+    dockerServer: { architecture: "arm64", operatingSystem: "linux", targetPlatform: "linux/arm64", version: "1" },
+    platform: "darwin",
+  },
+  image: {
+    architecture: "arm64",
+    digest: null,
+    id: `${side}-build-${repetition}`,
+    revision: "head+worktree",
+    sourceFingerprint: `${side}-change-${repetition}`,
+    targetPlatform: "linux/arm64",
+  },
+  builderCache: {
+    expected: expectedCache,
+    before: cacheSnapshot(side, repetition === 1 ? "warm" : "r1"),
+    after: cacheSnapshot(side, repetition === 1 ? "r1" : "r2"),
+    afterStability: cacheStability,
+  },
+  build: { nextCompileMs: side === "baseline" ? 200 : 100, wallTimeMs: side === "baseline" ? 300 : 150 },
+});
+
+const warmup = (side: "baseline" | "candidate"): BuildResult => {
+  const run = runWindow(side === "baseline" ? 0 : 1);
+  return {
+    schemaVersion: 4,
+    createdAt: run.finishedAt,
+    run,
+    claimEligible: false,
+    claimIneligibleReasons: ["cached mode does not force an application rebuild"],
+    mode: "cached",
+    builder: `${side}-builder`,
+    runLabel: side,
+    declaredRevision: "head+worktree",
+    series: { id: "series", cacheState: "cold", repetition: 1, totalRepetitions: 2 },
+    source: {
+      headSha: "head",
+      measuredRevision: "head+worktree",
+      fingerprint: `${side}-fingerprint`,
+      baseFingerprint: `${side}-fingerprint`,
+      syntheticSourceChange: null,
+    },
+    host: {
+      architecture: "arm64",
+      dockerServer: {
+        architecture: "arm64",
+        operatingSystem: "linux",
+        targetPlatform: "linux/arm64",
+        version: "1",
+      },
+      platform: "darwin",
+    },
+    image: {
+      architecture: "arm64",
+      digest: null,
+      id: `${side}-runtime-image`,
+      revision: "head+worktree",
+      sourceFingerprint: `${side}-fingerprint`,
+      targetPlatform: "linux/arm64",
+    },
+    builderCache: {
+      expected: expectedCache,
+      before: cacheSnapshot(side, "empty"),
+      after: cacheSnapshot(side, "warm"),
+      afterStability: cacheStability,
+    },
+    build: { nextCompileMs: side === "baseline" ? 400 : 300, wallTimeMs: side === "baseline" ? 500 : 400 },
+  };
+};
+
+const runtimeSample = (value: number) => ({
+  boardMountedMs: value,
+  fcpMs: value,
+  lcpFinalizedByInputMs: 5_100,
+  lcpMs: value,
+  ttfbMs: value,
+  widgetImplementationsMountedMs: value,
+  widgetDataSettledMs: value,
+});
+
+const runtime = (side: "baseline" | "candidate"): RuntimeResult => {
+  return {
+    schemaVersion: 7,
+    claimEligible: true,
+    claimIneligibleReasons: [],
+    claimScope: ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"],
+    metricEligibility: Object.fromEntries(
+      ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"].map((key) => [
+        key,
+        { claimEligible: true },
+      ]),
+    ),
+    image: {
+      architecture: "arm64",
+      digest: null,
+      id: `${side}-runtime-image`,
+      revision: "head+worktree",
+      sourceFingerprint: `${side}-fingerprint`,
+    },
+    host: { architecture: "arm64", node: { version: "24", v8Version: "13" }, platform: "darwin", release: "1" },
+    provenance: {
+      browser: {
+        engine: "chromium",
+        executablePath: "/chromium",
+        headless: true,
+        httpCachePolicy: "default",
+        launchArguments: [`--proxy-server=http://127.0.0.1:${side === "baseline" ? "31001" : "32002"}`],
+        version: "1",
+        viewport: { width: 1280, height: 720 },
+      },
+      container: {
+        cpuLimit: 2,
+        memoryLimitBytes: 1024,
+        network: { internal: true },
+        node: { architecture: "arm64", platform: "linux", version: "24", v8Version: "13" },
+      },
+      docker: { engine: { version: "1" }, version: { version: "1" } },
+      ingressProxy: { excludedFromMemorySamples: true, implementation: "Node.js TCP proxy" },
+    },
+    browserNetworkIsolation: {
+      active: true,
+      canaries: {
+        http: { attemptCount: 1, responseHeader: "true", status: 502 },
+        webSockets: [
+          { connectTarget: "browser-ws.invalid:80", url: "ws://browser-ws.invalid/canary" },
+          { connectTarget: "browser-wss.invalid:443", url: "wss://browser-wss.invalid/canary" },
+        ].map((canary) => ({
+          ...canary,
+          attemptCount: 1,
+          browserEventCount: 1,
+          closeCount: 1,
+          errorEvent: true,
+          frameReceivedCount: 0,
+          frameSentCount: 0,
+          opened: false,
+          socketErrorCount: 1,
+          timedOut: false,
+        })),
+      },
+      externalBrowserWebSockets: [],
+      unexpectedExternalResponses: [],
+    },
+    serverNetworkIsolation: {
+      active: true,
+      canary: { directIp: { blocked: true }, domain: { blocked: true } },
+      network: { internal: true },
+    },
+    workload: {
+      settleMs: 600_000,
+      sampleIntervalMs: 30_000,
+      settleSampleCount: 20,
+      lcpObservationMs: 5_000,
+      lcpStabilityMs: 500,
+      widgetDataQuietMs: 500,
+      spotlightIdleMs: 2_500,
+      spotlightIdlePolicy: side === "baseline" ? "mounted" : "preload-only",
+      interactionIterations: 7,
+      widgetCount: 32,
+      implementationMarkerPolicy: "true",
+      widgetImplementationMarkerCount: 32,
+      widgetErrorCount: 0,
+      startToReadyMs: 1_000,
+      coldSpotlightFeedbackMs: 20,
+      coldSpotlightReadyMs: 200,
+      postIdleWarmInteractionMs: 30,
+      routeTimings: [{ path: "/manage" }],
+      pageLoadSamples: Array.from({ length: 20 }, () => runtimeSample(side === "baseline" ? 200 : 100)),
+    },
+    checkpoints: [
+      { capturedAt: new Date(0).toISOString(), name: "startup" },
+      { capturedAt: new Date(1_000).toISOString(), name: "workload" },
+      ...Array.from({ length: 20 }, (_, index) => ({
+        capturedAt: new Date(1_000 + (index + 1) * 30_000).toISOString(),
+        name: `settle-${index + 1}`,
+      })),
+    ],
+    summary: { startupBytes: 1, workloadBytes: 2, settledBytes: 2, peakBytes: 3 },
+  };
+};
+
+const validInput = (): BenchmarkPairInput => ({
+  baselineWarmup: warmup("baseline"),
+  candidateWarmup: warmup("candidate"),
+  baselineBuilds: [build("baseline", 1), build("baseline", 2)],
+  candidateBuilds: [build("candidate", 1), build("candidate", 2)],
+  baselineRuntime: runtime("baseline"),
+  candidateRuntime: runtime("candidate"),
+});
+
+describe("benchmark pair validation", () => {
+  it("accepts a complete, isolated, provenance-matched comparison", () => {
+    const result = compareBenchmarkPair(validInput());
+
+    expect(result.claimEligible).toBe(true);
+    expect(result.claimIneligibleReasons).toEqual([]);
+    expect(result.runtime.lcpMs.comparison).toMatchObject({ baseline: 200, candidate: 100, improvementPercent: 50 });
+  });
+
+  it("rejects an incomplete build series", () => {
+    const input = validInput();
+    input.candidateBuilds.pop();
+
+    expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate build repetitions are incomplete");
+  });
+
+  it("rejects runtime source drift and uncontrolled server egress", () => {
+    const input = validInput();
+    input.candidateRuntime.image.sourceFingerprint = "wrong";
+    input.candidateRuntime.serverNetworkIsolation.canary.directIp.blocked = false;
+
+    const result = compareBenchmarkPair(input);
+    expect(result.claimEligible).toBe(false);
+    expect(result.claimIneligibleReasons).toContain(
+      "candidate runtime image does not match the candidate build source",
+    );
+    expect(result.claimIneligibleReasons).toContain("candidate server network is not isolated");
+  });
+
+  it("rejects runtime evidence without blocked HTTP, WS, and WSS browser canaries", () => {
+    const input = validInput();
+    const wssCanary = input.candidateRuntime.browserNetworkIsolation.canaries.webSockets[1];
+    if (!wssCanary) throw new Error("Missing WSS canary fixture");
+    wssCanary.opened = true;
+
+    expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate browser network is not isolated");
+  });
+
+  it("requires one WS and one WSS browser isolation canary", () => {
+    const input = validInput();
+    const wssCanary = input.candidateRuntime.browserNetworkIsolation.canaries.webSockets[1];
+    if (!wssCanary) throw new Error("Missing WSS canary fixture");
+    wssCanary.url = "ws://second-browser-ws.invalid/canary";
+    wssCanary.connectTarget = "second-browser-ws.invalid:80";
+
+    expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate browser network is not isolated");
+  });
+
+  it("recomputes runtime claim minima instead of trusting embedded eligibility flags", () => {
+    const input = validInput();
+    input.candidateRuntime.claimScope = [];
+    input.candidateRuntime.workload.lcpObservationMs = 1_000;
+    input.candidateRuntime.workload.widgetDataQuietMs = 100;
+    input.candidateRuntime.workload.settleMs = 1_000;
+    input.candidateRuntime.workload.settleSampleCount = 19;
+    input.candidateRuntime.checkpoints.pop();
+    const firstSample = input.candidateRuntime.workload.pageLoadSamples[0];
+    if (!firstSample) throw new Error("Missing runtime sample fixture");
+    firstSample.lcpMs = Number.POSITIVE_INFINITY;
+
+    const reasons = compareBenchmarkPair(input).claimIneligibleReasons;
+    expect(reasons).toContain("candidate runtime claim scope is incomplete");
+    expect(reasons).toContain("candidate runtime LCP observation is below 5000ms");
+    expect(reasons).toContain("candidate runtime widget-data quiet window is below 500ms");
+    expect(reasons).toContain("candidate runtime settle duration is below 600000ms");
+    expect(reasons).toContain("candidate runtime has fewer than 20 settle samples");
+    expect(reasons).toContain("candidate runtime metric lcpMs has invalid or missing samples");
+  });
+
+  it("recomputes measured cache-mount use instead of trusting build eligibility flags", () => {
+    const input = validInput();
+    const firstBuild = input.candidateBuilds[0];
+    const beforeMount = firstBuild?.builderCache.before?.cacheMounts[0];
+    const afterMount = firstBuild?.builderCache.after?.cacheMounts[0];
+    if (!beforeMount || !afterMount) throw new Error("Missing cache mount fixture");
+    afterMount.usageCount = beforeMount.usageCount;
+
+    expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain(
+      "candidate repetition 1 build cache mount pnpm-store usage did not increase",
+    );
+  });
+
+  it("verifies actual settle chronology, LCP finalization, and immutable build fingerprints", () => {
+    const input = validInput();
+    const finalSettle = input.candidateRuntime.checkpoints.at(-1);
+    const firstSample = input.candidateRuntime.workload.pageLoadSamples[0];
+    const firstBuild = input.candidateBuilds[0];
+    if (!finalSettle || !firstSample || !firstBuild) throw new Error("Missing evidence fixture");
+    finalSettle.capturedAt = new Date(500_000).toISOString();
+    firstSample.lcpFinalizedByInputMs = 4_999;
+    firstBuild.image.sourceFingerprint = "wrong-fingerprint";
+
+    const reasons = compareBenchmarkPair(input).claimIneligibleReasons;
+    expect(reasons).toContain("candidate runtime checkpoint span is shorter than the declared settle duration");
+    expect(reasons).toContain("candidate runtime LCP samples were not observed and finalized for the declared window");
+    expect(reasons).toContain("candidate build image fingerprint does not match its immutable source snapshot");
+  });
+
+  it("rejects a contaminated warmup or cache use outside the measured chain", () => {
+    const input = validInput();
+    if (input.candidateWarmup.builderCache.before) input.candidateWarmup.builderCache.before.recordCount = 1;
+    if (input.baselineBuilds[0]?.builderCache.before) {
+      input.baselineBuilds[0].builderCache.before.recordsFingerprint = "unrecorded-cache-use";
+    }
+
+    const result = compareBenchmarkPair(input);
+    expect(result.claimIneligibleReasons).toContain("candidate warmup builder did not begin empty");
+    expect(result.claimIneligibleReasons).toContain("baseline builder cache changed outside repetition 1");
+  });
+
+  it("rejects overlapping builds and a stale runtime image with copied labels", () => {
+    const input = validInput();
+    const candidateFirst = input.candidateBuilds[0];
+    if (candidateFirst) candidateFirst.run.startedAt = input.baselineBuilds[0]?.run.startedAt ?? "";
+    input.candidateRuntime.image.id = "stale-image-with-matching-labels";
+
+    const result = compareBenchmarkPair(input);
+    expect(result.claimIneligibleReasons).toContain("build runs overlap between baseline:1 and candidate:1");
+    expect(result.claimIneligibleReasons).toContain(
+      "candidate runtime image identity does not match the candidate warmup image",
+    );
+  });
+});