perf: optimize dashboard loading and builds
This commit is contained in:
135 files changed
+8521
-1718
No files matched your search
@@ -0,0 +1,40 @@
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { BenchmarkPairInput, BuildResult, RuntimeResult } from "./benchmark-pair-lib.mts";
|
||||
import { compareBenchmarkPair } from "./benchmark-pair-lib.mts";
|
||||
|
||||
const values = new Map<string, string[]>();
|
||||
for (let index = 2; index < process.argv.length; index += 2) {
|
||||
const flag = process.argv[index];
|
||||
const value = process.argv[index + 1];
|
||||
if (!flag?.startsWith("--") || !value) throw new Error(`Invalid argument at ${String(flag)}`);
|
||||
values.set(flag, [...(values.get(flag) ?? []), value]);
|
||||
}
|
||||
|
||||
const requireMany = (flag: string) => {
|
||||
const result = values.get(flag) ?? [];
|
||||
if (result.length === 0) throw new Error(`${flag} is required`);
|
||||
return result;
|
||||
};
|
||||
const requireOne = (flag: string) => {
|
||||
const result = requireMany(flag);
|
||||
if (result.length !== 1) throw new Error(`${flag} must be provided exactly once`);
|
||||
return result[0] as string;
|
||||
};
|
||||
const readJson = async <T,>(file: string) => JSON.parse(await readFile(path.resolve(file), "utf8")) as T;
|
||||
|
||||
const input: BenchmarkPairInput = {
|
||||
baselineWarmup: await readJson<BuildResult>(requireOne("--baseline-warmup")),
|
||||
candidateWarmup: await readJson<BuildResult>(requireOne("--candidate-warmup")),
|
||||
baselineBuilds: await Promise.all(requireMany("--baseline-build").map(readJson<BuildResult>)),
|
||||
candidateBuilds: await Promise.all(requireMany("--candidate-build").map(readJson<BuildResult>)),
|
||||
baselineRuntime: await readJson<RuntimeResult>(requireOne("--baseline-runtime")),
|
||||
candidateRuntime: await readJson<RuntimeResult>(requireOne("--candidate-runtime")),
|
||||
};
|
||||
const result = compareBenchmarkPair(input);
|
||||
const outputPath = path.resolve(requireOne("--output"));
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
|
||||
console.log(JSON.stringify({ outputPath, ...result }, null, 2));
|
||||
if (!result.claimEligible) process.exitCode = 1;
|
||||
@@ -0,0 +1,713 @@
|
||||
import { summarize } from "./dev-benchmark-lib.mts";
|
||||
|
||||
type BuildResult = {
|
||||
schemaVersion: number;
|
||||
createdAt: string;
|
||||
run: { startedAt: string; finishedAt: string };
|
||||
claimEligible: boolean;
|
||||
claimIneligibleReasons: string[];
|
||||
mode: string;
|
||||
builder: string | null;
|
||||
runLabel: string | null;
|
||||
declaredRevision: string | null;
|
||||
series: { id: string | null; cacheState: string | null; repetition: number; totalRepetitions: number };
|
||||
source: {
|
||||
headSha: string;
|
||||
measuredRevision: string;
|
||||
fingerprint: string;
|
||||
baseFingerprint: string;
|
||||
syntheticSourceChange: { file: string; id: string } | null;
|
||||
};
|
||||
host: {
|
||||
architecture: string;
|
||||
dockerServer: { architecture: string; operatingSystem: string; targetPlatform: string; version: string | null };
|
||||
platform: string;
|
||||
};
|
||||
image: {
|
||||
architecture: string;
|
||||
digest: string | null;
|
||||
id: string;
|
||||
revision: string | null;
|
||||
sourceFingerprint: string | null;
|
||||
targetPlatform: string;
|
||||
};
|
||||
builderCache: {
|
||||
expected: Array<{ logicalId: string; target: string; mustIncreaseUsage: boolean }>;
|
||||
before: BuildCacheSnapshot | null;
|
||||
after: BuildCacheSnapshot | null;
|
||||
afterStability: { attempts: number; pollIntervalMs: number; stableForMs: number; timeoutMs: number } | null;
|
||||
};
|
||||
build: { nextCompileMs: number | null; wallTimeMs: number };
|
||||
};
|
||||
|
||||
type BuildCacheSnapshot = {
|
||||
cacheMountCount: number;
|
||||
cacheMounts: Array<{
|
||||
logicalId: string | null;
|
||||
recordId: string;
|
||||
target: string | null;
|
||||
usageCount: number | null;
|
||||
}>;
|
||||
recordCount: number;
|
||||
recordsFingerprint: string;
|
||||
};
|
||||
|
||||
type RuntimePageSample = {
|
||||
boardMountedMs: number;
|
||||
fcpMs: number;
|
||||
lcpFinalizedByInputMs: number;
|
||||
lcpMs: number;
|
||||
ttfbMs: number;
|
||||
widgetImplementationsMountedMs: number | null;
|
||||
widgetDataSettledMs: number;
|
||||
};
|
||||
|
||||
type RuntimeResult = {
|
||||
schemaVersion: number;
|
||||
claimEligible: boolean;
|
||||
claimIneligibleReasons: string[];
|
||||
claimScope: string[];
|
||||
metricEligibility: Record<string, { claimEligible: boolean }>;
|
||||
image: {
|
||||
architecture: string;
|
||||
digest: string | null;
|
||||
id: string;
|
||||
revision: string | null;
|
||||
sourceFingerprint: string | null;
|
||||
};
|
||||
host: {
|
||||
architecture: string;
|
||||
node: { version: string; v8Version: string };
|
||||
platform: string;
|
||||
release: string;
|
||||
};
|
||||
provenance: {
|
||||
browser: {
|
||||
engine: string;
|
||||
executablePath: string;
|
||||
headless: boolean;
|
||||
httpCachePolicy: string;
|
||||
launchArguments?: string[];
|
||||
version: string;
|
||||
viewport: unknown;
|
||||
};
|
||||
container: {
|
||||
cpuLimit: number;
|
||||
memoryLimitBytes: number;
|
||||
network: { internal: boolean };
|
||||
node: { architecture: string; platform: string; version: string; v8Version: string };
|
||||
};
|
||||
docker: { engine: unknown; version: unknown };
|
||||
ingressProxy: { excludedFromMemorySamples: boolean; implementation: string };
|
||||
};
|
||||
browserNetworkIsolation: {
|
||||
active: boolean;
|
||||
canaries: {
|
||||
http: { attemptCount: number; responseHeader: string | null; status: number | null };
|
||||
webSockets: Array<{
|
||||
attemptCount: number;
|
||||
browserEventCount: number;
|
||||
closeCount: number;
|
||||
connectTarget: string;
|
||||
errorEvent: boolean;
|
||||
frameReceivedCount: number;
|
||||
frameSentCount: number;
|
||||
opened: boolean;
|
||||
socketErrorCount: number;
|
||||
timedOut: boolean;
|
||||
url: string;
|
||||
}>;
|
||||
};
|
||||
externalBrowserWebSockets: unknown[];
|
||||
unexpectedExternalResponses: unknown[];
|
||||
};
|
||||
serverNetworkIsolation: {
|
||||
active: boolean;
|
||||
canary: { directIp: { blocked: boolean }; domain: { blocked: boolean } };
|
||||
network: { internal: boolean };
|
||||
};
|
||||
checkpoints: Array<{ capturedAt: string; name: string }>;
|
||||
workload: {
|
||||
settleMs: number;
|
||||
sampleIntervalMs: number;
|
||||
settleSampleCount: number;
|
||||
lcpObservationMs: number;
|
||||
lcpStabilityMs: number;
|
||||
widgetDataQuietMs: number;
|
||||
spotlightIdleMs: number;
|
||||
spotlightIdlePolicy: string;
|
||||
interactionIterations: number;
|
||||
widgetCount: number;
|
||||
implementationMarkerPolicy: string;
|
||||
widgetImplementationMarkerCount: number;
|
||||
widgetErrorCount: number;
|
||||
startToReadyMs: number;
|
||||
coldSpotlightFeedbackMs: number | null;
|
||||
coldSpotlightReadyMs: number | null;
|
||||
postIdleWarmInteractionMs: number | null;
|
||||
routeTimings: Array<{ path: string }>;
|
||||
pageLoadSamples: RuntimePageSample[];
|
||||
};
|
||||
summary: {
|
||||
startupBytes: number | null;
|
||||
workloadBytes: number | null;
|
||||
settledBytes: number | null;
|
||||
peakBytes: number | null;
|
||||
};
|
||||
};
|
||||
|
||||
export type BenchmarkPairInput = {
|
||||
baselineWarmup: BuildResult;
|
||||
candidateWarmup: BuildResult;
|
||||
baselineBuilds: BuildResult[];
|
||||
candidateBuilds: BuildResult[];
|
||||
baselineRuntime: RuntimeResult;
|
||||
candidateRuntime: RuntimeResult;
|
||||
};
|
||||
|
||||
const stable = (value: unknown) => JSON.stringify(value);
|
||||
const unique = <T,>(values: T[]) => [...new Set(values)];
|
||||
|
||||
const improvement = (baseline: number | null, candidate: number | null) => ({
|
||||
baseline,
|
||||
candidate,
|
||||
delta: baseline === null || candidate === null ? null : candidate - baseline,
|
||||
improvementPercent:
|
||||
baseline === null || candidate === null || baseline === 0 ? null : ((baseline - candidate) / baseline) * 100,
|
||||
});
|
||||
|
||||
const validateRunWindow = (label: string, build: BuildResult, reasons: string[]) => {
|
||||
const startedAt = Date.parse(build.run.startedAt);
|
||||
const finishedAt = Date.parse(build.run.finishedAt);
|
||||
if (!Number.isFinite(startedAt) || !Number.isFinite(finishedAt) || finishedAt < startedAt) {
|
||||
reasons.push(`${label} build run window is invalid`);
|
||||
}
|
||||
if (build.createdAt !== build.run.finishedAt) reasons.push(`${label} build completion timestamp is inconsistent`);
|
||||
};
|
||||
|
||||
const validateMeasuredBuildCache = (label: string, build: BuildResult, reasons: string[]) => {
|
||||
const { after, before, expected } = build.builderCache;
|
||||
if (expected.length === 0) {
|
||||
reasons.push(`${label} build declares no expected cache mounts`);
|
||||
return;
|
||||
}
|
||||
if (!before || !after) {
|
||||
reasons.push(`${label} build cache continuity snapshots are missing`);
|
||||
return;
|
||||
}
|
||||
for (const expectedMount of expected) {
|
||||
const matches = (snapshot: BuildCacheSnapshot) =>
|
||||
snapshot.cacheMounts.filter(
|
||||
({ logicalId, target }) => logicalId === expectedMount.logicalId && target === expectedMount.target,
|
||||
);
|
||||
const beforeMatches = matches(before);
|
||||
const afterMatches = matches(after);
|
||||
if (beforeMatches.length !== 1 || afterMatches.length !== 1) {
|
||||
reasons.push(`${label} build cache mount ${expectedMount.logicalId} cannot be traced uniquely`);
|
||||
continue;
|
||||
}
|
||||
const beforeMount = beforeMatches[0];
|
||||
const afterMount = afterMatches[0];
|
||||
if (!beforeMount || !afterMount || beforeMount.recordId !== afterMount.recordId) {
|
||||
reasons.push(`${label} build cache mount ${expectedMount.logicalId} was replaced`);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
expectedMount.mustIncreaseUsage &&
|
||||
(beforeMount.usageCount === null ||
|
||||
afterMount.usageCount === null ||
|
||||
afterMount.usageCount <= beforeMount.usageCount)
|
||||
) {
|
||||
reasons.push(`${label} build cache mount ${expectedMount.logicalId} usage did not increase`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const validateBuildSide = (label: string, builds: BuildResult[], reasons: string[]) => {
|
||||
if (builds.length === 0) {
|
||||
reasons.push(`${label} build series is empty`);
|
||||
return;
|
||||
}
|
||||
|
||||
const first = builds[0];
|
||||
if (!first) return;
|
||||
for (const build of builds) {
|
||||
if (build.schemaVersion !== 4) reasons.push(`${label} build schema is not 4`);
|
||||
validateRunWindow(`${label} repetition ${build.series.repetition}`, build, reasons);
|
||||
if (!build.claimEligible || build.claimIneligibleReasons.length > 0) {
|
||||
reasons.push(`${label} build repetition ${build.series.repetition} is not claim eligible`);
|
||||
}
|
||||
if (build.mode !== "source-change") reasons.push(`${label} build is not a source-change rebuild`);
|
||||
if (build.runLabel !== label) reasons.push(`${label} build run label is not ${label}`);
|
||||
if (build.series.cacheState !== "warm") reasons.push(`${label} build cache is not warm`);
|
||||
if (!build.source.syntheticSourceChange) reasons.push(`${label} build has no synthetic source change`);
|
||||
if (build.source.fingerprint === build.source.baseFingerprint) {
|
||||
reasons.push(`${label} build synthetic source fingerprint did not change`);
|
||||
}
|
||||
if (build.image.sourceFingerprint !== build.source.fingerprint) {
|
||||
reasons.push(`${label} build image fingerprint does not match its immutable source snapshot`);
|
||||
}
|
||||
if (!Number.isFinite(build.build.nextCompileMs) || (build.build.nextCompileMs ?? 0) <= 0) {
|
||||
reasons.push(`${label} build has no valid Next compile timing`);
|
||||
}
|
||||
if (!Number.isFinite(build.build.wallTimeMs) || build.build.wallTimeMs <= 0) {
|
||||
reasons.push(`${label} build has no valid wall timing`);
|
||||
}
|
||||
validateMeasuredBuildCache(`${label} repetition ${build.series.repetition}`, build, reasons);
|
||||
if (!build.builderCache.afterStability || build.builderCache.afterStability.stableForMs < 10_000) {
|
||||
reasons.push(`${label} build cache snapshot did not settle for 10000ms`);
|
||||
}
|
||||
if (!Number.isFinite(Date.parse(build.createdAt))) reasons.push(`${label} build timestamp is invalid`);
|
||||
}
|
||||
|
||||
const expectedRepetitions = Array.from({ length: first.series.totalRepetitions }, (_, index) => index + 1);
|
||||
const actualRepetitions = builds.map(({ series }) => series.repetition).toSorted((left, right) => left - right);
|
||||
if (first.series.totalRepetitions < 2 || stable(actualRepetitions) !== stable(expectedRepetitions)) {
|
||||
reasons.push(`${label} build repetitions are incomplete`);
|
||||
}
|
||||
|
||||
const stableFields: Array<[string, unknown[]]> = [
|
||||
["builder", builds.map(({ builder }) => builder)],
|
||||
["series id", builds.map(({ series }) => series.id)],
|
||||
["total repetitions", builds.map(({ series }) => series.totalRepetitions)],
|
||||
["declared revision", builds.map(({ declaredRevision }) => declaredRevision)],
|
||||
["head SHA", builds.map(({ source }) => source.headSha)],
|
||||
["measured revision", builds.map(({ source }) => source.measuredRevision)],
|
||||
["base fingerprint", builds.map(({ source }) => source.baseFingerprint)],
|
||||
["host", builds.map(({ host }) => stable(host))],
|
||||
["image platform", builds.map(({ image }) => `${image.architecture}:${image.targetPlatform}`)],
|
||||
["expected cache mounts", builds.map(({ builderCache }) => stable(builderCache.expected))],
|
||||
];
|
||||
for (const [field, values] of stableFields) {
|
||||
if (unique(values).length !== 1) reasons.push(`${label} build ${field} changes within the series`);
|
||||
}
|
||||
if (!first.builder) reasons.push(`${label} build builder is missing`);
|
||||
if (!first.series.id) reasons.push(`${label} build series id is missing`);
|
||||
};
|
||||
|
||||
const validateWarmup = (label: string, warmup: BuildResult, builds: BuildResult[], reasons: string[]) => {
|
||||
if (warmup.schemaVersion !== 4) reasons.push(`${label} warmup schema is not 4`);
|
||||
validateRunWindow(`${label} warmup`, warmup, reasons);
|
||||
if (warmup.mode !== "cached") reasons.push(`${label} warmup mode is not cached`);
|
||||
if (warmup.runLabel !== label) reasons.push(`${label} warmup run label is not ${label}`);
|
||||
if (warmup.series.cacheState !== "cold") reasons.push(`${label} warmup cache state is not cold`);
|
||||
if (warmup.source.syntheticSourceChange !== null) reasons.push(`${label} warmup contains a synthetic source change`);
|
||||
const expectedWarmupReason = "cached mode does not force an application rebuild";
|
||||
if (
|
||||
warmup.claimEligible ||
|
||||
!warmup.claimIneligibleReasons.includes(expectedWarmupReason) ||
|
||||
warmup.claimIneligibleReasons.some((reason) => reason !== expectedWarmupReason)
|
||||
) {
|
||||
reasons.push(`${label} warmup has unexpected claim eligibility reasons`);
|
||||
}
|
||||
if (!warmup.builderCache.before || warmup.builderCache.before.recordCount !== 0) {
|
||||
reasons.push(`${label} warmup builder did not begin empty`);
|
||||
}
|
||||
if (!warmup.builderCache.after || warmup.builderCache.after.recordCount === 0) {
|
||||
reasons.push(`${label} warmup did not populate the builder cache`);
|
||||
}
|
||||
if (!warmup.builderCache.afterStability || warmup.builderCache.afterStability.stableForMs < 10_000) {
|
||||
reasons.push(`${label} warmup cache snapshot did not settle for 10000ms`);
|
||||
}
|
||||
for (const expected of warmup.builderCache.expected) {
|
||||
const matches =
|
||||
warmup.builderCache.after?.cacheMounts.filter(
|
||||
({ logicalId, target }) => logicalId === expected.logicalId && target === expected.target,
|
||||
) ?? [];
|
||||
if (matches.length !== 1) {
|
||||
reasons.push(`${label} warmup did not populate cache mount ${expected.logicalId} at ${expected.target}`);
|
||||
}
|
||||
}
|
||||
if (warmup.image.sourceFingerprint !== warmup.source.baseFingerprint) {
|
||||
reasons.push(`${label} warmup image fingerprint does not match its source`);
|
||||
}
|
||||
if (warmup.source.fingerprint !== warmup.source.baseFingerprint) {
|
||||
reasons.push(`${label} warmup immutable source fingerprint does not match its base source`);
|
||||
}
|
||||
if (warmup.image.revision !== warmup.source.measuredRevision) {
|
||||
reasons.push(`${label} warmup image revision does not match its source`);
|
||||
}
|
||||
|
||||
const first = builds[0];
|
||||
if (!first) return;
|
||||
const stableFields: Array<[string, unknown, unknown]> = [
|
||||
["builder", warmup.builder, first.builder],
|
||||
["series id", warmup.series.id, first.series.id],
|
||||
["declared revision", warmup.declaredRevision, first.declaredRevision],
|
||||
["head SHA", warmup.source.headSha, first.source.headSha],
|
||||
["measured revision", warmup.source.measuredRevision, first.source.measuredRevision],
|
||||
["base fingerprint", warmup.source.baseFingerprint, first.source.baseFingerprint],
|
||||
["host", stable(warmup.host), stable(first.host)],
|
||||
[
|
||||
"image platform",
|
||||
`${warmup.image.architecture}:${warmup.image.targetPlatform}`,
|
||||
`${first.image.architecture}:${first.image.targetPlatform}`,
|
||||
],
|
||||
["expected cache mounts", stable(warmup.builderCache.expected), stable(first.builderCache.expected)],
|
||||
];
|
||||
for (const [field, warmupValue, measuredValue] of stableFields) {
|
||||
if (warmupValue !== measuredValue) reasons.push(`${label} warmup ${field} does not match the measured series`);
|
||||
}
|
||||
|
||||
let previous = warmup;
|
||||
for (const build of builds.toSorted((left, right) => left.series.repetition - right.series.repetition)) {
|
||||
if (!previous.builderCache.after || !build.builderCache.before) {
|
||||
reasons.push(`${label} cache continuity snapshot is missing before repetition ${build.series.repetition}`);
|
||||
} else if (previous.builderCache.after.recordsFingerprint !== build.builderCache.before.recordsFingerprint) {
|
||||
reasons.push(`${label} builder cache changed outside repetition ${build.series.repetition}`);
|
||||
}
|
||||
previous = build;
|
||||
}
|
||||
};
|
||||
|
||||
const runtimeConfiguration = (runtime: RuntimeResult) => ({
|
||||
host: runtime.host,
|
||||
browser: {
|
||||
...runtime.provenance.browser,
|
||||
launchArguments: runtime.provenance.browser.launchArguments?.map((argument) =>
|
||||
argument.startsWith("--proxy-server=") ? "--proxy-server=<ephemeral-deny-proxy>" : argument,
|
||||
),
|
||||
},
|
||||
container: {
|
||||
cpuLimit: runtime.provenance.container.cpuLimit,
|
||||
memoryLimitBytes: runtime.provenance.container.memoryLimitBytes,
|
||||
node: runtime.provenance.container.node,
|
||||
},
|
||||
docker: runtime.provenance.docker,
|
||||
workload: {
|
||||
settleMs: runtime.workload.settleMs,
|
||||
sampleIntervalMs: runtime.workload.sampleIntervalMs,
|
||||
settleSampleCount: runtime.workload.settleSampleCount,
|
||||
lcpObservationMs: runtime.workload.lcpObservationMs,
|
||||
lcpStabilityMs: runtime.workload.lcpStabilityMs,
|
||||
widgetDataQuietMs: runtime.workload.widgetDataQuietMs,
|
||||
spotlightIdleMs: runtime.workload.spotlightIdleMs,
|
||||
interactionIterations: runtime.workload.interactionIterations,
|
||||
widgetCount: runtime.workload.widgetCount,
|
||||
implementationMarkerPolicy: runtime.workload.implementationMarkerPolicy,
|
||||
routePaths: runtime.workload.routeTimings.map(({ path }) => path),
|
||||
pageSampleCount: runtime.workload.pageLoadSamples.length,
|
||||
},
|
||||
});
|
||||
|
||||
const validateRuntime = (label: string, runtime: RuntimeResult, reasons: string[]) => {
|
||||
const expectedClaimScope = ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"];
|
||||
if (runtime.schemaVersion !== 7) reasons.push(`${label} runtime schema is not 7`);
|
||||
if (stable(runtime.claimScope) !== stable(expectedClaimScope)) {
|
||||
reasons.push(`${label} runtime claim scope is incomplete`);
|
||||
}
|
||||
if (!runtime.claimEligible || runtime.claimIneligibleReasons.length > 0) {
|
||||
reasons.push(`${label} runtime is not claim eligible`);
|
||||
}
|
||||
for (const metric of ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"]) {
|
||||
if (runtime.metricEligibility[metric]?.claimEligible !== true) {
|
||||
reasons.push(`${label} runtime metric ${metric} is not claim eligible`);
|
||||
}
|
||||
}
|
||||
if (runtime.workload.pageLoadSamples.length < 20) reasons.push(`${label} runtime has fewer than 20 page samples`);
|
||||
if (runtime.workload.lcpObservationMs < 5_000) reasons.push(`${label} runtime LCP observation is below 5000ms`);
|
||||
if (runtime.workload.widgetDataQuietMs < 500)
|
||||
reasons.push(`${label} runtime widget-data quiet window is below 500ms`);
|
||||
if (runtime.workload.settleMs < 600_000) reasons.push(`${label} runtime settle duration is below 600000ms`);
|
||||
const actualSettleSampleCount = runtime.checkpoints.filter(({ name }) => name.startsWith("settle-")).length;
|
||||
if (runtime.workload.settleSampleCount < 20 || actualSettleSampleCount < 20) {
|
||||
reasons.push(`${label} runtime has fewer than 20 settle samples`);
|
||||
}
|
||||
if (runtime.workload.settleSampleCount !== actualSettleSampleCount) {
|
||||
reasons.push(`${label} runtime settle sample count does not match checkpoints`);
|
||||
}
|
||||
const workloadCheckpoints = runtime.checkpoints.filter(({ name }) => name === "workload");
|
||||
const settleCheckpoints = runtime.checkpoints.filter(({ name }) => name.startsWith("settle-"));
|
||||
const expectedSettleNames = Array.from({ length: actualSettleSampleCount }, (_, index) => `settle-${index + 1}`);
|
||||
if (
|
||||
workloadCheckpoints.length !== 1 ||
|
||||
stable(settleCheckpoints.map(({ name }) => name)) !== stable(expectedSettleNames) ||
|
||||
runtime.checkpoints.some(({ capturedAt }) => !Number.isFinite(Date.parse(capturedAt))) ||
|
||||
runtime.checkpoints.some(
|
||||
({ capturedAt }, index, checkpoints) =>
|
||||
index > 0 && Date.parse(capturedAt) < Date.parse(checkpoints[index - 1]?.capturedAt ?? ""),
|
||||
)
|
||||
) {
|
||||
reasons.push(`${label} runtime checkpoint chronology is invalid`);
|
||||
}
|
||||
const workloadCapturedAt = Date.parse(workloadCheckpoints[0]?.capturedAt ?? "");
|
||||
const finalSettleCapturedAt = Date.parse(settleCheckpoints.at(-1)?.capturedAt ?? "");
|
||||
if (
|
||||
!Number.isFinite(workloadCapturedAt) ||
|
||||
!Number.isFinite(finalSettleCapturedAt) ||
|
||||
finalSettleCapturedAt - workloadCapturedAt < runtime.workload.settleMs
|
||||
) {
|
||||
reasons.push(`${label} runtime checkpoint span is shorter than the declared settle duration`);
|
||||
}
|
||||
for (const metric of [
|
||||
"boardMountedMs",
|
||||
"fcpMs",
|
||||
"lcpFinalizedByInputMs",
|
||||
"lcpMs",
|
||||
"ttfbMs",
|
||||
"widgetImplementationsMountedMs",
|
||||
"widgetDataSettledMs",
|
||||
] satisfies Array<keyof RuntimePageSample>) {
|
||||
const values = runtime.workload.pageLoadSamples.map((sample) => sample[metric]);
|
||||
if (
|
||||
values.some((value) => typeof value !== "number" || !Number.isFinite(value) || value < 0) ||
|
||||
runtimeMetric(runtime, metric).count !== runtime.workload.pageLoadSamples.length
|
||||
) {
|
||||
reasons.push(`${label} runtime metric ${metric} has invalid or missing samples`);
|
||||
}
|
||||
}
|
||||
if (
|
||||
runtime.workload.pageLoadSamples.some(
|
||||
({ lcpFinalizedByInputMs, lcpMs }) =>
|
||||
lcpFinalizedByInputMs < runtime.workload.lcpObservationMs || lcpMs > lcpFinalizedByInputMs,
|
||||
)
|
||||
) {
|
||||
reasons.push(`${label} runtime LCP samples were not observed and finalized for the declared window`);
|
||||
}
|
||||
if (runtime.workload.widgetErrorCount !== 0) reasons.push(`${label} runtime rendered widget errors`);
|
||||
if (runtime.workload.widgetImplementationMarkerCount !== runtime.workload.widgetCount) {
|
||||
reasons.push(`${label} runtime did not mount every widget implementation`);
|
||||
}
|
||||
const webSocketCanaryProtocols = runtime.browserNetworkIsolation.canaries.webSockets.flatMap(
|
||||
({ connectTarget, url }) => {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const defaultPort = parsed.protocol === "ws:" ? "80" : parsed.protocol === "wss:" ? "443" : null;
|
||||
return defaultPort && connectTarget === `${parsed.hostname}:${parsed.port || defaultPort}`
|
||||
? [parsed.protocol]
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
);
|
||||
if (
|
||||
!runtime.browserNetworkIsolation.active ||
|
||||
runtime.browserNetworkIsolation.unexpectedExternalResponses.length > 0 ||
|
||||
runtime.browserNetworkIsolation.externalBrowserWebSockets.length > 0 ||
|
||||
runtime.browserNetworkIsolation.canaries.http.status !== 502 ||
|
||||
runtime.browserNetworkIsolation.canaries.http.responseHeader !== "true" ||
|
||||
runtime.browserNetworkIsolation.canaries.http.attemptCount < 1 ||
|
||||
runtime.browserNetworkIsolation.canaries.webSockets.length !== 2 ||
|
||||
stable(webSocketCanaryProtocols.toSorted()) !== stable(["ws:", "wss:"]) ||
|
||||
runtime.browserNetworkIsolation.canaries.webSockets.some(
|
||||
({
|
||||
attemptCount,
|
||||
browserEventCount,
|
||||
closeCount,
|
||||
errorEvent,
|
||||
frameReceivedCount,
|
||||
frameSentCount,
|
||||
opened,
|
||||
socketErrorCount,
|
||||
timedOut,
|
||||
}) =>
|
||||
attemptCount < 1 ||
|
||||
browserEventCount < 1 ||
|
||||
closeCount < 1 ||
|
||||
!errorEvent ||
|
||||
frameReceivedCount !== 0 ||
|
||||
frameSentCount !== 0 ||
|
||||
opened ||
|
||||
socketErrorCount < 1 ||
|
||||
timedOut,
|
||||
)
|
||||
) {
|
||||
reasons.push(`${label} browser network is not isolated`);
|
||||
}
|
||||
if (
|
||||
!runtime.serverNetworkIsolation.active ||
|
||||
!runtime.serverNetworkIsolation.network.internal ||
|
||||
!runtime.serverNetworkIsolation.canary.domain.blocked ||
|
||||
!runtime.serverNetworkIsolation.canary.directIp.blocked
|
||||
) {
|
||||
reasons.push(`${label} server network is not isolated`);
|
||||
}
|
||||
if (!runtime.provenance.container.network.internal) reasons.push(`${label} app container network is not internal`);
|
||||
if (!runtime.provenance.ingressProxy.excludedFromMemorySamples) {
|
||||
reasons.push(`${label} ingress proxy memory accounting is ambiguous`);
|
||||
}
|
||||
};
|
||||
|
||||
const runtimeMetric = (runtime: RuntimeResult, key: keyof RuntimePageSample) =>
|
||||
summarize(runtime.workload.pageLoadSamples.map((sample) => sample[key]));
|
||||
|
||||
export const compareBenchmarkPair = (input: BenchmarkPairInput) => {
|
||||
const reasons: string[] = [];
|
||||
validateBuildSide("baseline", input.baselineBuilds, reasons);
|
||||
validateBuildSide("candidate", input.candidateBuilds, reasons);
|
||||
validateWarmup("baseline", input.baselineWarmup, input.baselineBuilds, reasons);
|
||||
validateWarmup("candidate", input.candidateWarmup, input.candidateBuilds, reasons);
|
||||
validateRuntime("baseline", input.baselineRuntime, reasons);
|
||||
validateRuntime("candidate", input.candidateRuntime, reasons);
|
||||
|
||||
const baselineBuild = input.baselineBuilds[0];
|
||||
const candidateBuild = input.candidateBuilds[0];
|
||||
if (baselineBuild && candidateBuild) {
|
||||
if (baselineBuild.series.id !== candidateBuild.series.id) reasons.push("build series ids do not match");
|
||||
if (baselineBuild.builder === candidateBuild.builder) reasons.push("build sides reuse the same builder cache");
|
||||
if (baselineBuild.source.measuredRevision !== candidateBuild.source.measuredRevision) {
|
||||
reasons.push("build measured revisions do not match");
|
||||
}
|
||||
if (stable(baselineBuild.host) !== stable(candidateBuild.host)) reasons.push("build hosts do not match");
|
||||
if (
|
||||
`${baselineBuild.image.architecture}:${baselineBuild.image.targetPlatform}` !==
|
||||
`${candidateBuild.image.architecture}:${candidateBuild.image.targetPlatform}`
|
||||
) {
|
||||
reasons.push("build image platforms do not match");
|
||||
}
|
||||
if (baselineBuild.source.baseFingerprint === candidateBuild.source.baseFingerprint) {
|
||||
reasons.push("baseline and candidate source fingerprints are identical");
|
||||
}
|
||||
|
||||
const candidateByRepetition = new Map(
|
||||
input.candidateBuilds.map((build) => [build.series.repetition, build.source.syntheticSourceChange?.id]),
|
||||
);
|
||||
for (const build of input.baselineBuilds) {
|
||||
if (build.source.syntheticSourceChange?.id !== candidateByRepetition.get(build.series.repetition)) {
|
||||
reasons.push(`synthetic source change differs at repetition ${build.series.repetition}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (baselineBuild.series.totalRepetitions !== 2 || candidateBuild.series.totalRepetitions !== 2) {
|
||||
reasons.push("balanced build order currently requires exactly two repetitions per side");
|
||||
} else {
|
||||
const chronologicalRuns = [
|
||||
{ build: input.baselineWarmup, label: "baseline:warmup" },
|
||||
{ build: input.candidateWarmup, label: "candidate:warmup" },
|
||||
...input.baselineBuilds.map((build) => ({ build, label: `baseline:${build.series.repetition}` })),
|
||||
...input.candidateBuilds.map((build) => ({ build, label: `candidate:${build.series.repetition}` })),
|
||||
].toSorted((left, right) => Date.parse(left.build.run.startedAt) - Date.parse(right.build.run.startedAt));
|
||||
const chronologicalOrder = chronologicalRuns.map(({ label }) => label);
|
||||
if (
|
||||
stable(chronologicalOrder) !==
|
||||
stable(["baseline:warmup", "candidate:warmup", "baseline:1", "candidate:1", "candidate:2", "baseline:2"])
|
||||
) {
|
||||
reasons.push(
|
||||
"build execution order is not cold-warmup baseline-candidate then balanced baseline-candidate-candidate-baseline",
|
||||
);
|
||||
}
|
||||
for (let index = 1; index < chronologicalRuns.length; index += 1) {
|
||||
const previous = chronologicalRuns[index - 1];
|
||||
const current = chronologicalRuns[index];
|
||||
if (
|
||||
previous &&
|
||||
current &&
|
||||
Date.parse(previous.build.run.finishedAt) > Date.parse(current.build.run.startedAt)
|
||||
) {
|
||||
reasons.push(`build runs overlap between ${previous.label} and ${current.label}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (input.baselineRuntime.image.sourceFingerprint !== baselineBuild.source.baseFingerprint) {
|
||||
reasons.push("baseline runtime image does not match the baseline build source");
|
||||
}
|
||||
if (input.baselineRuntime.image.sourceFingerprint !== input.baselineWarmup.image.sourceFingerprint) {
|
||||
reasons.push("baseline runtime image does not match the baseline warmup image");
|
||||
}
|
||||
if (input.candidateRuntime.image.sourceFingerprint !== input.candidateWarmup.image.sourceFingerprint) {
|
||||
reasons.push("candidate runtime image does not match the candidate warmup image");
|
||||
}
|
||||
if (
|
||||
input.baselineRuntime.image.id !== input.baselineWarmup.image.id ||
|
||||
input.baselineRuntime.image.digest !== input.baselineWarmup.image.digest
|
||||
) {
|
||||
reasons.push("baseline runtime image identity does not match the baseline warmup image");
|
||||
}
|
||||
if (
|
||||
input.candidateRuntime.image.id !== input.candidateWarmup.image.id ||
|
||||
input.candidateRuntime.image.digest !== input.candidateWarmup.image.digest
|
||||
) {
|
||||
reasons.push("candidate runtime image identity does not match the candidate warmup image");
|
||||
}
|
||||
if (input.candidateRuntime.image.sourceFingerprint !== candidateBuild.source.baseFingerprint) {
|
||||
reasons.push("candidate runtime image does not match the candidate build source");
|
||||
}
|
||||
if (
|
||||
input.baselineRuntime.image.revision !== baselineBuild.source.measuredRevision ||
|
||||
input.candidateRuntime.image.revision !== candidateBuild.source.measuredRevision
|
||||
) {
|
||||
reasons.push("runtime image revisions do not match their build revisions");
|
||||
}
|
||||
}
|
||||
|
||||
if (input.baselineRuntime.image.revision !== input.candidateRuntime.image.revision) {
|
||||
reasons.push("runtime image revisions do not match");
|
||||
}
|
||||
if (input.baselineRuntime.image.architecture !== input.candidateRuntime.image.architecture) {
|
||||
reasons.push("runtime image architectures do not match");
|
||||
}
|
||||
if (stable(runtimeConfiguration(input.baselineRuntime)) !== stable(runtimeConfiguration(input.candidateRuntime))) {
|
||||
reasons.push("runtime host, browser, container, or workload configuration does not match");
|
||||
}
|
||||
if (stable(input.baselineRuntime.claimScope) !== stable(input.candidateRuntime.claimScope)) {
|
||||
reasons.push("runtime claim scopes do not match");
|
||||
}
|
||||
if (input.baselineRuntime.workload.spotlightIdlePolicy !== "mounted") {
|
||||
reasons.push("baseline Spotlight idle policy is not mounted");
|
||||
}
|
||||
if (input.candidateRuntime.workload.spotlightIdlePolicy !== "preload-only") {
|
||||
reasons.push("candidate Spotlight idle policy is not preload-only");
|
||||
}
|
||||
|
||||
const buildMetric = (key: "wallTimeMs" | "nextCompileMs") => {
|
||||
const baseline = summarize(input.baselineBuilds.map(({ build }) => build[key]));
|
||||
const candidate = summarize(input.candidateBuilds.map(({ build }) => build[key]));
|
||||
return { baseline, candidate, comparison: improvement(baseline.median, candidate.median) };
|
||||
};
|
||||
const pageMetric = (key: keyof RuntimePageSample) => {
|
||||
const baseline = runtimeMetric(input.baselineRuntime, key);
|
||||
const candidate = runtimeMetric(input.candidateRuntime, key);
|
||||
return { baseline, candidate, comparison: improvement(baseline.median, candidate.median) };
|
||||
};
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
claimEligible: reasons.length === 0,
|
||||
claimIneligibleReasons: unique(reasons),
|
||||
claimScope: {
|
||||
build: "alternating warm incremental Docker source rebuilds",
|
||||
runtime: "fresh authenticated Chromium documents against one warmed, network-isolated app container",
|
||||
},
|
||||
limitations: [
|
||||
"build series report controlled medians, not statistical significance",
|
||||
"runtime documents share one warmed server",
|
||||
"widget settlement covers observed non-subscription tRPC requests followed by the configured quiet window",
|
||||
"startup, RAM levels, initial bundle data, and cold Spotlight timings remain single-container diagnostics",
|
||||
],
|
||||
build: {
|
||||
seriesId: baselineBuild?.series.id ?? null,
|
||||
baselineFingerprint: baselineBuild?.source.baseFingerprint ?? null,
|
||||
candidateFingerprint: candidateBuild?.source.baseFingerprint ?? null,
|
||||
wallTimeMs: buildMetric("wallTimeMs"),
|
||||
nextCompileMs: buildMetric("nextCompileMs"),
|
||||
},
|
||||
runtime: {
|
||||
boardMountedMs: pageMetric("boardMountedMs"),
|
||||
fcpMs: pageMetric("fcpMs"),
|
||||
lcpMs: pageMetric("lcpMs"),
|
||||
ttfbMs: pageMetric("ttfbMs"),
|
||||
widgetImplementationsMountedMs: pageMetric("widgetImplementationsMountedMs"),
|
||||
widgetDataSettledMs: pageMetric("widgetDataSettledMs"),
|
||||
},
|
||||
diagnostics: {
|
||||
baseline: {
|
||||
startToReadyMs: input.baselineRuntime.workload.startToReadyMs,
|
||||
coldSpotlightFeedbackMs: input.baselineRuntime.workload.coldSpotlightFeedbackMs,
|
||||
coldSpotlightReadyMs: input.baselineRuntime.workload.coldSpotlightReadyMs,
|
||||
postIdleFirstOpenMs: input.baselineRuntime.workload.postIdleWarmInteractionMs,
|
||||
memory: input.baselineRuntime.summary,
|
||||
},
|
||||
candidate: {
|
||||
startToReadyMs: input.candidateRuntime.workload.startToReadyMs,
|
||||
coldSpotlightFeedbackMs: input.candidateRuntime.workload.coldSpotlightFeedbackMs,
|
||||
coldSpotlightReadyMs: input.candidateRuntime.workload.coldSpotlightReadyMs,
|
||||
postIdleFirstOpenMs: input.candidateRuntime.workload.postIdleWarmInteractionMs,
|
||||
memory: input.candidateRuntime.summary,
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export type { BuildResult, RuntimeResult };
|
||||
@@ -0,0 +1,360 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { BenchmarkPairInput, BuildResult, RuntimeResult } from "./benchmark-pair-lib.mts";
|
||||
import { compareBenchmarkPair } from "./benchmark-pair-lib.mts";
|
||||
|
||||
const runWindow = (index: number) => {
|
||||
const startedAt = new Date(Date.UTC(2026, 0, 1, 0, 0, index * 2)).toISOString();
|
||||
const finishedAt = new Date(Date.UTC(2026, 0, 1, 0, 0, index * 2 + 1)).toISOString();
|
||||
return { startedAt, finishedAt };
|
||||
};
|
||||
|
||||
const expectedCache = [{ logicalId: "pnpm-store", target: "/pnpm/store", mustIncreaseUsage: true }];
|
||||
const cacheStability = { attempts: 6, pollIntervalMs: 2_000, stableForMs: 10_000, timeoutMs: 30_000 };
|
||||
const cacheSnapshot = (side: "baseline" | "candidate", stage: "empty" | "warm" | "r1" | "r2") => ({
|
||||
cacheMountCount: stage === "empty" ? 0 : 1,
|
||||
cacheMounts:
|
||||
stage === "empty"
|
||||
? []
|
||||
: [
|
||||
{
|
||||
logicalId: "pnpm-store",
|
||||
recordId: `${side}-pnpm-record`,
|
||||
target: "/pnpm/store",
|
||||
usageCount: { warm: 1, r1: 2, r2: 3 }[stage],
|
||||
},
|
||||
],
|
||||
recordCount: stage === "empty" ? 0 : { warm: 10, r1: 12, r2: 14 }[stage],
|
||||
recordsFingerprint: `${side}-${stage}`,
|
||||
});
|
||||
|
||||
const build = (side: "baseline" | "candidate", repetition: number): BuildResult => ({
|
||||
schemaVersion: 4,
|
||||
createdAt: runWindow(side === "baseline" ? (repetition === 1 ? 2 : 5) : repetition === 1 ? 3 : 4).finishedAt,
|
||||
run: runWindow(side === "baseline" ? (repetition === 1 ? 2 : 5) : repetition === 1 ? 3 : 4),
|
||||
claimEligible: true,
|
||||
claimIneligibleReasons: [],
|
||||
mode: "source-change",
|
||||
builder: `${side}-builder`,
|
||||
runLabel: side,
|
||||
declaredRevision: "head+worktree",
|
||||
series: { id: "series", cacheState: "warm", repetition, totalRepetitions: 2 },
|
||||
source: {
|
||||
headSha: "head",
|
||||
measuredRevision: "head+worktree",
|
||||
fingerprint: `${side}-change-${repetition}`,
|
||||
baseFingerprint: `${side}-fingerprint`,
|
||||
syntheticSourceChange: { file: "layout.tsx", id: `change-${repetition}` },
|
||||
},
|
||||
host: {
|
||||
architecture: "arm64",
|
||||
dockerServer: { architecture: "arm64", operatingSystem: "linux", targetPlatform: "linux/arm64", version: "1" },
|
||||
platform: "darwin",
|
||||
},
|
||||
image: {
|
||||
architecture: "arm64",
|
||||
digest: null,
|
||||
id: `${side}-build-${repetition}`,
|
||||
revision: "head+worktree",
|
||||
sourceFingerprint: `${side}-change-${repetition}`,
|
||||
targetPlatform: "linux/arm64",
|
||||
},
|
||||
builderCache: {
|
||||
expected: expectedCache,
|
||||
before: cacheSnapshot(side, repetition === 1 ? "warm" : "r1"),
|
||||
after: cacheSnapshot(side, repetition === 1 ? "r1" : "r2"),
|
||||
afterStability: cacheStability,
|
||||
},
|
||||
build: { nextCompileMs: side === "baseline" ? 200 : 100, wallTimeMs: side === "baseline" ? 300 : 150 },
|
||||
});
|
||||
|
||||
const warmup = (side: "baseline" | "candidate"): BuildResult => {
|
||||
const run = runWindow(side === "baseline" ? 0 : 1);
|
||||
return {
|
||||
schemaVersion: 4,
|
||||
createdAt: run.finishedAt,
|
||||
run,
|
||||
claimEligible: false,
|
||||
claimIneligibleReasons: ["cached mode does not force an application rebuild"],
|
||||
mode: "cached",
|
||||
builder: `${side}-builder`,
|
||||
runLabel: side,
|
||||
declaredRevision: "head+worktree",
|
||||
series: { id: "series", cacheState: "cold", repetition: 1, totalRepetitions: 2 },
|
||||
source: {
|
||||
headSha: "head",
|
||||
measuredRevision: "head+worktree",
|
||||
fingerprint: `${side}-fingerprint`,
|
||||
baseFingerprint: `${side}-fingerprint`,
|
||||
syntheticSourceChange: null,
|
||||
},
|
||||
host: {
|
||||
architecture: "arm64",
|
||||
dockerServer: {
|
||||
architecture: "arm64",
|
||||
operatingSystem: "linux",
|
||||
targetPlatform: "linux/arm64",
|
||||
version: "1",
|
||||
},
|
||||
platform: "darwin",
|
||||
},
|
||||
image: {
|
||||
architecture: "arm64",
|
||||
digest: null,
|
||||
id: `${side}-runtime-image`,
|
||||
revision: "head+worktree",
|
||||
sourceFingerprint: `${side}-fingerprint`,
|
||||
targetPlatform: "linux/arm64",
|
||||
},
|
||||
builderCache: {
|
||||
expected: expectedCache,
|
||||
before: cacheSnapshot(side, "empty"),
|
||||
after: cacheSnapshot(side, "warm"),
|
||||
afterStability: cacheStability,
|
||||
},
|
||||
build: { nextCompileMs: side === "baseline" ? 400 : 300, wallTimeMs: side === "baseline" ? 500 : 400 },
|
||||
};
|
||||
};
|
||||
|
||||
const runtimeSample = (value: number) => ({
|
||||
boardMountedMs: value,
|
||||
fcpMs: value,
|
||||
lcpFinalizedByInputMs: 5_100,
|
||||
lcpMs: value,
|
||||
ttfbMs: value,
|
||||
widgetImplementationsMountedMs: value,
|
||||
widgetDataSettledMs: value,
|
||||
});
|
||||
|
||||
const runtime = (side: "baseline" | "candidate"): RuntimeResult => {
|
||||
return {
|
||||
schemaVersion: 7,
|
||||
claimEligible: true,
|
||||
claimIneligibleReasons: [],
|
||||
claimScope: ["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"],
|
||||
metricEligibility: Object.fromEntries(
|
||||
["pageLoads", "largestContentfulPaint", "widgetDataSettled", "settleStability"].map((key) => [
|
||||
key,
|
||||
{ claimEligible: true },
|
||||
]),
|
||||
),
|
||||
image: {
|
||||
architecture: "arm64",
|
||||
digest: null,
|
||||
id: `${side}-runtime-image`,
|
||||
revision: "head+worktree",
|
||||
sourceFingerprint: `${side}-fingerprint`,
|
||||
},
|
||||
host: { architecture: "arm64", node: { version: "24", v8Version: "13" }, platform: "darwin", release: "1" },
|
||||
provenance: {
|
||||
browser: {
|
||||
engine: "chromium",
|
||||
executablePath: "/chromium",
|
||||
headless: true,
|
||||
httpCachePolicy: "default",
|
||||
launchArguments: [`--proxy-server=http://127.0.0.1:${side === "baseline" ? "31001" : "32002"}`],
|
||||
version: "1",
|
||||
viewport: { width: 1280, height: 720 },
|
||||
},
|
||||
container: {
|
||||
cpuLimit: 2,
|
||||
memoryLimitBytes: 1024,
|
||||
network: { internal: true },
|
||||
node: { architecture: "arm64", platform: "linux", version: "24", v8Version: "13" },
|
||||
},
|
||||
docker: { engine: { version: "1" }, version: { version: "1" } },
|
||||
ingressProxy: { excludedFromMemorySamples: true, implementation: "Node.js TCP proxy" },
|
||||
},
|
||||
browserNetworkIsolation: {
|
||||
active: true,
|
||||
canaries: {
|
||||
http: { attemptCount: 1, responseHeader: "true", status: 502 },
|
||||
webSockets: [
|
||||
{ connectTarget: "browser-ws.invalid:80", url: "ws://browser-ws.invalid/canary" },
|
||||
{ connectTarget: "browser-wss.invalid:443", url: "wss://browser-wss.invalid/canary" },
|
||||
].map((canary) => ({
|
||||
...canary,
|
||||
attemptCount: 1,
|
||||
browserEventCount: 1,
|
||||
closeCount: 1,
|
||||
errorEvent: true,
|
||||
frameReceivedCount: 0,
|
||||
frameSentCount: 0,
|
||||
opened: false,
|
||||
socketErrorCount: 1,
|
||||
timedOut: false,
|
||||
})),
|
||||
},
|
||||
externalBrowserWebSockets: [],
|
||||
unexpectedExternalResponses: [],
|
||||
},
|
||||
serverNetworkIsolation: {
|
||||
active: true,
|
||||
canary: { directIp: { blocked: true }, domain: { blocked: true } },
|
||||
network: { internal: true },
|
||||
},
|
||||
workload: {
|
||||
settleMs: 600_000,
|
||||
sampleIntervalMs: 30_000,
|
||||
settleSampleCount: 20,
|
||||
lcpObservationMs: 5_000,
|
||||
lcpStabilityMs: 500,
|
||||
widgetDataQuietMs: 500,
|
||||
spotlightIdleMs: 2_500,
|
||||
spotlightIdlePolicy: side === "baseline" ? "mounted" : "preload-only",
|
||||
interactionIterations: 7,
|
||||
widgetCount: 32,
|
||||
implementationMarkerPolicy: "true",
|
||||
widgetImplementationMarkerCount: 32,
|
||||
widgetErrorCount: 0,
|
||||
startToReadyMs: 1_000,
|
||||
coldSpotlightFeedbackMs: 20,
|
||||
coldSpotlightReadyMs: 200,
|
||||
postIdleWarmInteractionMs: 30,
|
||||
routeTimings: [{ path: "/manage" }],
|
||||
pageLoadSamples: Array.from({ length: 20 }, () => runtimeSample(side === "baseline" ? 200 : 100)),
|
||||
},
|
||||
checkpoints: [
|
||||
{ capturedAt: new Date(0).toISOString(), name: "startup" },
|
||||
{ capturedAt: new Date(1_000).toISOString(), name: "workload" },
|
||||
...Array.from({ length: 20 }, (_, index) => ({
|
||||
capturedAt: new Date(1_000 + (index + 1) * 30_000).toISOString(),
|
||||
name: `settle-${index + 1}`,
|
||||
})),
|
||||
],
|
||||
summary: { startupBytes: 1, workloadBytes: 2, settledBytes: 2, peakBytes: 3 },
|
||||
};
|
||||
};
|
||||
|
||||
const validInput = (): BenchmarkPairInput => ({
|
||||
baselineWarmup: warmup("baseline"),
|
||||
candidateWarmup: warmup("candidate"),
|
||||
baselineBuilds: [build("baseline", 1), build("baseline", 2)],
|
||||
candidateBuilds: [build("candidate", 1), build("candidate", 2)],
|
||||
baselineRuntime: runtime("baseline"),
|
||||
candidateRuntime: runtime("candidate"),
|
||||
});
|
||||
|
||||
describe("benchmark pair validation", () => {
|
||||
it("accepts a complete, isolated, provenance-matched comparison", () => {
|
||||
const result = compareBenchmarkPair(validInput());
|
||||
|
||||
expect(result.claimEligible).toBe(true);
|
||||
expect(result.claimIneligibleReasons).toEqual([]);
|
||||
expect(result.runtime.lcpMs.comparison).toMatchObject({ baseline: 200, candidate: 100, improvementPercent: 50 });
|
||||
});
|
||||
|
||||
it("rejects an incomplete build series", () => {
|
||||
const input = validInput();
|
||||
input.candidateBuilds.pop();
|
||||
|
||||
expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate build repetitions are incomplete");
|
||||
});
|
||||
|
||||
it("rejects runtime source drift and uncontrolled server egress", () => {
|
||||
const input = validInput();
|
||||
input.candidateRuntime.image.sourceFingerprint = "wrong";
|
||||
input.candidateRuntime.serverNetworkIsolation.canary.directIp.blocked = false;
|
||||
|
||||
const result = compareBenchmarkPair(input);
|
||||
expect(result.claimEligible).toBe(false);
|
||||
expect(result.claimIneligibleReasons).toContain(
|
||||
"candidate runtime image does not match the candidate build source",
|
||||
);
|
||||
expect(result.claimIneligibleReasons).toContain("candidate server network is not isolated");
|
||||
});
|
||||
|
||||
it("rejects runtime evidence without blocked HTTP, WS, and WSS browser canaries", () => {
|
||||
const input = validInput();
|
||||
const wssCanary = input.candidateRuntime.browserNetworkIsolation.canaries.webSockets[1];
|
||||
if (!wssCanary) throw new Error("Missing WSS canary fixture");
|
||||
wssCanary.opened = true;
|
||||
|
||||
expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate browser network is not isolated");
|
||||
});
|
||||
|
||||
it("requires one WS and one WSS browser isolation canary", () => {
|
||||
const input = validInput();
|
||||
const wssCanary = input.candidateRuntime.browserNetworkIsolation.canaries.webSockets[1];
|
||||
if (!wssCanary) throw new Error("Missing WSS canary fixture");
|
||||
wssCanary.url = "ws://second-browser-ws.invalid/canary";
|
||||
wssCanary.connectTarget = "second-browser-ws.invalid:80";
|
||||
|
||||
expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain("candidate browser network is not isolated");
|
||||
});
|
||||
|
||||
it("recomputes runtime claim minima instead of trusting embedded eligibility flags", () => {
|
||||
const input = validInput();
|
||||
input.candidateRuntime.claimScope = [];
|
||||
input.candidateRuntime.workload.lcpObservationMs = 1_000;
|
||||
input.candidateRuntime.workload.widgetDataQuietMs = 100;
|
||||
input.candidateRuntime.workload.settleMs = 1_000;
|
||||
input.candidateRuntime.workload.settleSampleCount = 19;
|
||||
input.candidateRuntime.checkpoints.pop();
|
||||
const firstSample = input.candidateRuntime.workload.pageLoadSamples[0];
|
||||
if (!firstSample) throw new Error("Missing runtime sample fixture");
|
||||
firstSample.lcpMs = Number.POSITIVE_INFINITY;
|
||||
|
||||
const reasons = compareBenchmarkPair(input).claimIneligibleReasons;
|
||||
expect(reasons).toContain("candidate runtime claim scope is incomplete");
|
||||
expect(reasons).toContain("candidate runtime LCP observation is below 5000ms");
|
||||
expect(reasons).toContain("candidate runtime widget-data quiet window is below 500ms");
|
||||
expect(reasons).toContain("candidate runtime settle duration is below 600000ms");
|
||||
expect(reasons).toContain("candidate runtime has fewer than 20 settle samples");
|
||||
expect(reasons).toContain("candidate runtime metric lcpMs has invalid or missing samples");
|
||||
});
|
||||
|
||||
it("recomputes measured cache-mount use instead of trusting build eligibility flags", () => {
|
||||
const input = validInput();
|
||||
const firstBuild = input.candidateBuilds[0];
|
||||
const beforeMount = firstBuild?.builderCache.before?.cacheMounts[0];
|
||||
const afterMount = firstBuild?.builderCache.after?.cacheMounts[0];
|
||||
if (!beforeMount || !afterMount) throw new Error("Missing cache mount fixture");
|
||||
afterMount.usageCount = beforeMount.usageCount;
|
||||
|
||||
expect(compareBenchmarkPair(input).claimIneligibleReasons).toContain(
|
||||
"candidate repetition 1 build cache mount pnpm-store usage did not increase",
|
||||
);
|
||||
});
|
||||
|
||||
it("verifies actual settle chronology, LCP finalization, and immutable build fingerprints", () => {
|
||||
const input = validInput();
|
||||
const finalSettle = input.candidateRuntime.checkpoints.at(-1);
|
||||
const firstSample = input.candidateRuntime.workload.pageLoadSamples[0];
|
||||
const firstBuild = input.candidateBuilds[0];
|
||||
if (!finalSettle || !firstSample || !firstBuild) throw new Error("Missing evidence fixture");
|
||||
finalSettle.capturedAt = new Date(500_000).toISOString();
|
||||
firstSample.lcpFinalizedByInputMs = 4_999;
|
||||
firstBuild.image.sourceFingerprint = "wrong-fingerprint";
|
||||
|
||||
const reasons = compareBenchmarkPair(input).claimIneligibleReasons;
|
||||
expect(reasons).toContain("candidate runtime checkpoint span is shorter than the declared settle duration");
|
||||
expect(reasons).toContain("candidate runtime LCP samples were not observed and finalized for the declared window");
|
||||
expect(reasons).toContain("candidate build image fingerprint does not match its immutable source snapshot");
|
||||
});
|
||||
|
||||
it("rejects a contaminated warmup or cache use outside the measured chain", () => {
|
||||
const input = validInput();
|
||||
if (input.candidateWarmup.builderCache.before) input.candidateWarmup.builderCache.before.recordCount = 1;
|
||||
if (input.baselineBuilds[0]?.builderCache.before) {
|
||||
input.baselineBuilds[0].builderCache.before.recordsFingerprint = "unrecorded-cache-use";
|
||||
}
|
||||
|
||||
const result = compareBenchmarkPair(input);
|
||||
expect(result.claimIneligibleReasons).toContain("candidate warmup builder did not begin empty");
|
||||
expect(result.claimIneligibleReasons).toContain("baseline builder cache changed outside repetition 1");
|
||||
});
|
||||
|
||||
it("rejects overlapping builds and a stale runtime image with copied labels", () => {
|
||||
const input = validInput();
|
||||
const candidateFirst = input.candidateBuilds[0];
|
||||
if (candidateFirst) candidateFirst.run.startedAt = input.baselineBuilds[0]?.run.startedAt ?? "";
|
||||
input.candidateRuntime.image.id = "stale-image-with-matching-labels";
|
||||
|
||||
const result = compareBenchmarkPair(input);
|
||||
expect(result.claimIneligibleReasons).toContain("build runs overlap between baseline:1 and candidate:1");
|
||||
expect(result.claimIneligibleReasons).toContain(
|
||||
"candidate runtime image identity does not match the candidate warmup image",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,235 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { parse } from "@babel/parser";
|
||||
|
||||
export const getRuntimeModuleSpecifiers = (sourceText: string, fileName: string) => {
|
||||
try {
|
||||
const ast = parse(sourceText, { sourceFilename: fileName, sourceType: "module", plugins: ["typescript", "jsx"] });
|
||||
return ast.program.body.flatMap((statement) => {
|
||||
if (statement.type === "ImportDeclaration") {
|
||||
if (statement.importKind === "type" || statement.importKind === "typeof") return [];
|
||||
const hasRuntimeBinding =
|
||||
statement.specifiers.length === 0 ||
|
||||
statement.specifiers.some(
|
||||
(specifier) => specifier.type !== "ImportSpecifier" || specifier.importKind !== "type",
|
||||
);
|
||||
return hasRuntimeBinding ? [statement.source.value] : [];
|
||||
}
|
||||
if (statement.type === "ExportAllDeclaration") {
|
||||
return statement.exportKind === "type" ? [] : [statement.source.value];
|
||||
}
|
||||
if (statement.type === "ExportNamedDeclaration" && statement.source) {
|
||||
if (statement.exportKind === "type") return [];
|
||||
const hasRuntimeBinding =
|
||||
statement.specifiers.length === 0 ||
|
||||
statement.specifiers.some((specifier) => specifier.exportKind !== "type");
|
||||
return hasRuntimeBinding ? [statement.source.value] : [];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(`Unable to parse static imports in ${fileName}`, { cause: error });
|
||||
}
|
||||
};
|
||||
|
||||
const canonicalize = (fileName: string) => {
|
||||
try {
|
||||
return fs.realpathSync.native(fileName);
|
||||
} catch {
|
||||
return path.resolve(fileName);
|
||||
}
|
||||
};
|
||||
|
||||
const resolveFile = (candidate: string) => {
|
||||
const candidates = path.extname(candidate)
|
||||
? [candidate]
|
||||
: [
|
||||
candidate,
|
||||
`${candidate}.ts`,
|
||||
`${candidate}.tsx`,
|
||||
`${candidate}.mts`,
|
||||
`${candidate}.js`,
|
||||
`${candidate}.jsx`,
|
||||
path.join(candidate, "index.ts"),
|
||||
path.join(candidate, "index.tsx"),
|
||||
path.join(candidate, "index.mts"),
|
||||
path.join(candidate, "index.js"),
|
||||
];
|
||||
return candidates.find((fileName) => fs.existsSync(fileName) && fs.statSync(fileName).isFile()) ?? null;
|
||||
};
|
||||
|
||||
const firstExportTarget = (value: unknown): string | null => {
|
||||
if (typeof value === "string") return value;
|
||||
if (!value || typeof value !== "object") return null;
|
||||
for (const nested of Object.values(value)) {
|
||||
const target = firstExportTarget(nested);
|
||||
if (target) return target;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const getPackageName = (specifier: string) => specifier.split("/").slice(0, 2).join("/");
|
||||
|
||||
const resolvePackageExport = (exports: unknown, exportKey: string) => {
|
||||
if (exportKey === "." && typeof exports === "string") return exports;
|
||||
if (!exports || typeof exports !== "object") return null;
|
||||
const exportMap = exports as Record<string, unknown>;
|
||||
const exact = firstExportTarget(exportMap[exportKey]);
|
||||
if (exact) return exact;
|
||||
|
||||
for (const [pattern, value] of Object.entries(exportMap)) {
|
||||
const wildcard = pattern.indexOf("*");
|
||||
if (wildcard < 0) continue;
|
||||
const prefix = pattern.slice(0, wildcard);
|
||||
const suffix = pattern.slice(wildcard + 1);
|
||||
if (!exportKey.startsWith(prefix) || !exportKey.endsWith(suffix)) continue;
|
||||
const match = exportKey.slice(prefix.length, exportKey.length - suffix.length);
|
||||
const target = firstExportTarget(value);
|
||||
if (target) return target.replaceAll("*", match);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const getWorkspacePackages = (root: string) => {
|
||||
const packages = new Map<string, { directory: string; exports: unknown }>();
|
||||
for (const parent of ["apps", "packages", "tooling"]) {
|
||||
const parentDirectory = path.join(root, parent);
|
||||
if (!fs.existsSync(parentDirectory)) continue;
|
||||
for (const directoryEntry of fs.readdirSync(parentDirectory, { withFileTypes: true })) {
|
||||
if (!directoryEntry.isDirectory()) continue;
|
||||
const directory = path.join(parentDirectory, directoryEntry.name);
|
||||
const manifestPath = path.join(directory, "package.json");
|
||||
if (!fs.existsSync(manifestPath)) continue;
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { name?: string; exports?: unknown };
|
||||
if (manifest.name) packages.set(manifest.name, { directory, exports: manifest.exports });
|
||||
}
|
||||
}
|
||||
return packages;
|
||||
};
|
||||
|
||||
export const collectBoardStaticGraph = (repositoryRoot: string) => {
|
||||
const root = canonicalize(repositoryRoot);
|
||||
const entries = [
|
||||
"apps/nextjs/src/app/[locale]/layout.tsx",
|
||||
"apps/nextjs/src/app/[locale]/(home)/(board)/layout.tsx",
|
||||
"apps/nextjs/src/app/[locale]/(home)/(board)/page.tsx",
|
||||
"apps/nextjs/src/app/[locale]/boards/(content)/[name]/(board)/layout.tsx",
|
||||
"apps/nextjs/src/app/[locale]/boards/(content)/[name]/(board)/page.tsx",
|
||||
"apps/nextjs/src/app/[locale]/boards/(content)/_client.tsx",
|
||||
].map((entry) => path.join(root, entry));
|
||||
const workspacePackages = getWorkspacePackages(root);
|
||||
const unresolvedWorkspaceImports: Array<{ importer: string; specifier: string }> = [];
|
||||
|
||||
const resolveSpecifier = (specifier: string, importer: string) => {
|
||||
if (specifier.startsWith(".")) return resolveFile(path.resolve(path.dirname(importer), specifier));
|
||||
if (specifier.startsWith("~/")) return resolveFile(path.join(root, "apps/nextjs/src", specifier.slice(2)));
|
||||
if (specifier.startsWith("@static-data/")) {
|
||||
return resolveFile(path.join(root, "static-data", specifier.slice("@static-data/".length)));
|
||||
}
|
||||
if (!specifier.startsWith("@homarr/") || specifier === "@homarr/node-unifi") return null;
|
||||
|
||||
const [, , ...subpathParts] = specifier.split("/");
|
||||
const workspacePackage = workspacePackages.get(getPackageName(specifier));
|
||||
if (!workspacePackage) return null;
|
||||
const exportKey = subpathParts.length === 0 ? "." : `./${subpathParts.join("/")}`;
|
||||
const target = resolvePackageExport(workspacePackage.exports, exportKey);
|
||||
return target ? resolveFile(path.resolve(workspacePackage.directory, target)) : null;
|
||||
};
|
||||
|
||||
const queue = [...entries];
|
||||
const visited = new Set<string>();
|
||||
const parents = new Map<string, { importer: string; specifier: string }>();
|
||||
const bareWidgetImports: Array<{ importer: string; specifier: string }> = [];
|
||||
const forbiddenModules = new Set([
|
||||
path.join(root, "packages/widgets/index.ts"),
|
||||
path.join(root, "packages/widgets/src/index.tsx"),
|
||||
path.join(root, "packages/spotlight/src/component.ts"),
|
||||
path.join(root, "packages/spotlight/src/components/spotlight.tsx"),
|
||||
path.join(root, "packages/spotlight/src/spotlight-store.ts"),
|
||||
path.join(root, "packages/widgets/src/modals/widget-edit-modal.tsx"),
|
||||
path.join(root, "apps/nextjs/src/components/board/items/item-move-modal.tsx"),
|
||||
path.join(root, "apps/nextjs/src/components/onboarding/board-tour.tsx"),
|
||||
path.join(root, "apps/nextjs/src/components/onboarding/tour-shell.tsx"),
|
||||
]);
|
||||
|
||||
while (queue.length > 0) {
|
||||
const next = queue.shift();
|
||||
if (!next) continue;
|
||||
const current = canonicalize(next);
|
||||
if (visited.has(current)) continue;
|
||||
visited.add(current);
|
||||
if (!/\.[cm]?[jt]sx?$/.test(current)) continue;
|
||||
|
||||
const sourceText = fs.readFileSync(current, "utf8");
|
||||
for (const specifier of getRuntimeModuleSpecifiers(sourceText, current)) {
|
||||
if (specifier === "@homarr/widgets") bareWidgetImports.push({ importer: current, specifier });
|
||||
const resolved = resolveSpecifier(specifier, current);
|
||||
if (!resolved) {
|
||||
if (workspacePackages.has(getPackageName(specifier))) {
|
||||
unresolvedWorkspaceImports.push({ importer: current, specifier });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const target = canonicalize(resolved);
|
||||
if (!target.startsWith(`${root}${path.sep}`) || target.includes(`${path.sep}node_modules${path.sep}`)) continue;
|
||||
if (!parents.has(target)) parents.set(target, { importer: current, specifier });
|
||||
queue.push(target);
|
||||
}
|
||||
}
|
||||
|
||||
const reachedForbiddenModules = [...forbiddenModules].filter((fileName) => visited.has(fileName));
|
||||
const formatChain = (target: string) => {
|
||||
const chain = [path.relative(root, target)];
|
||||
let current = target;
|
||||
while (parents.has(current)) {
|
||||
const parent = parents.get(current);
|
||||
if (!parent) break;
|
||||
chain.unshift(`${path.relative(root, parent.importer)} --${parent.specifier}-->`);
|
||||
current = parent.importer;
|
||||
}
|
||||
return chain.join(" ");
|
||||
};
|
||||
|
||||
return {
|
||||
entries: entries.map((entry) => path.relative(root, entry)),
|
||||
visitedModuleCount: visited.size,
|
||||
bareWidgetImports: bareWidgetImports.map(({ importer, specifier }) => ({
|
||||
importer: path.relative(root, importer),
|
||||
specifier,
|
||||
})),
|
||||
reachedForbiddenModules: reachedForbiddenModules.map(formatChain),
|
||||
unresolvedWorkspaceImports: unresolvedWorkspaceImports.map(({ importer, specifier }) => ({
|
||||
importer: path.relative(root, importer),
|
||||
specifier,
|
||||
})),
|
||||
};
|
||||
};
|
||||
|
||||
const run = () => {
|
||||
const repositoryRoot = path.resolve(process.env.BOARD_GRAPH_ROOT ?? ".");
|
||||
const result = collectBoardStaticGraph(repositoryRoot);
|
||||
if (
|
||||
result.bareWidgetImports.length > 0 ||
|
||||
result.reachedForbiddenModules.length > 0 ||
|
||||
result.unresolvedWorkspaceImports.length > 0
|
||||
) {
|
||||
throw new Error(`Board static graph reaches a forbidden eager module:\n${JSON.stringify(result, null, 2)}`);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
...result,
|
||||
eagerWidgetRegistryReachable: false,
|
||||
eagerSpotlightImplementationReachable: false,
|
||||
eagerWidgetEditorReachable: false,
|
||||
eagerItemMoveEditorReachable: false,
|
||||
eagerBoardTourReachable: false,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) run();
|
||||
@@ -0,0 +1,37 @@
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { collectBoardStaticGraph, getRuntimeModuleSpecifiers } from "./board-static-graph.mts";
|
||||
|
||||
describe("board static dependency graph", () => {
|
||||
it("ignores type-only and dynamic imports", () => {
|
||||
expect(
|
||||
getRuntimeModuleSpecifiers(
|
||||
[
|
||||
'import type { A } from "type-only";',
|
||||
'import { type B } from "also-type-only";',
|
||||
'import { C, type D } from "runtime";',
|
||||
'export type { E } from "export-type";',
|
||||
'export { type F } from "also-export-type";',
|
||||
'export { G } from "runtime-export";',
|
||||
'export * from "runtime-star";',
|
||||
'import "runtime-side-effect";',
|
||||
"import {",
|
||||
" H,",
|
||||
" type I,",
|
||||
'} from "runtime-multiline";',
|
||||
'void import("lazy");',
|
||||
].join("\n"),
|
||||
"fixture.ts",
|
||||
),
|
||||
).toEqual(["runtime", "runtime-export", "runtime-star", "runtime-side-effect", "runtime-multiline"]);
|
||||
});
|
||||
|
||||
it("keeps widget registries, editors, Spotlight, and onboarding tour out of the board's static graph", () => {
|
||||
const result = collectBoardStaticGraph(path.resolve("."));
|
||||
expect(result.visitedModuleCount).toBeGreaterThan(0);
|
||||
expect(result.bareWidgetImports).toEqual([]);
|
||||
expect(result.reachedForbiddenModules).toEqual([]);
|
||||
expect(result.unresolvedWorkspaceImports).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,15 +1,17 @@
|
||||
export type NumericSummary = {
|
||||
count: number;
|
||||
medianMs: number | null;
|
||||
p95Ms: number | null;
|
||||
standardDeviationMs: number | null;
|
||||
outliersMs: number[];
|
||||
min: number | null;
|
||||
median: number | null;
|
||||
p95: number | null;
|
||||
max: number | null;
|
||||
standardDeviation: number | null;
|
||||
outliers: number[];
|
||||
};
|
||||
|
||||
export const normalizeTrpcPath = (requestUrl: string) => {
|
||||
try {
|
||||
const url = new URL(requestUrl, "http://benchmark.invalid");
|
||||
url.searchParams.delete("input");
|
||||
url.searchParams.sort();
|
||||
return url.pathname + url.search;
|
||||
} catch {
|
||||
return requestUrl;
|
||||
@@ -27,13 +29,13 @@ const values = (numbers: Array<number | null | undefined>) =>
|
||||
export const summarize = (numbers: Array<number | null | undefined>): NumericSummary => {
|
||||
const sorted = values(numbers);
|
||||
if (sorted.length === 0) {
|
||||
return { count: 0, medianMs: null, p95Ms: null, standardDeviationMs: null, outliersMs: [] };
|
||||
return { count: 0, min: null, median: null, p95: null, max: null, standardDeviation: null, outliers: [] };
|
||||
}
|
||||
|
||||
const middleIndex = Math.floor(sorted.length / 2);
|
||||
let medianMs = sorted[middleIndex] ?? 0;
|
||||
let median = sorted[middleIndex] ?? 0;
|
||||
if (sorted.length % 2 === 0) {
|
||||
medianMs = ((sorted[middleIndex - 1] ?? 0) + medianMs) / 2;
|
||||
median = ((sorted[middleIndex - 1] ?? 0) + median) / 2;
|
||||
}
|
||||
const p95Index = Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1);
|
||||
const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length;
|
||||
@@ -45,10 +47,12 @@ export const summarize = (numbers: Array<number | null | undefined>): NumericSum
|
||||
|
||||
return {
|
||||
count: sorted.length,
|
||||
medianMs,
|
||||
p95Ms: sorted[p95Index] ?? 0,
|
||||
standardDeviationMs: Math.round(Math.sqrt(variance)),
|
||||
outliersMs: sorted.filter((value) => value > outlierLimit),
|
||||
min: sorted[0] ?? null,
|
||||
median,
|
||||
p95: sorted.length >= 20 ? (sorted[p95Index] ?? null) : null,
|
||||
max: sorted.at(-1) ?? null,
|
||||
standardDeviation: Math.round(Math.sqrt(variance)),
|
||||
outliers: sorted.filter((value) => value > outlierLimit),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -56,7 +60,10 @@ export const validateBrowserMeasurement = (measurement: {
|
||||
responseStatus: number | null;
|
||||
domContentLoadedMs: number;
|
||||
boardHydratedMs: number | null;
|
||||
firstInteractionMs: number | null;
|
||||
widgetsReadyMs?: number | null;
|
||||
widgetCount?: number;
|
||||
coldImmediateInteractionMs?: number | null;
|
||||
postIdleWarmInteractionMs?: number | null;
|
||||
error?: string;
|
||||
}) => {
|
||||
const errors: string[] = [];
|
||||
@@ -66,6 +73,9 @@ export const validateBrowserMeasurement = (measurement: {
|
||||
errors.push(`browser navigation returned HTTP ${measurement.responseStatus}`);
|
||||
if (measurement.domContentLoadedMs < 0) errors.push("DOMContentLoaded timing is missing");
|
||||
if (measurement.boardHydratedMs === null) errors.push("board hydration marker is missing");
|
||||
if (measurement.firstInteractionMs === null) errors.push("first interaction timing is missing");
|
||||
if (measurement.widgetsReadyMs === null) errors.push("widget readiness marker is missing");
|
||||
if (measurement.widgetCount === 0) errors.push("representative board contains no widgets");
|
||||
if (measurement.coldImmediateInteractionMs === null) errors.push("cold immediate interaction timing is missing");
|
||||
if (measurement.postIdleWarmInteractionMs === null) errors.push("post-idle warm interaction timing is missing");
|
||||
return errors;
|
||||
};
|
||||
@@ -8,26 +8,29 @@ import {
|
||||
} from "./dev-benchmark-lib.mts";
|
||||
|
||||
describe("development benchmark measurements", () => {
|
||||
it("normalizes tRPC input payloads before counting duplicates", () => {
|
||||
it("preserves tRPC input payloads when counting duplicates", () => {
|
||||
const first = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A1%7D";
|
||||
const second = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A2%7D";
|
||||
|
||||
expect(normalizeTrpcPath(first)).toBe("/api/trpc/board.get?batch=1");
|
||||
expect(countDuplicateTrpcPaths([first, second, "/api/trpc/user.get"])).toBe(1);
|
||||
expect(normalizeTrpcPath(first)).toBe("/api/trpc/board.get?batch=1&input=%7B%22json%22%3A1%7D");
|
||||
expect(countDuplicateTrpcPaths([first, second, first, "/api/trpc/user.get"])).toBe(1);
|
||||
});
|
||||
|
||||
it("reports median, p95, deviation, and high outliers", () => {
|
||||
expect(summarize([1, 2, 3, 4, 5, 6, 100])).toEqual({
|
||||
count: 7,
|
||||
medianMs: 4,
|
||||
p95Ms: 100,
|
||||
standardDeviationMs: 34,
|
||||
outliersMs: [100],
|
||||
min: 1,
|
||||
median: 4,
|
||||
p95: null,
|
||||
max: 100,
|
||||
standardDeviation: 34,
|
||||
outliers: [100],
|
||||
});
|
||||
});
|
||||
|
||||
it("averages both middle values for an even sample count", () => {
|
||||
expect(summarize([100, 200]).medianMs).toBe(150);
|
||||
expect(summarize([100, 200]).median).toBe(150);
|
||||
expect(summarize(Array.from({ length: 20 }, (_, index) => index + 1)).p95).toBe(19);
|
||||
});
|
||||
|
||||
it("requires a successful authenticated board hydration and interaction", () => {
|
||||
@@ -36,12 +39,18 @@ describe("development benchmark measurements", () => {
|
||||
responseStatus: 307,
|
||||
domContentLoadedMs: 120,
|
||||
boardHydratedMs: null,
|
||||
firstInteractionMs: null,
|
||||
widgetsReadyMs: null,
|
||||
widgetCount: 0,
|
||||
coldImmediateInteractionMs: null,
|
||||
postIdleWarmInteractionMs: null,
|
||||
}),
|
||||
).toEqual([
|
||||
"browser navigation returned HTTP 307",
|
||||
"board hydration marker is missing",
|
||||
"first interaction timing is missing",
|
||||
"widget readiness marker is missing",
|
||||
"representative board contains no widgets",
|
||||
"cold immediate interaction timing is missing",
|
||||
"post-idle warm interaction timing is missing",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -2,25 +2,39 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import type { Page } from "@playwright/test";
|
||||
import type { BrowserContext, Page } from "@playwright/test";
|
||||
import { chromium } from "@playwright/test";
|
||||
|
||||
import { countDuplicateTrpcPaths, summarize, validateBrowserMeasurement } from "./dev-benchmark-lib.mts";
|
||||
|
||||
type BoardMeasurement = {
|
||||
boardReadyMs: number;
|
||||
boardMountedMs: number;
|
||||
domContentLoadedMs: number;
|
||||
firstInteractionMs: number;
|
||||
postIdleWarmInteractionMs: number;
|
||||
widgetCount: number;
|
||||
widgetsReadyMs: number;
|
||||
pageErrors: string[];
|
||||
responseStatus: number | null;
|
||||
trpcDuplicateRequestCount: number;
|
||||
trpcRequestCount: number;
|
||||
webSocketCount: number;
|
||||
performance: {
|
||||
cls: number;
|
||||
decodedScriptBytes: number;
|
||||
jsHeapBytes: number | null;
|
||||
lcpMs: number | null;
|
||||
longTaskCount: number;
|
||||
longTaskTotalMs: number;
|
||||
resourceCount: number;
|
||||
transferBytes: number;
|
||||
};
|
||||
};
|
||||
|
||||
const root = path.resolve(import.meta.dirname, "../..");
|
||||
const argumentsSet = new Set(process.argv.slice(2));
|
||||
const smokeOnly = argumentsSet.has("--smoke");
|
||||
const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 7);
|
||||
const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 10);
|
||||
const spotlightIdleMs = Number(process.env.DEV_BENCHMARK_SPOTLIGHT_IDLE_MS ?? 2_500);
|
||||
const outputDirectory = path.resolve(
|
||||
process.env.DEV_BENCHMARK_OUTPUT_DIR ?? path.join(root, "benchmark-results", "dev"),
|
||||
);
|
||||
@@ -30,10 +44,15 @@ const readinessUrl = process.env.DEV_BENCHMARK_READINESS_URL ?? "http://127.0.0.
|
||||
const boardSelector = "[data-homarr-dev-benchmark-board]";
|
||||
const interactionSelector = '[data-homarr-dev-benchmark-interaction="search"]';
|
||||
const spotlightSelector = "[data-homarr-dev-benchmark-spotlight]";
|
||||
const spotlightFeedbackSelector =
|
||||
"[data-homarr-dev-benchmark-spotlight-feedback], [data-homarr-dev-benchmark-spotlight]";
|
||||
|
||||
if (!Number.isInteger(iterations) || iterations < 7) {
|
||||
throw new Error("DEV_BENCHMARK_ITERATIONS must be an integer of at least 7");
|
||||
}
|
||||
if (!Number.isFinite(spotlightIdleMs) || spotlightIdleMs < 0) {
|
||||
throw new Error("DEV_BENCHMARK_SPOTLIGHT_IDLE_MS must be non-negative");
|
||||
}
|
||||
|
||||
await fs.mkdir(outputDirectory, { recursive: true });
|
||||
|
||||
@@ -50,6 +69,7 @@ if (smokeOnly) {
|
||||
const result = {
|
||||
kind: "readiness-smoke",
|
||||
claimEligible: false,
|
||||
reason: "Readiness checks do not exercise the browser or dashboard graph",
|
||||
url: readinessUrl,
|
||||
iterations,
|
||||
summary: summarize(measurements),
|
||||
@@ -62,50 +82,139 @@ if (smokeOnly) {
|
||||
|
||||
if (!boardUrl || !storageStatePath) {
|
||||
throw new Error(
|
||||
"Authenticated claim mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " +
|
||||
"Authenticated mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " +
|
||||
"Use --smoke only for the non-claimable readiness check.",
|
||||
);
|
||||
}
|
||||
|
||||
await fs.access(storageStatePath);
|
||||
|
||||
const installPerformanceObservers = () => {
|
||||
const state = { cls: 0, lcpMs: null as number | null, longTaskCount: 0, longTaskTotalMs: 0 };
|
||||
Object.defineProperty(window, "homarrBenchmarkVitals", { value: state, configurable: true });
|
||||
|
||||
if (PerformanceObserver.supportedEntryTypes.includes("largest-contentful-paint")) {
|
||||
new PerformanceObserver((list) => {
|
||||
const latest = list.getEntries().at(-1);
|
||||
if (latest) state.lcpMs = latest.startTime;
|
||||
}).observe({ type: "largest-contentful-paint", buffered: true });
|
||||
}
|
||||
if (PerformanceObserver.supportedEntryTypes.includes("layout-shift")) {
|
||||
new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) {
|
||||
const shift = entry as PerformanceEntry & { hadRecentInput?: boolean; value?: number };
|
||||
if (!shift.hadRecentInput) state.cls += shift.value ?? 0;
|
||||
}
|
||||
}).observe({ type: "layout-shift", buffered: true });
|
||||
}
|
||||
if (PerformanceObserver.supportedEntryTypes.includes("longtask")) {
|
||||
new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) {
|
||||
state.longTaskCount += 1;
|
||||
state.longTaskTotalMs += entry.duration;
|
||||
}
|
||||
}).observe({ type: "longtask", buffered: true });
|
||||
}
|
||||
};
|
||||
|
||||
const createContextAsync = async (): Promise<BrowserContext> => {
|
||||
const context = await browser.newContext({ storageState: storageStatePath });
|
||||
await context.addInitScript(installPerformanceObservers);
|
||||
return context;
|
||||
};
|
||||
|
||||
const browser = await chromium.launch();
|
||||
const cold: BoardMeasurement[] = [];
|
||||
const warm: BoardMeasurement[] = [];
|
||||
const freshContext: BoardMeasurement[] = [];
|
||||
const warmPage: BoardMeasurement[] = [];
|
||||
const coldInteractionSamples: Array<{ feedbackMs: number; readyMs: number }> = [];
|
||||
|
||||
const measureBoardAsync = async (page: Page) => {
|
||||
const trpcRequests: string[] = [];
|
||||
const pageErrors: string[] = [];
|
||||
let webSocketCount = 0;
|
||||
|
||||
page.on("request", (request) => {
|
||||
if (request.url().includes("/api/trpc/")) trpcRequests.push(request.url());
|
||||
});
|
||||
page.on("websocket", () => {
|
||||
webSocketCount += 1;
|
||||
});
|
||||
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||
|
||||
const startedAt = performance.now();
|
||||
const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
|
||||
const domContentLoadedMs = Math.round(performance.now() - startedAt);
|
||||
await page.locator(boardSelector).waitFor({ state: "visible" });
|
||||
const boardReadyMs = Math.round(performance.now() - startedAt);
|
||||
const boardMountedMs = Math.round(performance.now() - startedAt);
|
||||
|
||||
const interactionStartedAt = performance.now();
|
||||
await page.locator(interactionSelector).first().click();
|
||||
await page.waitForFunction(
|
||||
(selector) => {
|
||||
const board = document.querySelector(selector);
|
||||
if (!board) return false;
|
||||
const widgetCount = board.querySelectorAll('[data-type="item"]').length;
|
||||
return widgetCount > 0 && board.querySelectorAll("[data-homarr-widget-ready]").length === widgetCount;
|
||||
},
|
||||
boardSelector,
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const widgetsReadyMs = Math.round(performance.now() - startedAt);
|
||||
const widgetCount = await page.locator(`${boardSelector} [data-type="item"]`).count();
|
||||
|
||||
await page.waitForTimeout(spotlightIdleMs);
|
||||
const warmInteractionStartedAt = performance.now();
|
||||
await page.locator(`${interactionSelector}:visible`).click();
|
||||
await page.locator(spotlightSelector).waitFor({ state: "visible" });
|
||||
const firstInteractionMs = Math.round(performance.now() - interactionStartedAt);
|
||||
const postIdleWarmInteractionMs = Math.round(performance.now() - warmInteractionStartedAt);
|
||||
await page.keyboard.press("Escape");
|
||||
await page.locator(spotlightSelector).waitFor({ state: "hidden" });
|
||||
|
||||
const browserPerformance = await page.evaluate(() => {
|
||||
const resources = performance.getEntriesByType("resource") as PerformanceResourceTiming[];
|
||||
const vitals = (
|
||||
window as Window & {
|
||||
homarrBenchmarkVitals?: {
|
||||
cls: number;
|
||||
lcpMs: number | null;
|
||||
longTaskCount: number;
|
||||
longTaskTotalMs: number;
|
||||
};
|
||||
}
|
||||
).homarrBenchmarkVitals;
|
||||
const memory = performance as Performance & { memory?: { usedJSHeapSize: number } };
|
||||
return {
|
||||
cls: Number((vitals?.cls ?? 0).toFixed(4)),
|
||||
decodedScriptBytes: resources
|
||||
.filter((resource) => resource.initiatorType === "script")
|
||||
.reduce((total, resource) => total + resource.decodedBodySize, 0),
|
||||
jsHeapBytes: memory.memory?.usedJSHeapSize ?? null,
|
||||
lcpMs: vitals?.lcpMs === null || vitals?.lcpMs === undefined ? null : Math.round(vitals.lcpMs),
|
||||
longTaskCount: vitals?.longTaskCount ?? 0,
|
||||
longTaskTotalMs: Math.round(vitals?.longTaskTotalMs ?? 0),
|
||||
resourceCount: resources.length,
|
||||
transferBytes: resources.reduce((total, resource) => total + resource.transferSize, 0),
|
||||
};
|
||||
});
|
||||
|
||||
const measurement: BoardMeasurement = {
|
||||
responseStatus: response?.status() ?? null,
|
||||
domContentLoadedMs,
|
||||
boardReadyMs,
|
||||
firstInteractionMs,
|
||||
boardMountedMs,
|
||||
widgetsReadyMs,
|
||||
widgetCount,
|
||||
postIdleWarmInteractionMs,
|
||||
trpcRequestCount: trpcRequests.length,
|
||||
trpcDuplicateRequestCount: countDuplicateTrpcPaths(trpcRequests),
|
||||
webSocketCount,
|
||||
performance: browserPerformance,
|
||||
pageErrors,
|
||||
};
|
||||
const errors = validateBrowserMeasurement({
|
||||
responseStatus: measurement.responseStatus,
|
||||
domContentLoadedMs,
|
||||
boardHydratedMs: boardReadyMs,
|
||||
firstInteractionMs,
|
||||
boardHydratedMs: boardMountedMs,
|
||||
widgetsReadyMs,
|
||||
widgetCount,
|
||||
postIdleWarmInteractionMs,
|
||||
error: pageErrors.length > 0 ? pageErrors.join("; ") : undefined,
|
||||
});
|
||||
if (errors.length > 0) throw new Error(errors.map((error) => `${boardUrl}: ${error}`).join("; "));
|
||||
@@ -113,47 +222,103 @@ const measureBoardAsync = async (page: Page) => {
|
||||
return measurement;
|
||||
};
|
||||
|
||||
const measureColdInteractionAsync = async (page: Page) => {
|
||||
const pageErrors: string[] = [];
|
||||
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||
|
||||
const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
|
||||
if (!response?.ok()) {
|
||||
throw new Error(`Cold-interaction board returned HTTP ${String(response?.status() ?? "none")}`);
|
||||
}
|
||||
await page.locator(boardSelector).waitFor({ state: "visible" });
|
||||
const interactionStartedAt = performance.now();
|
||||
await page.locator(`${interactionSelector}:visible`).click();
|
||||
await page.locator(spotlightFeedbackSelector).waitFor({ state: "visible" });
|
||||
const feedbackMs = Math.round(performance.now() - interactionStartedAt);
|
||||
await page.locator(spotlightSelector).waitFor({ state: "visible" });
|
||||
const readyMs = Math.round(performance.now() - interactionStartedAt);
|
||||
if (pageErrors.length > 0) throw new Error(`Cold-interaction page errors: ${pageErrors.join("; ")}`);
|
||||
return { feedbackMs, readyMs };
|
||||
};
|
||||
|
||||
try {
|
||||
for (let iteration = 0; iteration < iterations; iteration++) {
|
||||
const context = await browser.newContext({ storageState: storageStatePath });
|
||||
const coldPage = await context.newPage();
|
||||
cold.push(await measureBoardAsync(coldPage));
|
||||
await coldPage.close();
|
||||
const warmPage = await context.newPage();
|
||||
warm.push(await measureBoardAsync(warmPage));
|
||||
const context = await createContextAsync();
|
||||
const firstPage = await context.newPage();
|
||||
freshContext.push(await measureBoardAsync(firstPage));
|
||||
await firstPage.close();
|
||||
const secondPage = await context.newPage();
|
||||
warmPage.push(await measureBoardAsync(secondPage));
|
||||
await context.close();
|
||||
|
||||
const coldContext = await createContextAsync();
|
||||
await coldContext.addInitScript(() => {
|
||||
// Measure the actual first-open path instead of racing the intentional
|
||||
// requestIdleCallback prefetch used during normal idle time.
|
||||
Object.defineProperty(window, "requestIdleCallback", { value: () => 0, configurable: true });
|
||||
Object.defineProperty(window, "cancelIdleCallback", { value: () => undefined, configurable: true });
|
||||
});
|
||||
const coldPage = await coldContext.newPage();
|
||||
coldInteractionSamples.push(await measureColdInteractionAsync(coldPage));
|
||||
await coldContext.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
const summarizeMeasurements = (measurements: BoardMeasurement[]) => ({
|
||||
boardReady: summarize(measurements.map(({ boardReadyMs }) => boardReadyMs)),
|
||||
domContentLoaded: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)),
|
||||
firstInteraction: summarize(measurements.map(({ firstInteractionMs }) => firstInteractionMs)),
|
||||
boardMountedMs: summarize(measurements.map(({ boardMountedMs }) => boardMountedMs)),
|
||||
widgetsReadyMs: summarize(measurements.map(({ widgetsReadyMs }) => widgetsReadyMs)),
|
||||
domContentLoadedMs: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)),
|
||||
postIdleWarmInteractionMs: summarize(measurements.map(({ postIdleWarmInteractionMs }) => postIdleWarmInteractionMs)),
|
||||
lcpMs: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.lcpMs)),
|
||||
cls: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.cls)),
|
||||
longTaskTotalMs: summarize(
|
||||
measurements.map(({ performance: browserPerformance }) => browserPerformance.longTaskTotalMs),
|
||||
),
|
||||
decodedScriptBytes: summarize(
|
||||
measurements.map(({ performance: browserPerformance }) => browserPerformance.decodedScriptBytes),
|
||||
),
|
||||
transferBytes: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.transferBytes)),
|
||||
jsHeapBytes: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.jsHeapBytes)),
|
||||
trpcRequests: summarize(measurements.map(({ trpcRequestCount }) => trpcRequestCount)),
|
||||
duplicateTrpcRequests: summarize(measurements.map(({ trpcDuplicateRequestCount }) => trpcDuplicateRequestCount)),
|
||||
});
|
||||
|
||||
const sha = process.env.DEV_BENCHMARK_SHA ?? null;
|
||||
const image = process.env.DEV_BENCHMARK_IMAGE ?? null;
|
||||
const runtime = process.env.DEV_BENCHMARK_RUNTIME ?? "development";
|
||||
const claimEligible = false;
|
||||
const result = {
|
||||
kind: "authenticated-board",
|
||||
claimEligible: true,
|
||||
claimEligible,
|
||||
claimIneligibleReasons: [
|
||||
"served revision and production runtime are self-reported, not independently verified",
|
||||
...(runtime === "production" ? [] : ["runtime is not production"]),
|
||||
...(sha ? [] : ["DEV_BENCHMARK_SHA is missing"]),
|
||||
...(image ? [] : ["DEV_BENCHMARK_IMAGE is missing"]),
|
||||
],
|
||||
boardUrl,
|
||||
createdAt: new Date().toISOString(),
|
||||
environment: {
|
||||
architecture: process.arch,
|
||||
cpus: os.cpus().map(({ model }) => model),
|
||||
image,
|
||||
node: process.version,
|
||||
platform: `${process.platform} ${os.release()}`,
|
||||
sha: process.env.DEV_BENCHMARK_SHA ?? null,
|
||||
runtime,
|
||||
sha,
|
||||
},
|
||||
iterations,
|
||||
measurements: { cold, warm },
|
||||
spotlightIdleMs,
|
||||
measurements: { freshContext, warmPage, coldInteractionSamples },
|
||||
summary: {
|
||||
cold: summarizeMeasurements(cold),
|
||||
warm: summarizeMeasurements(warm),
|
||||
coldInteractionFeedbackMs: summarize(coldInteractionSamples.map(({ feedbackMs }) => feedbackMs)),
|
||||
coldInteractionReadyMs: summarize(coldInteractionSamples.map(({ readyMs }) => readyMs)),
|
||||
freshContext: summarizeMeasurements(freshContext),
|
||||
warmPage: summarizeMeasurements(warmPage),
|
||||
},
|
||||
};
|
||||
const outputPath = path.join(outputDirectory, `board-${Date.now()}.json`);
|
||||
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
|
||||
console.log(JSON.stringify({ outputPath, summary: result.summary }, null, 2));
|
||||
console.log(JSON.stringify({ outputPath, claimEligible, summary: result.summary }, null, 2));
|
||||
@@ -0,0 +1,259 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
export type DockerBuildSourceEntry =
|
||||
| { file: string; mode: number; content: Buffer; linkTarget?: never }
|
||||
| { file: string; mode: number; content?: never; linkTarget: string };
|
||||
|
||||
export type DockerBuildSyntheticSourceChange = {
|
||||
file: string;
|
||||
id: string;
|
||||
from: string;
|
||||
to: string;
|
||||
};
|
||||
|
||||
export const nextBuildCacheTarget = "/app/apps/nextjs/.next/cache";
|
||||
|
||||
export type ExpectedDockerBuildCacheMount = {
|
||||
logicalId: string;
|
||||
target: string;
|
||||
mustIncreaseUsage: boolean;
|
||||
};
|
||||
|
||||
export type DockerBuildCacheMount = {
|
||||
description: string | null;
|
||||
logicalId: string | null;
|
||||
recordId: string;
|
||||
size: string | null;
|
||||
target: string | null;
|
||||
usageCount: number | null;
|
||||
};
|
||||
|
||||
export type DockerBuildCacheSnapshot = {
|
||||
cacheMountCount: number;
|
||||
cacheMounts: DockerBuildCacheMount[];
|
||||
recordCount: number;
|
||||
};
|
||||
|
||||
export const fingerprintDockerBuildSource = (entries: DockerBuildSourceEntry[]) => {
|
||||
const hash = createHash("sha256");
|
||||
for (const entry of entries.toSorted((left, right) => left.file.localeCompare(right.file))) {
|
||||
hash.update(entry.file);
|
||||
hash.update("\0");
|
||||
hash.update(String(entry.mode));
|
||||
hash.update("\0");
|
||||
hash.update(entry.linkTarget ?? entry.content);
|
||||
hash.update("\0");
|
||||
}
|
||||
return hash.digest("hex");
|
||||
};
|
||||
|
||||
export const applyDockerBuildSyntheticSourceChange = (
|
||||
entries: DockerBuildSourceEntry[],
|
||||
id: string,
|
||||
): { entries: DockerBuildSourceEntry[]; change: DockerBuildSyntheticSourceChange } => {
|
||||
const file = "apps/nextjs/src/app/[locale]/layout.tsx";
|
||||
const from = 'siteName: "Homarr",';
|
||||
const to = `siteName: ${JSON.stringify(`Homarr build benchmark ${id}`)},`;
|
||||
let changed = false;
|
||||
const nextEntries = entries.map((entry) => {
|
||||
if (entry.file !== file || entry.content === undefined) return entry;
|
||||
const source = entry.content.toString("utf8");
|
||||
if (source.split(from).length !== 2) {
|
||||
throw new Error(`Expected exactly one build benchmark source marker in ${file}`);
|
||||
}
|
||||
changed = true;
|
||||
return { ...entry, content: Buffer.from(source.replace(from, to)) };
|
||||
});
|
||||
if (!changed) throw new Error(`Build benchmark source marker file is missing: ${file}`);
|
||||
return { entries: nextEntries, change: { file, id, from, to } };
|
||||
};
|
||||
|
||||
const expandDockerfileMountValue = (value: string, variables: Record<string, string>) => {
|
||||
const expanded = value.replaceAll(
|
||||
/\$(?:\{([A-Za-z_][A-Za-z0-9_]*)\}|([A-Za-z_][A-Za-z0-9_]*))/g,
|
||||
(reference, braced, plain) => {
|
||||
const name = (braced ?? plain) as string;
|
||||
const replacement = variables[name];
|
||||
if (replacement === undefined) {
|
||||
throw new Error(`Dockerfile cache mount contains unresolved variable ${reference}`);
|
||||
}
|
||||
return replacement;
|
||||
},
|
||||
);
|
||||
if (expanded.includes("$")) {
|
||||
throw new Error(`Dockerfile cache mount contains unresolved variable in ${value}`);
|
||||
}
|
||||
return expanded;
|
||||
};
|
||||
|
||||
export const parseDockerfileCacheMounts = (
|
||||
dockerfile: string,
|
||||
targetPlatform: string,
|
||||
): ExpectedDockerBuildCacheMount[] => {
|
||||
const mounts = [...dockerfile.matchAll(/--mount=([^\s\\]+)/g)]
|
||||
.map(([, serialized = ""]) =>
|
||||
Object.fromEntries(
|
||||
serialized.split(",").map((option) => {
|
||||
const separator = option.indexOf("=");
|
||||
return separator === -1 ? [option, ""] : [option.slice(0, separator), option.slice(separator + 1)];
|
||||
}),
|
||||
),
|
||||
)
|
||||
.filter(({ type }) => type === "cache")
|
||||
.map((options) => {
|
||||
const id = options.id;
|
||||
const target = options.target ?? options.dst ?? options.destination;
|
||||
if (!id || !target) {
|
||||
throw new Error("Dockerfile cache mounts must declare explicit id and target options");
|
||||
}
|
||||
const variables = { TARGETPLATFORM: targetPlatform };
|
||||
const logicalId = expandDockerfileMountValue(id, variables);
|
||||
const expandedTarget = expandDockerfileMountValue(target, variables);
|
||||
return {
|
||||
logicalId,
|
||||
target: expandedTarget,
|
||||
mustIncreaseUsage: expandedTarget === nextBuildCacheTarget,
|
||||
};
|
||||
});
|
||||
|
||||
const uniqueMounts = new Map(mounts.map((mount) => [`${mount.logicalId}\0${mount.target}`, mount]));
|
||||
return [...uniqueMounts.values()].toSorted(
|
||||
(left, right) => left.logicalId.localeCompare(right.logicalId) || left.target.localeCompare(right.target),
|
||||
);
|
||||
};
|
||||
|
||||
export const parseDockerBuildCacheMountDescription = (description: string | null) => {
|
||||
if (!description) return null;
|
||||
const match = /^cached mount (\S+) from exec .* with id "(\/[^"\n]+)"$/.exec(description);
|
||||
if (!match) return null;
|
||||
const [, target = "", printedId = ""] = match;
|
||||
return { logicalId: printedId.slice(1), target };
|
||||
};
|
||||
|
||||
export const normalizeDockerArchitecture = (architecture: string) => {
|
||||
switch (architecture.toLowerCase()) {
|
||||
case "x64":
|
||||
case "x86_64":
|
||||
return "amd64";
|
||||
case "aarch64":
|
||||
return "arm64";
|
||||
case "armv6l":
|
||||
return "arm/v6";
|
||||
case "armv7l":
|
||||
return "arm/v7";
|
||||
default:
|
||||
return architecture.toLowerCase();
|
||||
}
|
||||
};
|
||||
|
||||
export const parseDockerBuildOutput = (output: string) => {
|
||||
const compileDurationsMs = [...output.matchAll(/Compiled (?:successfully )?in\s+([\d.]+)\s*(ms|s)/gi)].map(
|
||||
([, value = "0", unit = "ms"]) => Number(value) * (unit.toLowerCase() === "s" ? 1_000 : 1),
|
||||
);
|
||||
|
||||
return {
|
||||
cachedStepCount: (output.match(/^#\d+ CACHED$/gm) ?? []).length,
|
||||
completedStepCount: (output.match(/^#\d+ DONE(?:\s+[\d.]+s)?$/gm) ?? []).length,
|
||||
nextCompileMs: compileDurationsMs.at(-1) ?? null,
|
||||
};
|
||||
};
|
||||
|
||||
export const getMeasuredRevision = (headSha: string, dirty: boolean) => (dirty ? `${headSha}+worktree` : headSha);
|
||||
|
||||
export const getBuildCacheStateIneligibleReasons = (
|
||||
cacheState: string | null,
|
||||
snapshot: DockerBuildCacheSnapshot | null,
|
||||
expectedMounts: ExpectedDockerBuildCacheMount[],
|
||||
) => {
|
||||
if (cacheState !== "cold" && cacheState !== "warm") {
|
||||
return ["BUILD_BENCHMARK_CACHE_STATE must be cold or warm"];
|
||||
}
|
||||
if (!snapshot) return ["Docker builder cache state could not be inspected"];
|
||||
if (cacheState === "cold" && snapshot.recordCount !== 0) {
|
||||
return [`cold builder already contains ${snapshot.recordCount} cache records`];
|
||||
}
|
||||
if (cacheState === "warm" && expectedMounts.length === 0) {
|
||||
return ["Dockerfile declares no cache mounts for a warm build"];
|
||||
}
|
||||
if (cacheState === "warm") {
|
||||
return expectedMounts.flatMap((expected) => {
|
||||
const matches = snapshot.cacheMounts.filter(
|
||||
({ logicalId, target }) => logicalId === expected.logicalId && target === expected.target,
|
||||
);
|
||||
if (matches.length === 0) {
|
||||
return [`warm builder is missing cache mount ${expected.logicalId} at ${expected.target}`];
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
return [`warm builder contains duplicate cache mounts ${expected.logicalId} at ${expected.target}`];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
export const getBuildCacheContinuityIneligibleReasons = (
|
||||
cacheState: string | null,
|
||||
before: DockerBuildCacheSnapshot | null,
|
||||
after: DockerBuildCacheSnapshot | null,
|
||||
expectedMounts: ExpectedDockerBuildCacheMount[],
|
||||
) => {
|
||||
if (cacheState !== "warm") return [];
|
||||
if (!before || !after) return ["warm builder cache continuity could not be inspected"];
|
||||
|
||||
return expectedMounts.flatMap((expected) => {
|
||||
const beforeMatches = before.cacheMounts.filter(
|
||||
({ logicalId, target }) => logicalId === expected.logicalId && target === expected.target,
|
||||
);
|
||||
const afterMatches = after.cacheMounts.filter(
|
||||
({ logicalId, target }) => logicalId === expected.logicalId && target === expected.target,
|
||||
);
|
||||
if (beforeMatches.length !== 1 || afterMatches.length !== 1) {
|
||||
return [`warm cache mount ${expected.logicalId} at ${expected.target} could not be traced across the build`];
|
||||
}
|
||||
const beforeMount = beforeMatches[0];
|
||||
const afterMount = afterMatches[0];
|
||||
if (!beforeMount || !afterMount || beforeMount.recordId !== afterMount.recordId) {
|
||||
return [`warm cache mount ${expected.logicalId} at ${expected.target} was replaced during the build`];
|
||||
}
|
||||
if (!expected.mustIncreaseUsage) return [];
|
||||
if (beforeMount.usageCount === null || afterMount.usageCount === null) {
|
||||
return [`warm cache mount ${expected.logicalId} usage could not be verified`];
|
||||
}
|
||||
if (afterMount.usageCount <= beforeMount.usageCount) {
|
||||
return [`warm cache mount ${expected.logicalId} was not used by the application build`];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
};
|
||||
|
||||
export const getDockerBuildClaimIneligibleReasons = ({
|
||||
expectedRevision,
|
||||
expectedSourceFingerprint,
|
||||
imageArchitecture,
|
||||
imageRevision,
|
||||
imageSourceFingerprint,
|
||||
expectedImageArchitecture,
|
||||
}: {
|
||||
expectedRevision: string | null;
|
||||
expectedSourceFingerprint: string;
|
||||
imageArchitecture: string;
|
||||
imageRevision: string | null;
|
||||
imageSourceFingerprint: string | null;
|
||||
expectedImageArchitecture: string;
|
||||
}) => {
|
||||
const reasons: string[] = [];
|
||||
|
||||
if (!expectedRevision) reasons.push("BUILD_BENCHMARK_REVISION is missing");
|
||||
else if (imageRevision !== expectedRevision) {
|
||||
reasons.push(`image revision ${imageRevision ?? "missing"} does not match ${expectedRevision}`);
|
||||
}
|
||||
if (imageSourceFingerprint !== expectedSourceFingerprint) {
|
||||
reasons.push("image source fingerprint does not match the measured context");
|
||||
}
|
||||
if (imageArchitecture !== expectedImageArchitecture) {
|
||||
reasons.push(`image architecture ${imageArchitecture} is not Docker server ${expectedImageArchitecture}`);
|
||||
}
|
||||
|
||||
return reasons;
|
||||
};
|
||||
@@ -0,0 +1,224 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
applyDockerBuildSyntheticSourceChange,
|
||||
fingerprintDockerBuildSource,
|
||||
getBuildCacheContinuityIneligibleReasons,
|
||||
getBuildCacheStateIneligibleReasons,
|
||||
getDockerBuildClaimIneligibleReasons,
|
||||
getMeasuredRevision,
|
||||
nextBuildCacheTarget,
|
||||
normalizeDockerArchitecture,
|
||||
parseDockerBuildCacheMountDescription,
|
||||
parseDockerBuildOutput,
|
||||
parseDockerfileCacheMounts,
|
||||
} from "./docker-build-lib.mts";
|
||||
|
||||
const pnpmMount = { logicalId: "pnpm-store", target: "/pnpm/store", mustIncreaseUsage: false };
|
||||
const nextMount = {
|
||||
logicalId: "homarr-next-build-linux/arm64",
|
||||
target: nextBuildCacheTarget,
|
||||
mustIncreaseUsage: true,
|
||||
};
|
||||
const cacheRecord = ({
|
||||
logicalId,
|
||||
recordId,
|
||||
target,
|
||||
usageCount,
|
||||
}: {
|
||||
logicalId: string;
|
||||
recordId: string;
|
||||
target: string;
|
||||
usageCount: number | null;
|
||||
}) => ({
|
||||
description: `cached mount ${target} from exec benchmark with id "/${logicalId}"`,
|
||||
logicalId,
|
||||
recordId,
|
||||
size: "1MB",
|
||||
target,
|
||||
usageCount,
|
||||
});
|
||||
|
||||
describe("Docker build benchmark", () => {
|
||||
it("fingerprints the exact imported source mutation used to force incremental compilation", () => {
|
||||
const entries = [
|
||||
{
|
||||
file: "apps/nextjs/src/app/[locale]/layout.tsx",
|
||||
mode: 0o100644,
|
||||
content: Buffer.from('openGraph: {\n siteName: "Homarr",\n },\n'),
|
||||
},
|
||||
];
|
||||
const before = fingerprintDockerBuildSource(entries);
|
||||
const { entries: changedEntries, change } = applyDockerBuildSyntheticSourceChange(entries, "paired-edit-1");
|
||||
|
||||
expect(change).toEqual({
|
||||
file: "apps/nextjs/src/app/[locale]/layout.tsx",
|
||||
id: "paired-edit-1",
|
||||
from: 'siteName: "Homarr",',
|
||||
to: 'siteName: "Homarr build benchmark paired-edit-1",',
|
||||
});
|
||||
expect(changedEntries[0]?.content?.toString("utf8")).toContain(change.to);
|
||||
expect(fingerprintDockerBuildSource(changedEntries)).not.toBe(before);
|
||||
});
|
||||
|
||||
it("fails closed when the real source marker is absent or ambiguous", () => {
|
||||
expect(() =>
|
||||
applyDockerBuildSyntheticSourceChange(
|
||||
[{ file: "apps/nextjs/src/app/[locale]/layout.tsx", mode: 0o100644, content: Buffer.from("no marker") }],
|
||||
"edit",
|
||||
),
|
||||
).toThrow("Expected exactly one build benchmark source marker");
|
||||
expect(() => applyDockerBuildSyntheticSourceChange([], "edit")).toThrow("source marker file is missing");
|
||||
});
|
||||
|
||||
it("derives the exact context-specific cache mounts for the target platform", () => {
|
||||
const dockerfile = `RUN --mount=type=cache,id=pnpm-store,target=/pnpm/store echo fetch
|
||||
RUN --mount=type=cache,id=pnpm-store,target=/pnpm/store echo install
|
||||
RUN --mount=type=cache,id=homarr-next-build-\${TARGETPLATFORM},target=${nextBuildCacheTarget},sharing=locked echo build`;
|
||||
|
||||
expect(parseDockerfileCacheMounts(dockerfile, "linux/arm64")).toEqual([nextMount, pnpmMount]);
|
||||
expect(parseDockerfileCacheMounts(dockerfile.split("\n").slice(0, 2).join("\n"), "linux/arm64")).toEqual([
|
||||
pnpmMount,
|
||||
]);
|
||||
});
|
||||
|
||||
it("fails closed for implicit or unresolved Dockerfile cache mount identity", () => {
|
||||
expect(() => parseDockerfileCacheMounts("RUN --mount=type=cache,target=/cache echo build", "linux/arm64")).toThrow(
|
||||
"explicit id and target",
|
||||
);
|
||||
expect(() =>
|
||||
parseDockerfileCacheMounts(
|
||||
"RUN --mount=type=cache,id=homarr-${BUILDPLATFORM},target=/cache echo build",
|
||||
"linux/arm64",
|
||||
),
|
||||
).toThrow("unresolved variable ${BUILDPLATFORM}");
|
||||
expect(() =>
|
||||
parseDockerfileCacheMounts(
|
||||
"RUN --mount=type=cache,id=homarr-${INVALID-NAME},target=/cache echo build",
|
||||
"linux/arm64",
|
||||
),
|
||||
).toThrow("unresolved variable");
|
||||
});
|
||||
|
||||
it("reads exact logical cache identity from real buildx du descriptions", () => {
|
||||
expect(
|
||||
parseDockerBuildCacheMountDescription(
|
||||
'cached mount /app/apps/nextjs/.next/cache from exec /bin/sh -c pnpm build with id "/homarr-next-build-linux/arm64"',
|
||||
),
|
||||
).toEqual({ logicalId: "homarr-next-build-linux/arm64", target: nextBuildCacheTarget });
|
||||
expect(parseDockerBuildCacheMountDescription("unrecognized output")).toBeNull();
|
||||
});
|
||||
|
||||
it("extracts cache and Next compilation measurements from plain BuildKit output", () => {
|
||||
const output = `#7 CACHED
|
||||
#8 DONE 0.4s
|
||||
#9 12.0 ✓ Compiled successfully in 21.7s
|
||||
#9 DONE 22.3s`;
|
||||
|
||||
expect(parseDockerBuildOutput(output)).toEqual({
|
||||
cachedStepCount: 1,
|
||||
completedStepCount: 2,
|
||||
nextCompileMs: 21_700,
|
||||
});
|
||||
});
|
||||
|
||||
it("only permits claims for a native image labeled with the exact measured source", () => {
|
||||
expect(
|
||||
getDockerBuildClaimIneligibleReasons({
|
||||
expectedRevision: "head+worktree",
|
||||
expectedSourceFingerprint: "source-hash",
|
||||
imageArchitecture: "amd64",
|
||||
imageRevision: "head+worktree",
|
||||
imageSourceFingerprint: "source-hash",
|
||||
expectedImageArchitecture: "amd64",
|
||||
}),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("normalizes Docker server architectures into OCI platform names", () => {
|
||||
expect(normalizeDockerArchitecture("x86_64")).toBe("amd64");
|
||||
expect(normalizeDockerArchitecture("aarch64")).toBe("arm64");
|
||||
expect(normalizeDockerArchitecture("armv7l")).toBe("arm/v7");
|
||||
});
|
||||
|
||||
it("derives revisions from Git state instead of trusting image-label input", () => {
|
||||
expect(getMeasuredRevision("abc123", false)).toBe("abc123");
|
||||
expect(getMeasuredRevision("abc123", true)).toBe("abc123+worktree");
|
||||
});
|
||||
|
||||
it("verifies declared cold and warm cache preconditions", () => {
|
||||
const empty = { cacheMountCount: 0, cacheMounts: [], recordCount: 0 };
|
||||
const warm = {
|
||||
cacheMountCount: 2,
|
||||
cacheMounts: [
|
||||
cacheRecord({ logicalId: pnpmMount.logicalId, recordId: "pnpm", target: pnpmMount.target, usageCount: 2 }),
|
||||
cacheRecord({ logicalId: nextMount.logicalId, recordId: "next", target: nextMount.target, usageCount: 4 }),
|
||||
],
|
||||
recordCount: 4,
|
||||
};
|
||||
expect(getBuildCacheStateIneligibleReasons("cold", empty, [pnpmMount])).toEqual([]);
|
||||
expect(getBuildCacheStateIneligibleReasons("cold", warm, [pnpmMount])).toEqual([
|
||||
"cold builder already contains 4 cache records",
|
||||
]);
|
||||
expect(getBuildCacheStateIneligibleReasons("warm", warm, [pnpmMount, nextMount])).toEqual([]);
|
||||
|
||||
const staleNext = {
|
||||
...warm,
|
||||
cacheMounts: [
|
||||
warm.cacheMounts[0]!,
|
||||
cacheRecord({
|
||||
logicalId: "homarr-next-build",
|
||||
recordId: "old-next",
|
||||
target: nextBuildCacheTarget,
|
||||
usageCount: 9,
|
||||
}),
|
||||
],
|
||||
};
|
||||
expect(getBuildCacheStateIneligibleReasons("warm", staleNext, [pnpmMount, nextMount])).toEqual([
|
||||
`warm builder is missing cache mount ${nextMount.logicalId} at ${nextBuildCacheTarget}`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("requires every warm cache record to survive and the Next cache to be used", () => {
|
||||
const before = {
|
||||
cacheMountCount: 2,
|
||||
cacheMounts: [
|
||||
cacheRecord({ logicalId: pnpmMount.logicalId, recordId: "pnpm", target: pnpmMount.target, usageCount: 2 }),
|
||||
cacheRecord({ logicalId: nextMount.logicalId, recordId: "next", target: nextMount.target, usageCount: 4 }),
|
||||
],
|
||||
recordCount: 4,
|
||||
};
|
||||
const after = {
|
||||
cacheMountCount: 2,
|
||||
cacheMounts: [
|
||||
cacheRecord({ logicalId: pnpmMount.logicalId, recordId: "pnpm", target: pnpmMount.target, usageCount: 2 }),
|
||||
cacheRecord({ logicalId: nextMount.logicalId, recordId: "next", target: nextMount.target, usageCount: 6 }),
|
||||
],
|
||||
recordCount: 8,
|
||||
};
|
||||
|
||||
expect(getBuildCacheContinuityIneligibleReasons("warm", before, after, [pnpmMount, nextMount])).toEqual([]);
|
||||
expect(
|
||||
getBuildCacheContinuityIneligibleReasons(
|
||||
"warm",
|
||||
before,
|
||||
{
|
||||
...after,
|
||||
cacheMounts: [after.cacheMounts[0]!, { ...after.cacheMounts[1]!, recordId: "replacement" }],
|
||||
},
|
||||
[pnpmMount, nextMount],
|
||||
),
|
||||
).toEqual([`warm cache mount ${nextMount.logicalId} at ${nextBuildCacheTarget} was replaced during the build`]);
|
||||
expect(
|
||||
getBuildCacheContinuityIneligibleReasons(
|
||||
"warm",
|
||||
before,
|
||||
{
|
||||
...after,
|
||||
cacheMounts: [after.cacheMounts[0]!, { ...after.cacheMounts[1]!, usageCount: 4 }],
|
||||
},
|
||||
[pnpmMount, nextMount],
|
||||
),
|
||||
).toEqual([`warm cache mount ${nextMount.logicalId} was not used by the application build`]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,351 @@
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, lstat, mkdir, mkdtemp, readFile, readlink, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import {
|
||||
applyDockerBuildSyntheticSourceChange,
|
||||
fingerprintDockerBuildSource,
|
||||
getBuildCacheContinuityIneligibleReasons,
|
||||
getBuildCacheStateIneligibleReasons,
|
||||
getDockerBuildClaimIneligibleReasons,
|
||||
getMeasuredRevision,
|
||||
normalizeDockerArchitecture,
|
||||
parseDockerBuildCacheMountDescription,
|
||||
parseDockerBuildOutput,
|
||||
parseDockerfileCacheMounts,
|
||||
} from "./docker-build-lib.mts";
|
||||
import type { DockerBuildSourceEntry } from "./docker-build-lib.mts";
|
||||
|
||||
const runStartedAt = new Date().toISOString();
|
||||
const execFileAsync = promisify(execFile);
|
||||
const contextDirectory = path.resolve(process.env.BUILD_BENCHMARK_CONTEXT ?? ".");
|
||||
const declaredRevision = process.env.BUILD_BENCHMARK_REVISION?.trim() || null;
|
||||
const mode = process.env.BUILD_BENCHMARK_MODE ?? "cached";
|
||||
if (mode !== "cached" && mode !== "no-cache" && mode !== "source-change") {
|
||||
throw new Error("BUILD_BENCHMARK_MODE must be cached, no-cache, or source-change");
|
||||
}
|
||||
const sourceChangeId = process.env.BUILD_BENCHMARK_SOURCE_CHANGE_ID?.trim() || null;
|
||||
if (mode === "source-change" && !sourceChangeId) {
|
||||
throw new Error("BUILD_BENCHMARK_SOURCE_CHANGE_ID is required in source-change mode");
|
||||
}
|
||||
|
||||
const imageTag = process.env.BUILD_BENCHMARK_IMAGE ?? `homarr-build-benchmark:${process.pid}`;
|
||||
const builder = process.env.BUILD_BENCHMARK_BUILDER?.trim() || null;
|
||||
const runLabel = process.env.BUILD_BENCHMARK_RUN_LABEL?.trim() || null;
|
||||
const seriesId = process.env.BUILD_BENCHMARK_SERIES_ID?.trim() || null;
|
||||
const cacheState = process.env.BUILD_BENCHMARK_CACHE_STATE?.trim() || null;
|
||||
const repetition = Number(process.env.BUILD_BENCHMARK_REPETITION ?? 0);
|
||||
const totalRepetitions = Number(process.env.BUILD_BENCHMARK_TOTAL_REPETITIONS ?? 0);
|
||||
const outputDirectory = path.resolve(process.env.BUILD_BENCHMARK_OUTPUT_DIR ?? "benchmark-results/docker-build");
|
||||
const cacheSnapshotPollMs = 2_000;
|
||||
const cacheSnapshotStableMs = 10_000;
|
||||
const cacheSnapshotTimeoutMs = 30_000;
|
||||
const execGitAsync = async (args: string[]) =>
|
||||
await execFileAsync("git", ["-C", contextDirectory, ...args], { maxBuffer: 16 * 1024 * 1024 });
|
||||
|
||||
const inspectDockerServerAsync = async () => {
|
||||
const { stdout } = await execFileAsync("docker", ["version", "--format", "{{json .Server}}"], {
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
const server = JSON.parse(stdout) as {
|
||||
ApiVersion?: string;
|
||||
Arch?: string;
|
||||
Os?: string;
|
||||
Version?: string;
|
||||
} | null;
|
||||
if (!server?.Arch || !server.Os) throw new Error("Docker server OS and architecture could not be inspected");
|
||||
const architecture = normalizeDockerArchitecture(server.Arch);
|
||||
return {
|
||||
apiVersion: server.ApiVersion ?? null,
|
||||
architecture,
|
||||
operatingSystem: server.Os.toLowerCase(),
|
||||
targetPlatform: `${server.Os.toLowerCase()}/${architecture}`,
|
||||
version: server.Version ?? null,
|
||||
};
|
||||
};
|
||||
|
||||
const fingerprintSourceAsync = async () => {
|
||||
const { stdout } = await execGitAsync(["ls-files", "--cached", "--others", "--exclude-standard", "-z"]);
|
||||
const files = stdout.split("\0").filter(Boolean).toSorted();
|
||||
let fileCount = 0;
|
||||
const entries: DockerBuildSourceEntry[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const absolutePath = path.join(contextDirectory, file);
|
||||
const stats = await lstat(absolutePath).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT") return null;
|
||||
throw error;
|
||||
});
|
||||
if (!stats) continue;
|
||||
fileCount += 1;
|
||||
const entry: DockerBuildSourceEntry = stats.isSymbolicLink()
|
||||
? { file, mode: stats.mode, linkTarget: await readlink(absolutePath) }
|
||||
: { file, mode: stats.mode, content: await readFile(absolutePath) };
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
return { fingerprint: fingerprintDockerBuildSource(entries), fileCount, entries };
|
||||
};
|
||||
|
||||
const materializeSourceSnapshotAsync = async (entries: DockerBuildSourceEntry[]) => {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), "homarr-build-benchmark-"));
|
||||
for (const entry of entries) {
|
||||
const destination = path.join(directory, entry.file);
|
||||
await mkdir(path.dirname(destination), { recursive: true });
|
||||
if (entry.linkTarget !== undefined) {
|
||||
await symlink(entry.linkTarget, destination);
|
||||
} else {
|
||||
await writeFile(destination, entry.content);
|
||||
await chmod(destination, entry.mode & 0o777);
|
||||
}
|
||||
}
|
||||
return directory;
|
||||
};
|
||||
|
||||
const runDockerBuildAsync = async (args: string[]) =>
|
||||
await new Promise<{ output: string; wallTimeMs: number }>((resolve, reject) => {
|
||||
const startedAt = performance.now();
|
||||
const child = spawn("docker", args, { cwd: contextDirectory, env: process.env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
const chunks: Buffer[] = [];
|
||||
const capture = (chunk: Buffer) => {
|
||||
chunks.push(chunk);
|
||||
process.stdout.write(chunk);
|
||||
};
|
||||
child.stdout.on("data", capture);
|
||||
child.stderr.on("data", capture);
|
||||
child.once("error", reject);
|
||||
child.once("close", (code) => {
|
||||
const output = Buffer.concat(chunks).toString("utf8");
|
||||
const wallTimeMs = Math.round(performance.now() - startedAt);
|
||||
if (code !== 0) {
|
||||
reject(new Error(`docker build exited with ${String(code)} after ${wallTimeMs}ms`));
|
||||
return;
|
||||
}
|
||||
resolve({ output, wallTimeMs });
|
||||
});
|
||||
});
|
||||
|
||||
const inspectBuilderCacheAsync = async () => {
|
||||
if (!builder) return null;
|
||||
await execFileAsync("docker", ["buildx", "inspect", "--builder", builder, "--bootstrap"], {
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
const { stdout } = await execFileAsync("docker", ["buildx", "du", "--builder", builder, "--format", "json"], {
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
const records = stdout
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(
|
||||
(line) =>
|
||||
JSON.parse(line) as {
|
||||
CreatedAt?: string;
|
||||
Description?: string;
|
||||
ID: string;
|
||||
Mutable?: boolean;
|
||||
Parents?: string[];
|
||||
Reclaimable?: boolean;
|
||||
Shared?: boolean;
|
||||
Size?: string;
|
||||
Type?: string;
|
||||
UsageCount?: number;
|
||||
},
|
||||
);
|
||||
const normalizedRecords = records
|
||||
.map(({ CreatedAt, Description, ID, Mutable, Parents, Reclaimable, Shared, Size, Type, UsageCount }) => ({
|
||||
createdAt: CreatedAt ?? null,
|
||||
description: Description ?? null,
|
||||
id: ID,
|
||||
mutable: Mutable ?? null,
|
||||
parents: Parents?.toSorted() ?? [],
|
||||
reclaimable: Reclaimable ?? null,
|
||||
shared: Shared ?? null,
|
||||
size: Size ?? null,
|
||||
type: Type ?? null,
|
||||
usageCount: UsageCount ?? null,
|
||||
}))
|
||||
.toSorted((left, right) => left.id.localeCompare(right.id));
|
||||
const cacheMounts = records
|
||||
.filter(({ Type }) => Type === "exec.cachemount")
|
||||
.map(({ Description, ID, Size, UsageCount }) => {
|
||||
const description = Description ?? null;
|
||||
const identity = parseDockerBuildCacheMountDescription(description);
|
||||
return {
|
||||
description,
|
||||
logicalId: identity?.logicalId ?? null,
|
||||
recordId: ID,
|
||||
size: Size ?? null,
|
||||
target: identity?.target ?? null,
|
||||
usageCount: UsageCount ?? null,
|
||||
};
|
||||
});
|
||||
return {
|
||||
cacheMountCount: cacheMounts.length,
|
||||
cacheMounts,
|
||||
recordCount: records.length,
|
||||
recordsFingerprint: createHash("sha256").update(JSON.stringify(normalizedRecords)).digest("hex"),
|
||||
};
|
||||
};
|
||||
|
||||
const inspectSettledBuilderCacheAsync = async () => {
|
||||
if (!builder) return { snapshot: null, stability: null };
|
||||
const startedAt = performance.now();
|
||||
let attempts = 1;
|
||||
let previous = await inspectBuilderCacheAsync();
|
||||
let stableSince = performance.now();
|
||||
|
||||
while (performance.now() - startedAt < cacheSnapshotTimeoutMs) {
|
||||
await new Promise((resolve) => setTimeout(resolve, cacheSnapshotPollMs));
|
||||
const current = await inspectBuilderCacheAsync();
|
||||
attempts += 1;
|
||||
if (current?.recordsFingerprint !== previous?.recordsFingerprint) stableSince = performance.now();
|
||||
previous = current;
|
||||
const stableForMs = Math.round(performance.now() - stableSince);
|
||||
if (stableForMs >= cacheSnapshotStableMs) {
|
||||
return {
|
||||
snapshot: current,
|
||||
stability: { attempts, pollIntervalMs: cacheSnapshotPollMs, stableForMs, timeoutMs: cacheSnapshotTimeoutMs },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Docker builder cache did not settle for ${cacheSnapshotStableMs}ms`);
|
||||
};
|
||||
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const [{ stdout: headOutput }, { stdout: statusOutput }, source, dockerServer] = await Promise.all([
|
||||
execGitAsync(["rev-parse", "HEAD"]),
|
||||
execGitAsync(["status", "--porcelain=v1"]),
|
||||
fingerprintSourceAsync(),
|
||||
inspectDockerServerAsync(),
|
||||
]);
|
||||
const headSha = headOutput.trim();
|
||||
const dirty = statusOutput.trim().length > 0;
|
||||
const measuredRevision = getMeasuredRevision(headSha, dirty);
|
||||
const dockerfile = source.entries.find(({ file }) => file === "Dockerfile");
|
||||
if (!dockerfile?.content) throw new Error("Docker build context contains no regular Dockerfile");
|
||||
const expectedCacheMounts = parseDockerfileCacheMounts(
|
||||
dockerfile.content.toString("utf8"),
|
||||
dockerServer.targetPlatform,
|
||||
);
|
||||
const cacheBefore = await inspectBuilderCacheAsync();
|
||||
const syntheticSource =
|
||||
mode === "source-change"
|
||||
? applyDockerBuildSyntheticSourceChange(source.entries, sourceChangeId as string)
|
||||
: { entries: source.entries, change: null };
|
||||
const measuredSourceFingerprint = fingerprintDockerBuildSource(syntheticSource.entries);
|
||||
|
||||
const buildArgs = [
|
||||
"build",
|
||||
"--progress=plain",
|
||||
"--platform",
|
||||
dockerServer.targetPlatform,
|
||||
"--label",
|
||||
`org.opencontainers.image.revision=${measuredRevision}`,
|
||||
"--label",
|
||||
`dev.homarr.source-fingerprint=${measuredSourceFingerprint}`,
|
||||
"--tag",
|
||||
imageTag,
|
||||
];
|
||||
if (builder) buildArgs.push("--builder", builder, "--load");
|
||||
if (mode === "no-cache") buildArgs.push("--no-cache");
|
||||
const snapshotDirectory = await materializeSourceSnapshotAsync(syntheticSource.entries);
|
||||
buildArgs.push(snapshotDirectory);
|
||||
|
||||
const build = await runDockerBuildAsync(buildArgs).finally(
|
||||
async () => await rm(snapshotDirectory, { recursive: true }),
|
||||
);
|
||||
const { snapshot: cacheAfter, stability: cacheAfterStability } = await inspectSettledBuilderCacheAsync();
|
||||
const imageInspect = JSON.parse((await execFileAsync("docker", ["image", "inspect", imageTag])).stdout)[0] as {
|
||||
Architecture: string;
|
||||
Config?: { Labels?: Record<string, string> };
|
||||
Id: string;
|
||||
RepoDigests?: string[];
|
||||
Size: number;
|
||||
};
|
||||
const labels = imageInspect.Config?.Labels ?? {};
|
||||
const parsedBuild = parseDockerBuildOutput(build.output);
|
||||
const claimIneligibleReasons = getDockerBuildClaimIneligibleReasons({
|
||||
expectedRevision: measuredRevision,
|
||||
expectedSourceFingerprint: measuredSourceFingerprint,
|
||||
imageArchitecture: imageInspect.Architecture,
|
||||
imageRevision: labels["org.opencontainers.image.revision"] ?? null,
|
||||
imageSourceFingerprint: labels["dev.homarr.source-fingerprint"] ?? null,
|
||||
expectedImageArchitecture: dockerServer.architecture.split("/")[0] as string,
|
||||
});
|
||||
if (!declaredRevision) claimIneligibleReasons.push("BUILD_BENCHMARK_REVISION is missing");
|
||||
else if (declaredRevision !== measuredRevision) {
|
||||
claimIneligibleReasons.push(`declared revision ${declaredRevision} does not match measured ${measuredRevision}`);
|
||||
}
|
||||
if (!builder) claimIneligibleReasons.push("BUILD_BENCHMARK_BUILDER is missing");
|
||||
if (!seriesId) claimIneligibleReasons.push("BUILD_BENCHMARK_SERIES_ID is missing");
|
||||
claimIneligibleReasons.push(...getBuildCacheStateIneligibleReasons(cacheState, cacheBefore, expectedCacheMounts));
|
||||
claimIneligibleReasons.push(
|
||||
...getBuildCacheContinuityIneligibleReasons(cacheState, cacheBefore, cacheAfter, expectedCacheMounts),
|
||||
);
|
||||
if (mode === "cached") claimIneligibleReasons.push("cached mode does not force an application rebuild");
|
||||
if (mode === "no-cache" && cacheState === "warm") {
|
||||
claimIneligibleReasons.push("no-cache mode resets BuildKit cache mounts and cannot measure a warm build");
|
||||
}
|
||||
if (mode !== "cached" && parsedBuild.nextCompileMs === null) {
|
||||
claimIneligibleReasons.push("application build did not expose a Next compilation measurement");
|
||||
}
|
||||
if (!Number.isInteger(totalRepetitions) || totalRepetitions < 2) {
|
||||
claimIneligibleReasons.push("BUILD_BENCHMARK_TOTAL_REPETITIONS must be at least 2");
|
||||
}
|
||||
if (!Number.isInteger(repetition) || repetition < 1 || repetition > totalRepetitions) {
|
||||
claimIneligibleReasons.push("BUILD_BENCHMARK_REPETITION is outside the declared series");
|
||||
}
|
||||
const timestamp = Date.now();
|
||||
const logPath = path.join(outputDirectory, `build-${timestamp}.log`);
|
||||
const outputPath = path.join(outputDirectory, `build-${timestamp}.json`);
|
||||
const runFinishedAt = new Date().toISOString();
|
||||
const result = {
|
||||
schemaVersion: 4,
|
||||
claimEligible: claimIneligibleReasons.length === 0,
|
||||
claimIneligibleReasons,
|
||||
createdAt: runFinishedAt,
|
||||
run: { startedAt: runStartedAt, finishedAt: runFinishedAt },
|
||||
mode,
|
||||
builder,
|
||||
runLabel,
|
||||
declaredRevision,
|
||||
series: { id: seriesId, cacheState, repetition, totalRepetitions },
|
||||
source: {
|
||||
contextDirectory,
|
||||
headSha,
|
||||
dirty,
|
||||
measuredRevision,
|
||||
fingerprint: measuredSourceFingerprint,
|
||||
baseFingerprint: source.fingerprint,
|
||||
fileCount: source.fileCount,
|
||||
immutableSnapshot: true,
|
||||
syntheticSourceChange: syntheticSource.change,
|
||||
},
|
||||
builderCache: {
|
||||
expected: expectedCacheMounts,
|
||||
before: cacheBefore,
|
||||
after: cacheAfter,
|
||||
afterStability: cacheAfterStability,
|
||||
},
|
||||
host: { architecture: process.arch, cpuCount: os.cpus().length, dockerServer, platform: os.platform() },
|
||||
image: {
|
||||
requested: imageTag,
|
||||
id: imageInspect.Id,
|
||||
digest: imageInspect.RepoDigests?.[0] ?? null,
|
||||
revision: labels["org.opencontainers.image.revision"] ?? null,
|
||||
sourceFingerprint: labels["dev.homarr.source-fingerprint"] ?? null,
|
||||
architecture: imageInspect.Architecture,
|
||||
targetPlatform: dockerServer.targetPlatform,
|
||||
localUncompressedBytes: imageInspect.Size,
|
||||
},
|
||||
build: { wallTimeMs: build.wallTimeMs, ...parsedBuild },
|
||||
logPath,
|
||||
};
|
||||
|
||||
await Promise.all([writeFile(logPath, build.output), writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`)]);
|
||||
console.log(JSON.stringify({ outputPath, ...result }, null, 2));
|
||||
@@ -0,0 +1,649 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { jsonlStreamConsumer } from "@trpc/server/unstable-core-do-not-import";
|
||||
import { deserialize } from "superjson";
|
||||
import type { SuperJSONResult } from "superjson";
|
||||
|
||||
import { isWidgetDataTrpcPath } from "../../packages/api/src/query-cache.ts";
|
||||
|
||||
export type RuntimeMemoryCheckpoint = {
|
||||
name: string;
|
||||
capturedAt: string;
|
||||
container: {
|
||||
currentBytes: number;
|
||||
peakBytes: number | null;
|
||||
anonymousBytes: number;
|
||||
fileBytes: number;
|
||||
};
|
||||
cpu: {
|
||||
usageUsec: number;
|
||||
userUsec: number;
|
||||
systemUsec: number;
|
||||
nrThrottled: number;
|
||||
throttledUsec: number;
|
||||
};
|
||||
processes: Array<{
|
||||
command: string;
|
||||
parentPid: number;
|
||||
pid: number;
|
||||
pssBytes: number;
|
||||
rssBytes: number;
|
||||
}>;
|
||||
redis: { usedMemoryBytes: number; peakMemoryBytes: number } | null;
|
||||
};
|
||||
|
||||
export const minimumClaimSettleMs = 10 * 60 * 1_000;
|
||||
export const minimumClaimSettleSamples = 20;
|
||||
export const minimumClaimPageSamples = 20;
|
||||
export const minimumClaimLcpObservationMs = 5_000;
|
||||
export const minimumClaimWidgetDataQuietMs = 500;
|
||||
|
||||
export const getBrowserNetworkIsolationLaunchArgs = (denyProxyUrl: string) => [
|
||||
`--proxy-server=${denyProxyUrl}`,
|
||||
"--proxy-bypass-list=127.0.0.1;localhost",
|
||||
"--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE 127.0.0.1, EXCLUDE localhost",
|
||||
];
|
||||
|
||||
export const normalizeDockerArchitecture = (nodeArchitecture: string) =>
|
||||
nodeArchitecture === "x64" ? "amd64" : nodeArchitecture;
|
||||
|
||||
export const getTrpcProcedures = (requestUrl: string) => {
|
||||
const pathname = new URL(requestUrl).pathname;
|
||||
const prefix = "/api/trpc/";
|
||||
const index = pathname.indexOf(prefix);
|
||||
if (index < 0) return [];
|
||||
|
||||
return decodeURIComponent(pathname.slice(index + prefix.length))
|
||||
.split(",")
|
||||
.filter(Boolean);
|
||||
};
|
||||
|
||||
const canonicalizeJson = (value: unknown): string => {
|
||||
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "undefined";
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalizeJson).join(",")}]`;
|
||||
|
||||
return `{${Object.entries(value)
|
||||
.toSorted(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
||||
.map(([key, nestedValue]) => `${JSON.stringify(key)}:${canonicalizeJson(nestedValue)}`)
|
||||
.join(",")}}`;
|
||||
};
|
||||
|
||||
const hashRequestInput = (value: unknown) => createHash("sha256").update(canonicalizeJson(value)).digest("hex");
|
||||
|
||||
export type TrpcRequestIdentity = { batchIndex: number; identity: string; inputHash: string; procedure: string };
|
||||
|
||||
export const getTrpcRequestIdentities = (requestUrl: string): TrpcRequestIdentity[] => {
|
||||
const url = new URL(requestUrl);
|
||||
const procedures = getTrpcProcedures(requestUrl);
|
||||
if (procedures.length === 0) return [];
|
||||
const rawInput = url.searchParams.get("input");
|
||||
let parsedInput: unknown = null;
|
||||
if (rawInput !== null) {
|
||||
try {
|
||||
parsedInput = JSON.parse(rawInput) as unknown;
|
||||
} catch {
|
||||
throw new Error("Malformed serialized tRPC request input");
|
||||
}
|
||||
}
|
||||
|
||||
const isBatch = url.searchParams.get("batch") === "1";
|
||||
return procedures.map((procedure, index) => {
|
||||
const input =
|
||||
isBatch && parsedInput !== null && typeof parsedInput === "object" && !Array.isArray(parsedInput)
|
||||
? (parsedInput as Record<string, unknown>)[String(index)]
|
||||
: parsedInput;
|
||||
const inputHash = hashRequestInput(input);
|
||||
return { batchIndex: index, procedure, inputHash, identity: `${procedure}#${inputHash}` };
|
||||
});
|
||||
};
|
||||
|
||||
export const isRuntimeWidgetDataProcedure = (procedure: string) => {
|
||||
if (procedure.split(".").some((part) => part.startsWith("subscribe"))) return false;
|
||||
if (procedure.startsWith("widget.")) return procedure !== "widget.app.ping";
|
||||
return isWidgetDataTrpcPath(procedure);
|
||||
};
|
||||
|
||||
export const isRuntimeWidgetDataRequest = (method: string, procedure: string) =>
|
||||
method === "GET" && isRuntimeWidgetDataProcedure(procedure);
|
||||
|
||||
export const recordWidgetDataRequestOutcome = (
|
||||
recoveryBalance: Map<string, number>,
|
||||
requestIdentities: string[],
|
||||
outcome: "aborted" | "completed",
|
||||
) => {
|
||||
const delta = outcome === "aborted" ? 1 : -1;
|
||||
for (const requestIdentity of requestIdentities) {
|
||||
recoveryBalance.set(requestIdentity, (recoveryBalance.get(requestIdentity) ?? 0) + delta);
|
||||
}
|
||||
};
|
||||
|
||||
export const getUnrecoveredWidgetDataRequests = (recoveryBalance: Map<string, number>) =>
|
||||
[...recoveryBalance.entries()]
|
||||
.filter(([, balance]) => balance > 0)
|
||||
.flatMap(([requestIdentity, balance]) => Array.from({ length: balance }, () => requestIdentity))
|
||||
.toSorted();
|
||||
|
||||
export const hasBlockingWidgetDataAborts = (allowUnrecoveredAborts: boolean, unrecoveredProcedureCount: number) =>
|
||||
!allowUnrecoveredAborts && unrecoveredProcedureCount > 0;
|
||||
|
||||
export const getSpotlightIdleNetworkFailure = (policy: string, nonWidgetTrpcRequestCount: number) =>
|
||||
policy === "preload-only" && nonWidgetTrpcRequestCount > 0
|
||||
? `Idle Spotlight preload-only preparation made ${nonWidgetTrpcRequestCount} non-widget tRPC requests`
|
||||
: null;
|
||||
|
||||
export const getSpotlightIdleMountFailure = (policy: string, mountedCount: number) =>
|
||||
policy === "preload-only" && mountedCount > 0 ? "Idle Spotlight preload mounted the UI before user intent" : null;
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
|
||||
export type TrpcOperationFailure = {
|
||||
batchIndex: number;
|
||||
kind: "http-status" | "invalid-jsonl" | "index-mismatch" | "invalid-envelope" | "trpc-error" | "stream-error";
|
||||
httpStatus?: number;
|
||||
jsonRpcCode?: number;
|
||||
status?: number;
|
||||
trpcCode?: string;
|
||||
};
|
||||
|
||||
export type TrpcBatchInspection = { failures: TrpcOperationFailure[]; successfulIndexes: number[] };
|
||||
|
||||
const failEveryOperation = (
|
||||
operationCount: number,
|
||||
failure: Omit<TrpcOperationFailure, "batchIndex">,
|
||||
): TrpcBatchInspection => ({
|
||||
failures: Array.from({ length: operationCount }, (_, batchIndex) => ({ batchIndex, ...failure })),
|
||||
successfulIndexes: [],
|
||||
});
|
||||
|
||||
const safeTrpcError = (batchIndex: number, error: unknown): TrpcOperationFailure => {
|
||||
const record = isRecord(error) ? error : {};
|
||||
const data = isRecord(record.data) ? record.data : {};
|
||||
return {
|
||||
batchIndex,
|
||||
kind: "trpc-error",
|
||||
...(typeof data.httpStatus === "number" ? { httpStatus: data.httpStatus } : {}),
|
||||
...(typeof record.code === "number" ? { jsonRpcCode: record.code } : {}),
|
||||
...(typeof data.code === "string" ? { trpcCode: data.code } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
export const inspectTrpcBatchStreamResponse = async (
|
||||
status: number,
|
||||
responseBody: string,
|
||||
expectedOperationCount: number,
|
||||
): Promise<TrpcBatchInspection> => {
|
||||
if (!Number.isInteger(expectedOperationCount) || expectedOperationCount < 1) {
|
||||
throw new Error("expectedOperationCount must be a positive integer");
|
||||
}
|
||||
if (status < 200 || status >= 300) {
|
||||
return failEveryOperation(expectedOperationCount, { kind: "http-status", status });
|
||||
}
|
||||
if (responseBody.trim().length === 0) return failEveryOperation(expectedOperationCount, { kind: "invalid-jsonl" });
|
||||
|
||||
const physicalLines = responseBody.split("\n");
|
||||
const trailing = physicalLines.pop() ?? "";
|
||||
if (trailing.trim() !== "") return failEveryOperation(expectedOperationCount, { kind: "invalid-jsonl" });
|
||||
try {
|
||||
const records = physicalLines.filter((line) => line.trim().length > 0);
|
||||
if (records.length === 0) throw new Error("missing JSONL records");
|
||||
for (const line of records) {
|
||||
const serialized = JSON.parse(line) as unknown;
|
||||
if (!isRecord(serialized) || !Object.hasOwn(serialized, "json")) throw new Error("not SuperJSON");
|
||||
deserialize(serialized as SuperJSONResult);
|
||||
}
|
||||
} catch {
|
||||
return failEveryOperation(expectedOperationCount, { kind: "invalid-jsonl" });
|
||||
}
|
||||
|
||||
const abortController = new AbortController();
|
||||
try {
|
||||
const bytes = new TextEncoder().encode(responseBody);
|
||||
const from = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(bytes);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const [head] = await jsonlStreamConsumer<Record<string, Promise<unknown>>>({
|
||||
abortController,
|
||||
deserialize: (value) => deserialize(value as SuperJSONResult),
|
||||
formatError: () => new Error("tRPC stream operation error"),
|
||||
from,
|
||||
});
|
||||
if (!isRecord(head)) return failEveryOperation(expectedOperationCount, { kind: "invalid-envelope" });
|
||||
|
||||
const actualIndexes = Object.keys(head);
|
||||
const expectedIndexes = Array.from({ length: expectedOperationCount }, (_, index) => String(index));
|
||||
if (
|
||||
actualIndexes.length !== expectedIndexes.length ||
|
||||
expectedIndexes.some((index) => !Object.hasOwn(head, index))
|
||||
) {
|
||||
for (const value of Object.values(head)) void Promise.resolve(value).catch(() => undefined);
|
||||
return failEveryOperation(expectedOperationCount, { kind: "index-mismatch" });
|
||||
}
|
||||
|
||||
const outcomes = await Promise.all(
|
||||
expectedIndexes.map(async (key): Promise<{ failure: TrpcOperationFailure | null; index: number }> => {
|
||||
const batchIndex = Number(key);
|
||||
try {
|
||||
const envelope = await Promise.resolve(head[key]);
|
||||
if (!isRecord(envelope)) return { failure: { batchIndex, kind: "invalid-envelope" }, index: batchIndex };
|
||||
if (Object.hasOwn(envelope, "error")) {
|
||||
return { failure: safeTrpcError(batchIndex, envelope.error), index: batchIndex };
|
||||
}
|
||||
if (!Object.hasOwn(envelope, "result")) {
|
||||
return { failure: { batchIndex, kind: "invalid-envelope" }, index: batchIndex };
|
||||
}
|
||||
const result = await Promise.resolve(envelope.result);
|
||||
if (!isRecord(result) || !Object.hasOwn(result, "data")) {
|
||||
return { failure: { batchIndex, kind: "invalid-envelope" }, index: batchIndex };
|
||||
}
|
||||
await Promise.resolve(result.data);
|
||||
return { failure: null, index: batchIndex };
|
||||
} catch {
|
||||
return { failure: { batchIndex, kind: "stream-error" }, index: batchIndex };
|
||||
}
|
||||
}),
|
||||
);
|
||||
return {
|
||||
failures: outcomes.flatMap(({ failure }) => (failure ? [failure] : [])),
|
||||
successfulIndexes: outcomes.flatMap(({ failure, index }) => (failure ? [] : [index])),
|
||||
};
|
||||
} catch {
|
||||
return failEveryOperation(expectedOperationCount, { kind: "invalid-jsonl" });
|
||||
} finally {
|
||||
abortController.abort();
|
||||
}
|
||||
};
|
||||
|
||||
type RuntimeMetricEligibilityInput = {
|
||||
expectedSha: string | null;
|
||||
expectedSourceFingerprint: string | null;
|
||||
browserNetworkIsolated: boolean;
|
||||
serverNetworkIsolated: boolean;
|
||||
imageArchitecture: string;
|
||||
lcpObservationMs: number;
|
||||
lcpSampleCount: number;
|
||||
nodeArchitecture: string;
|
||||
pageSampleCount: number;
|
||||
requireImplementationMarkers: boolean;
|
||||
revision: string | null;
|
||||
sourceFingerprint: string | null;
|
||||
settleMs: number;
|
||||
settleSampleCount: number;
|
||||
widgetDataQuietMs: number;
|
||||
widgetDataSampleCount: number;
|
||||
};
|
||||
|
||||
const getRuntimeProvenanceIneligibleReasons = ({
|
||||
browserNetworkIsolated,
|
||||
serverNetworkIsolated,
|
||||
expectedSha,
|
||||
expectedSourceFingerprint,
|
||||
imageArchitecture,
|
||||
nodeArchitecture,
|
||||
revision,
|
||||
sourceFingerprint,
|
||||
}: Pick<
|
||||
RuntimeMetricEligibilityInput,
|
||||
| "browserNetworkIsolated"
|
||||
| "serverNetworkIsolated"
|
||||
| "expectedSha"
|
||||
| "expectedSourceFingerprint"
|
||||
| "imageArchitecture"
|
||||
| "nodeArchitecture"
|
||||
| "revision"
|
||||
| "sourceFingerprint"
|
||||
>) => {
|
||||
const reasons: string[] = [];
|
||||
const expectedArchitecture = normalizeDockerArchitecture(nodeArchitecture);
|
||||
|
||||
if (imageArchitecture !== expectedArchitecture) {
|
||||
reasons.push(`image architecture ${imageArchitecture} is not host ${expectedArchitecture}`);
|
||||
}
|
||||
if (!expectedSha) reasons.push("RUNTIME_BENCHMARK_SHA is missing");
|
||||
else if (!revision) reasons.push("image revision label is missing");
|
||||
else if (revision !== expectedSha) reasons.push(`image revision ${revision} does not match ${expectedSha}`);
|
||||
if (!expectedSourceFingerprint) reasons.push("RUNTIME_BENCHMARK_SOURCE_FINGERPRINT is missing");
|
||||
else if (!sourceFingerprint) reasons.push("image source fingerprint label is missing");
|
||||
else if (sourceFingerprint !== expectedSourceFingerprint) {
|
||||
reasons.push(`image source fingerprint ${sourceFingerprint} does not match ${expectedSourceFingerprint}`);
|
||||
}
|
||||
if (!browserNetworkIsolated) reasons.push("browser external network isolation is not active");
|
||||
if (!serverNetworkIsolated) reasons.push("server external network isolation is not active");
|
||||
|
||||
return reasons;
|
||||
};
|
||||
|
||||
const createEligibility = ({
|
||||
ineligibleReasons,
|
||||
kind,
|
||||
limitations,
|
||||
requiredSampleCount,
|
||||
sampleCount,
|
||||
sampleUnit,
|
||||
scope,
|
||||
}: {
|
||||
ineligibleReasons: string[];
|
||||
kind: "comparative" | "diagnostic" | "within-run";
|
||||
limitations: string[];
|
||||
requiredSampleCount: number | null;
|
||||
sampleCount: number;
|
||||
sampleUnit: string;
|
||||
scope: string;
|
||||
}) => ({
|
||||
claimEligible: ineligibleReasons.length === 0,
|
||||
ineligibleReasons,
|
||||
kind,
|
||||
limitations,
|
||||
requiredSampleCount,
|
||||
sampleCount,
|
||||
sampleUnit,
|
||||
scope,
|
||||
});
|
||||
|
||||
export const getRuntimeMetricEligibility = (input: RuntimeMetricEligibilityInput) => {
|
||||
const provenanceReasons = getRuntimeProvenanceIneligibleReasons(input);
|
||||
const markerReasons = input.requireImplementationMarkers ? [] : ["widget implementation markers are not required"];
|
||||
const pageReasons = [
|
||||
...provenanceReasons,
|
||||
...markerReasons,
|
||||
...(input.pageSampleCount < minimumClaimPageSamples
|
||||
? [`page sample count ${input.pageSampleCount} is below ${minimumClaimPageSamples}`]
|
||||
: []),
|
||||
];
|
||||
const lcpReasons = [
|
||||
...provenanceReasons,
|
||||
...markerReasons,
|
||||
...(input.lcpObservationMs < minimumClaimLcpObservationMs
|
||||
? [`LCP observation ${input.lcpObservationMs}ms is below ${minimumClaimLcpObservationMs}ms`]
|
||||
: []),
|
||||
...(input.lcpSampleCount < minimumClaimPageSamples
|
||||
? [`LCP sample count ${input.lcpSampleCount} is below ${minimumClaimPageSamples}`]
|
||||
: []),
|
||||
];
|
||||
const settleReasons = [
|
||||
...provenanceReasons,
|
||||
...markerReasons,
|
||||
...(input.settleMs < minimumClaimSettleMs
|
||||
? [`settle duration ${input.settleMs}ms is below ${minimumClaimSettleMs}ms`]
|
||||
: []),
|
||||
...(input.settleSampleCount < minimumClaimSettleSamples
|
||||
? [`settle sample count ${input.settleSampleCount} is below ${minimumClaimSettleSamples}`]
|
||||
: []),
|
||||
];
|
||||
const widgetDataReasons = [
|
||||
...provenanceReasons,
|
||||
...markerReasons,
|
||||
...(input.widgetDataQuietMs < minimumClaimWidgetDataQuietMs
|
||||
? [`widget data quiet window ${input.widgetDataQuietMs}ms is below ${minimumClaimWidgetDataQuietMs}ms`]
|
||||
: []),
|
||||
...(input.widgetDataSampleCount < minimumClaimPageSamples
|
||||
? [`widget data sample count ${input.widgetDataSampleCount} is below ${minimumClaimPageSamples}`]
|
||||
: []),
|
||||
];
|
||||
|
||||
return {
|
||||
pageLoads: createEligibility({
|
||||
ineligibleReasons: pageReasons,
|
||||
kind: "comparative",
|
||||
limitations: [
|
||||
"samples share one container instance",
|
||||
"application ingress traverses a local TCP proxy sidecar excluded from memory samples",
|
||||
],
|
||||
requiredSampleCount: minimumClaimPageSamples,
|
||||
sampleCount: input.pageSampleCount,
|
||||
sampleUnit: "fresh authenticated Chromium document",
|
||||
scope: "fresh authenticated dashboard document load timings",
|
||||
}),
|
||||
largestContentfulPaint: createEligibility({
|
||||
ineligibleReasons: lcpReasons,
|
||||
kind: "comparative",
|
||||
limitations: [
|
||||
"samples share one container instance",
|
||||
"application ingress traverses a local TCP proxy sidecar excluded from memory samples",
|
||||
],
|
||||
requiredSampleCount: minimumClaimPageSamples,
|
||||
sampleCount: input.lcpSampleCount,
|
||||
sampleUnit: "fresh authenticated Chromium document",
|
||||
scope: `dashboard LCP observed for at least ${minimumClaimLcpObservationMs}ms and finalized by trusted input`,
|
||||
}),
|
||||
widgetDataSettled: createEligibility({
|
||||
ineligibleReasons: widgetDataReasons,
|
||||
kind: "comparative",
|
||||
limitations: [
|
||||
"samples share one container instance",
|
||||
"covers observed widget read-side tRPC requests; excludes subscriptions and widget.app.ping",
|
||||
"application ingress traverses a local TCP proxy sidecar excluded from memory samples",
|
||||
],
|
||||
requiredSampleCount: minimumClaimPageSamples,
|
||||
sampleCount: input.widgetDataSampleCount,
|
||||
sampleUnit: "fresh authenticated Chromium document",
|
||||
scope: `widget tRPC completion followed by a ${minimumClaimWidgetDataQuietMs}ms quiet window and two animation frames`,
|
||||
}),
|
||||
settleStability: createEligibility({
|
||||
ineligibleReasons: settleReasons,
|
||||
kind: "within-run",
|
||||
limitations: [
|
||||
"one container instance",
|
||||
"eligible only for within-run memory stability and trend, not cross-build RAM level claims",
|
||||
],
|
||||
requiredSampleCount: minimumClaimSettleSamples,
|
||||
sampleCount: input.settleSampleCount,
|
||||
sampleUnit: "time-series checkpoint",
|
||||
scope: `one warmed container observed for at least ${minimumClaimSettleMs}ms`,
|
||||
}),
|
||||
startup: createEligibility({
|
||||
ineligibleReasons: ["single container start is diagnostic only"],
|
||||
kind: "diagnostic",
|
||||
limitations: ["n=1 container start"],
|
||||
requiredSampleCount: null,
|
||||
sampleCount: 1,
|
||||
sampleUnit: "container start",
|
||||
scope: "start-to-ready timing",
|
||||
}),
|
||||
memoryLevels: createEligibility({
|
||||
ineligibleReasons: ["memory levels have one container observation per workload stage"],
|
||||
kind: "diagnostic",
|
||||
limitations: ["n=1 container per workload stage", "settle checkpoints are correlated time samples"],
|
||||
requiredSampleCount: null,
|
||||
sampleCount: 1,
|
||||
sampleUnit: "container instance",
|
||||
scope: "startup, workload, peak, and settled RAM levels",
|
||||
}),
|
||||
coldSpotlightInteraction: createEligibility({
|
||||
ineligibleReasons: ["cold Spotlight interaction has one observation"],
|
||||
kind: "diagnostic",
|
||||
limitations: ["n=1 cold interaction"],
|
||||
requiredSampleCount: null,
|
||||
sampleCount: 1,
|
||||
sampleUnit: "cold browser interaction",
|
||||
scope: "first feedback and full-ready Spotlight latency",
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export const getRuntimeClaimIneligibleReasons = (input: RuntimeMetricEligibilityInput) => {
|
||||
const eligibility = getRuntimeMetricEligibility(input);
|
||||
return [
|
||||
...new Set([
|
||||
...eligibility.pageLoads.ineligibleReasons,
|
||||
...eligibility.largestContentfulPaint.ineligibleReasons,
|
||||
...eligibility.widgetDataSettled.ineligibleReasons,
|
||||
...eligibility.settleStability.ineligibleReasons,
|
||||
]),
|
||||
];
|
||||
};
|
||||
|
||||
const readNumericLine = (input: string, key: string) => {
|
||||
const match = input.match(new RegExp(`^${key}[=:]\\s*(\\d+)`, "m"));
|
||||
return match ? Number(match[1]) : null;
|
||||
};
|
||||
|
||||
export const parseRuntimeMemorySnapshot = (name: string, output: string): RuntimeMemoryCheckpoint => {
|
||||
const currentBytes = readNumericLine(output, "memory_current");
|
||||
const anonymousBytes = readNumericLine(output, "cgroup_anon");
|
||||
const fileBytes = readNumericLine(output, "cgroup_file");
|
||||
if (currentBytes === null || anonymousBytes === null || fileBytes === null) {
|
||||
throw new Error(`Incomplete cgroup memory snapshot for ${name}`);
|
||||
}
|
||||
|
||||
const usageUsec = readNumericLine(output, "cpu_usage_usec");
|
||||
const userUsec = readNumericLine(output, "cpu_user_usec");
|
||||
const systemUsec = readNumericLine(output, "cpu_system_usec");
|
||||
const nrThrottled = readNumericLine(output, "cpu_nr_throttled");
|
||||
const throttledUsec = readNumericLine(output, "cpu_throttled_usec");
|
||||
if (
|
||||
usageUsec === null ||
|
||||
userUsec === null ||
|
||||
systemUsec === null ||
|
||||
nrThrottled === null ||
|
||||
throttledUsec === null
|
||||
) {
|
||||
throw new Error(`Incomplete cgroup CPU snapshot for ${name}`);
|
||||
}
|
||||
|
||||
const processes = output
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("process="))
|
||||
.flatMap((line) => {
|
||||
const [pid, parentPid, rssKiB, command, pssKiB] = line.slice("process=".length).split("|");
|
||||
if (!pid || !parentPid || !rssKiB || !command || !pssKiB) return [];
|
||||
return [
|
||||
{
|
||||
pid: Number(pid),
|
||||
parentPid: Number(parentPid),
|
||||
rssBytes: Number(rssKiB) * 1024,
|
||||
pssBytes: Number(pssKiB) * 1024,
|
||||
command,
|
||||
},
|
||||
];
|
||||
});
|
||||
const redisUsed = readNumericLine(output, "redis_used_memory");
|
||||
const redisPeak = readNumericLine(output, "redis_used_memory_peak");
|
||||
|
||||
return {
|
||||
name,
|
||||
capturedAt: new Date().toISOString(),
|
||||
container: {
|
||||
currentBytes,
|
||||
peakBytes: readNumericLine(output, "memory_peak"),
|
||||
anonymousBytes,
|
||||
fileBytes,
|
||||
},
|
||||
cpu: { usageUsec, userUsec, systemUsec, nrThrottled, throttledUsec },
|
||||
processes,
|
||||
redis: redisUsed === null || redisPeak === null ? null : { usedMemoryBytes: redisUsed, peakMemoryBytes: redisPeak },
|
||||
};
|
||||
};
|
||||
|
||||
const cpuDelta = (start: RuntimeMemoryCheckpoint | undefined, end: RuntimeMemoryCheckpoint | undefined) =>
|
||||
start && end
|
||||
? {
|
||||
usageUsec: end.cpu.usageUsec - start.cpu.usageUsec,
|
||||
userUsec: end.cpu.userUsec - start.cpu.userUsec,
|
||||
systemUsec: end.cpu.systemUsec - start.cpu.systemUsec,
|
||||
nrThrottled: end.cpu.nrThrottled - start.cpu.nrThrottled,
|
||||
throttledUsec: end.cpu.throttledUsec - start.cpu.throttledUsec,
|
||||
}
|
||||
: null;
|
||||
|
||||
const summarizeSamples = (values: number[]) => {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = values.toSorted((left, right) => left - right);
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
const median =
|
||||
sorted.length % 2 === 0 ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 : (sorted[middle] ?? 0);
|
||||
const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length;
|
||||
const variance = sorted.reduce((total, value) => total + (value - mean) ** 2, 0) / sorted.length;
|
||||
return {
|
||||
count: sorted.length,
|
||||
minBytes: sorted[0] ?? null,
|
||||
medianBytes: median,
|
||||
p95Bytes: sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)] ?? null,
|
||||
maxBytes: sorted.at(-1) ?? null,
|
||||
standardDeviationBytes: Math.round(Math.sqrt(variance)),
|
||||
};
|
||||
};
|
||||
|
||||
const getTrendBytesPerHour = (samples: Array<{ capturedAt: string; value: number }>) => {
|
||||
if (samples.length < 2) return null;
|
||||
const originMs = Date.parse(samples[0]?.capturedAt ?? "");
|
||||
const timedSamples = samples.flatMap(({ capturedAt, value }) => {
|
||||
const capturedAtMs = Date.parse(capturedAt);
|
||||
return Number.isFinite(originMs) && Number.isFinite(capturedAtMs)
|
||||
? [{ elapsedMs: capturedAtMs - originMs, value }]
|
||||
: [];
|
||||
});
|
||||
if (timedSamples.length < 2) return null;
|
||||
const xMean = timedSamples.reduce((total, { elapsedMs }) => total + elapsedMs, 0) / timedSamples.length;
|
||||
const yMean = timedSamples.reduce((total, { value }) => total + value, 0) / timedSamples.length;
|
||||
let numerator = 0;
|
||||
let denominator = 0;
|
||||
for (const { elapsedMs, value } of timedSamples) {
|
||||
numerator += (elapsedMs - xMean) * (value - yMean);
|
||||
denominator += (elapsedMs - xMean) ** 2;
|
||||
}
|
||||
return denominator === 0 ? null : (numerator / denominator) * 3_600_000;
|
||||
};
|
||||
|
||||
export const summarizeRuntimeMemory = (checkpoints: RuntimeMemoryCheckpoint[]) => {
|
||||
const startup = checkpoints.find(({ name }) => name === "startup");
|
||||
const workload = checkpoints.find(({ name }) => name === "workload");
|
||||
const settled = checkpoints.at(-1);
|
||||
const settleValues = checkpoints
|
||||
.filter(({ name }) => name.startsWith("settle-"))
|
||||
.map(({ container }) => container.currentBytes);
|
||||
const settleCurrentSamples = checkpoints
|
||||
.filter(({ name }) => name.startsWith("settle-"))
|
||||
.map(({ capturedAt, container }) => ({ capturedAt, value: container.currentBytes }));
|
||||
const settleAnonymousValues = checkpoints
|
||||
.filter(({ name }) => name.startsWith("settle-"))
|
||||
.map(({ container }) => container.anonymousBytes);
|
||||
const settleAnonymousSamples = checkpoints
|
||||
.filter(({ name }) => name.startsWith("settle-"))
|
||||
.map(({ capturedAt, container }) => ({ capturedAt, value: container.anonymousBytes }));
|
||||
const settleFileValues = checkpoints
|
||||
.filter(({ name }) => name.startsWith("settle-"))
|
||||
.map(({ container }) => container.fileBytes);
|
||||
const settleFileSamples = checkpoints
|
||||
.filter(({ name }) => name.startsWith("settle-"))
|
||||
.map(({ capturedAt, container }) => ({ capturedAt, value: container.fileBytes }));
|
||||
const settle = summarizeSamples(settleValues);
|
||||
const settleAnonymous = summarizeSamples(settleAnonymousValues);
|
||||
const settleFile = summarizeSamples(settleFileValues);
|
||||
const currentValues = checkpoints.map(({ container }) => container.currentBytes);
|
||||
const peakValues = checkpoints.flatMap(({ container }) =>
|
||||
container.peakBytes === null ? [] : [container.peakBytes],
|
||||
);
|
||||
const growthBytes = workload && settle ? settle.medianBytes - workload.container.currentBytes : null;
|
||||
return {
|
||||
startupBytes: startup?.container.currentBytes ?? null,
|
||||
workloadBytes: workload?.container.currentBytes ?? null,
|
||||
settledBytes: settled?.container.currentBytes ?? null,
|
||||
maximumSampleBytes: currentValues.length > 0 ? Math.max(...currentValues) : null,
|
||||
peakBytes: peakValues.length > 0 ? Math.max(...peakValues) : null,
|
||||
growthBytes,
|
||||
growthPercent:
|
||||
workload && growthBytes !== null && workload.container.currentBytes > 0
|
||||
? (growthBytes / workload.container.currentBytes) * 100
|
||||
: null,
|
||||
settle: settle
|
||||
? {
|
||||
...settle,
|
||||
trendBytesPerHour: getTrendBytesPerHour(settleCurrentSamples),
|
||||
anonymous: settleAnonymous
|
||||
? {
|
||||
...settleAnonymous,
|
||||
trendBytesPerHour: getTrendBytesPerHour(settleAnonymousSamples),
|
||||
}
|
||||
: null,
|
||||
file: settleFile ? { ...settleFile, trendBytesPerHour: getTrendBytesPerHour(settleFileSamples) } : null,
|
||||
}
|
||||
: null,
|
||||
cpu: {
|
||||
startupToWorkload: cpuDelta(startup, workload),
|
||||
workloadToSettled: cpuDelta(workload, settled),
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,427 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
getBrowserNetworkIsolationLaunchArgs,
|
||||
getSpotlightIdleMountFailure,
|
||||
getSpotlightIdleNetworkFailure,
|
||||
getTrpcRequestIdentities,
|
||||
getUnrecoveredWidgetDataRequests,
|
||||
getTrpcProcedures,
|
||||
getRuntimeClaimIneligibleReasons,
|
||||
getRuntimeMetricEligibility,
|
||||
hasBlockingWidgetDataAborts,
|
||||
normalizeDockerArchitecture,
|
||||
isRuntimeWidgetDataProcedure,
|
||||
isRuntimeWidgetDataRequest,
|
||||
inspectTrpcBatchStreamResponse,
|
||||
parseRuntimeMemorySnapshot,
|
||||
recordWidgetDataRequestOutcome,
|
||||
summarizeRuntimeMemory,
|
||||
} from "./docker-runtime-lib.mts";
|
||||
|
||||
const snapshot = (current: number, peak: number, cpuUsageUsec = 1_000) => `memory_current=${current}
|
||||
memory_peak=${peak}
|
||||
cgroup_anon=100
|
||||
cgroup_file=20
|
||||
cpu_usage_usec=${cpuUsageUsec}
|
||||
cpu_user_usec=${cpuUsageUsec - 300}
|
||||
cpu_system_usec=300
|
||||
cpu_nr_throttled=2
|
||||
cpu_throttled_usec=50
|
||||
process=42|1|200|next-server|180
|
||||
process=7|1|10|redis-server|8
|
||||
redis_used_memory=30
|
||||
redis_used_memory_peak=40`;
|
||||
|
||||
const trpcRequestUrl = (input: string) =>
|
||||
`https://homarr.test/api/trpc/widget.weather.atLocation,app.byIds?batch=1&input=${input}`;
|
||||
|
||||
describe("Docker runtime memory parsing", () => {
|
||||
it("classifies every procedure in batched tRPC requests", () => {
|
||||
expect(getTrpcProcedures("https://homarr.test/api/trpc/widget.foo,queryCache.setItem?batch=1")).toEqual([
|
||||
"widget.foo",
|
||||
"queryCache.setItem",
|
||||
]);
|
||||
expect(getTrpcProcedures("https://homarr.test/api/trpc/widget.foo%2Csearch.query?batch=1")).toEqual([
|
||||
"widget.foo",
|
||||
"search.query",
|
||||
]);
|
||||
expect(getTrpcProcedures("https://homarr.test/api/health/ready")).toEqual([]);
|
||||
expect(isRuntimeWidgetDataProcedure("widget.calendar.findAllEvents")).toBe(true);
|
||||
expect(isRuntimeWidgetDataProcedure("app.byIds")).toBe(true);
|
||||
expect(isRuntimeWidgetDataProcedure("integration.byIds")).toBe(true);
|
||||
expect(isRuntimeWidgetDataProcedure("docker.getContainers")).toBe(true);
|
||||
expect(isRuntimeWidgetDataProcedure("widget.beszel.getSystemStats")).toBe(true);
|
||||
expect(isRuntimeWidgetDataProcedure("widget.beszel.subscribeSystemStats")).toBe(false);
|
||||
expect(isRuntimeWidgetDataProcedure("widget.app.ping")).toBe(false);
|
||||
expect(isRuntimeWidgetDataProcedure("board.getBoardByName")).toBe(false);
|
||||
expect(isRuntimeWidgetDataRequest("GET", "widget.weather.atLocation")).toBe(true);
|
||||
expect(isRuntimeWidgetDataRequest("POST", "widget.options.saveItemOptions")).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes Node's x64 architecture to Docker's amd64 name", () => {
|
||||
expect(normalizeDockerArchitecture("x64")).toBe("amd64");
|
||||
expect(normalizeDockerArchitecture("arm64")).toBe("arm64");
|
||||
});
|
||||
|
||||
it("keys batched tRPC operations by procedure and canonical serialized input", () => {
|
||||
const first = encodeURIComponent(
|
||||
JSON.stringify({
|
||||
0: { json: { location: "Paris", units: "metric" } },
|
||||
1: { json: { ids: ["app-1"] } },
|
||||
}),
|
||||
);
|
||||
const reordered = encodeURIComponent(
|
||||
JSON.stringify({
|
||||
0: { json: { units: "metric", location: "Paris" } },
|
||||
1: { json: { ids: ["app-1"] } },
|
||||
}),
|
||||
);
|
||||
const different = encodeURIComponent(
|
||||
JSON.stringify({
|
||||
0: { json: { location: "Lyon", units: "metric" } },
|
||||
1: { json: { ids: ["app-1"] } },
|
||||
}),
|
||||
);
|
||||
const identities = getTrpcRequestIdentities(trpcRequestUrl(first));
|
||||
expect(identities.map(({ procedure }) => procedure)).toEqual(["widget.weather.atLocation", "app.byIds"]);
|
||||
expect(identities.map(({ batchIndex }) => batchIndex)).toEqual([0, 1]);
|
||||
expect(identities).toEqual(getTrpcRequestIdentities(trpcRequestUrl(reordered)));
|
||||
expect(getTrpcRequestIdentities(trpcRequestUrl(different))[0]?.identity).not.toBe(identities[0]?.identity);
|
||||
expect(getTrpcRequestIdentities(trpcRequestUrl(different))[1]?.identity).toBe(identities[1]?.identity);
|
||||
expect(identities.every(({ identity }) => /^[^.]+(?:\.[^#]+)+#[a-f0-9]{64}$/.test(identity))).toBe(true);
|
||||
expect(JSON.stringify(identities)).not.toContain("Paris");
|
||||
expect(() =>
|
||||
getTrpcRequestIdentities("https://homarr.test/api/trpc/widget.weather.atLocation?batch=1&input=%7Bmalformed"),
|
||||
).toThrow("Malformed serialized tRPC request input");
|
||||
});
|
||||
|
||||
it("requires every aborted widget-data request identity to be recovered by an exact successful request", () => {
|
||||
const recoveryBalance = new Map<string, number>();
|
||||
recordWidgetDataRequestOutcome(recoveryBalance, ["weather#paris", "app#1"], "aborted");
|
||||
recordWidgetDataRequestOutcome(recoveryBalance, ["app#1", "weather#lyon"], "completed");
|
||||
|
||||
expect(getUnrecoveredWidgetDataRequests(recoveryBalance)).toEqual(["weather#paris"]);
|
||||
|
||||
recordWidgetDataRequestOutcome(recoveryBalance, ["weather#paris"], "completed");
|
||||
expect(getUnrecoveredWidgetDataRequests(recoveryBalance)).toEqual([]);
|
||||
});
|
||||
|
||||
it("credits an earlier successful response when a superseding refetch is aborted", () => {
|
||||
const recoveryBalance = new Map<string, number>();
|
||||
recordWidgetDataRequestOutcome(recoveryBalance, ["widget.weather.atLocation"], "completed");
|
||||
recordWidgetDataRequestOutcome(recoveryBalance, ["widget.weather.atLocation"], "aborted");
|
||||
|
||||
expect(getUnrecoveredWidgetDataRequests(recoveryBalance)).toEqual([]);
|
||||
});
|
||||
|
||||
it("allows unrecovered aborts only for the non-claim login transition", () => {
|
||||
expect(hasBlockingWidgetDataAborts(true, 2)).toBe(false);
|
||||
expect(hasBlockingWidgetDataAborts(false, 0)).toBe(false);
|
||||
expect(hasBlockingWidgetDataAborts(false, 1)).toBe(true);
|
||||
});
|
||||
|
||||
it("allows eager baseline Spotlight work but keeps preload-only preparation network-free", () => {
|
||||
expect(getSpotlightIdleNetworkFailure("mounted", 1)).toBeNull();
|
||||
expect(getSpotlightIdleNetworkFailure("preload-only", 0)).toBeNull();
|
||||
expect(getSpotlightIdleNetworkFailure("preload-only", 1)).toBe(
|
||||
"Idle Spotlight preload-only preparation made 1 non-widget tRPC requests",
|
||||
);
|
||||
expect(getSpotlightIdleMountFailure("mounted", 1)).toBeNull();
|
||||
expect(getSpotlightIdleMountFailure("preload-only", 0)).toBeNull();
|
||||
expect(getSpotlightIdleMountFailure("preload-only", 1)).toBe(
|
||||
"Idle Spotlight preload mounted the UI before user intent",
|
||||
);
|
||||
});
|
||||
|
||||
it("fails closed on HTTP, malformed, missing, and per-operation streamed tRPC errors", async () => {
|
||||
const errorShape = {
|
||||
code: -32603,
|
||||
data: { code: "INTERNAL_SERVER_ERROR", httpStatus: 500 },
|
||||
message: "boom",
|
||||
};
|
||||
const head = `${JSON.stringify({ json: { 0: [[0], [null, 0, 0]] } })}\n`;
|
||||
const errorBody = `${head}${JSON.stringify({ json: [0, 0, [[{ error: errorShape }]]] })}\n`;
|
||||
const rejectedBody = `${head}${JSON.stringify({ json: [0, 1, errorShape] })}\n`;
|
||||
const rejectedDataBody = [
|
||||
head.trimEnd(),
|
||||
JSON.stringify({ json: [0, 0, [[{ result: 0 }], ["result", 0, 1]]] }),
|
||||
JSON.stringify({ json: [1, 0, [[{ data: 0 }], ["data", 0, 2]]] }),
|
||||
JSON.stringify({ json: [2, 1, errorShape] }),
|
||||
"",
|
||||
].join("\n");
|
||||
const successfulUserDataBody = [
|
||||
head.trimEnd(),
|
||||
JSON.stringify({ json: [0, 0, [[{ result: 0 }], ["result", 0, 1]]] }),
|
||||
JSON.stringify({ json: [1, 0, [[{ data: 0 }], ["data", 0, 2]]] }),
|
||||
JSON.stringify({ json: [2, 0, [[{ error: errorShape }]]] }),
|
||||
"",
|
||||
].join("\n");
|
||||
const mixedBody = [
|
||||
JSON.stringify({ json: { 0: [[0], [null, 0, 0]], 1: [[0], [null, 0, 1]] } }),
|
||||
JSON.stringify({ json: [0, 0, [[{ result: 0 }], ["result", 0, 2]]] }),
|
||||
JSON.stringify({ json: [1, 0, [[{ error: errorShape }]]] }),
|
||||
JSON.stringify({ json: [2, 0, [[{ data: "ok" }]]] }),
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
await expect(inspectTrpcBatchStreamResponse(500, errorBody, 2)).resolves.toEqual({
|
||||
failures: [
|
||||
{ batchIndex: 0, kind: "http-status", status: 500 },
|
||||
{ batchIndex: 1, kind: "http-status", status: 500 },
|
||||
],
|
||||
successfulIndexes: [],
|
||||
});
|
||||
for (const body of ["", "not-json\n", "{}\n", head.trimEnd()]) {
|
||||
await expect(inspectTrpcBatchStreamResponse(200, body, 1)).resolves.toEqual({
|
||||
failures: [{ batchIndex: 0, kind: "invalid-jsonl" }],
|
||||
successfulIndexes: [],
|
||||
});
|
||||
}
|
||||
await expect(inspectTrpcBatchStreamResponse(200, head, 2)).resolves.toEqual({
|
||||
failures: [
|
||||
{ batchIndex: 0, kind: "index-mismatch" },
|
||||
{ batchIndex: 1, kind: "index-mismatch" },
|
||||
],
|
||||
successfulIndexes: [],
|
||||
});
|
||||
await expect(inspectTrpcBatchStreamResponse(200, errorBody, 1)).resolves.toEqual({
|
||||
failures: [
|
||||
{
|
||||
batchIndex: 0,
|
||||
httpStatus: 500,
|
||||
jsonRpcCode: -32603,
|
||||
kind: "trpc-error",
|
||||
trpcCode: "INTERNAL_SERVER_ERROR",
|
||||
},
|
||||
],
|
||||
successfulIndexes: [],
|
||||
});
|
||||
for (const body of [rejectedBody, rejectedDataBody]) {
|
||||
await expect(inspectTrpcBatchStreamResponse(200, body, 1)).resolves.toEqual({
|
||||
failures: [{ batchIndex: 0, kind: "stream-error" }],
|
||||
successfulIndexes: [],
|
||||
});
|
||||
}
|
||||
await expect(inspectTrpcBatchStreamResponse(200, successfulUserDataBody, 1)).resolves.toEqual({
|
||||
failures: [],
|
||||
successfulIndexes: [0],
|
||||
});
|
||||
await expect(inspectTrpcBatchStreamResponse(200, mixedBody, 2)).resolves.toEqual({
|
||||
failures: [
|
||||
{
|
||||
batchIndex: 1,
|
||||
httpStatus: 500,
|
||||
jsonRpcCode: -32603,
|
||||
kind: "trpc-error",
|
||||
trpcCode: "INTERNAL_SERVER_ERROR",
|
||||
},
|
||||
],
|
||||
successfulIndexes: [0],
|
||||
});
|
||||
});
|
||||
|
||||
it("isolates external browser traffic without disabling Chromium's HTTP cache", () => {
|
||||
expect(getBrowserNetworkIsolationLaunchArgs("http://127.0.0.1:43123")).toEqual([
|
||||
"--proxy-server=http://127.0.0.1:43123",
|
||||
"--proxy-bypass-list=127.0.0.1;localhost",
|
||||
"--host-resolver-rules=MAP * ~NOTFOUND, EXCLUDE 127.0.0.1, EXCLUDE localhost",
|
||||
]);
|
||||
expect(getBrowserNetworkIsolationLaunchArgs("http://127.0.0.1:43123").join(" ")).not.toContain("disable-cache");
|
||||
});
|
||||
|
||||
it("only permits claims with native, SHA-pinned images and enough settle samples", () => {
|
||||
expect(
|
||||
getRuntimeClaimIneligibleReasons({
|
||||
expectedSha: "abc123",
|
||||
expectedSourceFingerprint: "source-hash",
|
||||
browserNetworkIsolated: true,
|
||||
serverNetworkIsolated: true,
|
||||
imageArchitecture: "amd64",
|
||||
lcpObservationMs: 5_000,
|
||||
lcpSampleCount: 20,
|
||||
nodeArchitecture: "x64",
|
||||
pageSampleCount: 20,
|
||||
requireImplementationMarkers: true,
|
||||
revision: "abc123",
|
||||
sourceFingerprint: "source-hash",
|
||||
settleMs: 10 * 60 * 1_000,
|
||||
settleSampleCount: 20,
|
||||
widgetDataQuietMs: 500,
|
||||
widgetDataSampleCount: 20,
|
||||
}),
|
||||
).toEqual([]);
|
||||
|
||||
expect(
|
||||
getRuntimeClaimIneligibleReasons({
|
||||
expectedSha: "abc123",
|
||||
expectedSourceFingerprint: "source-hash",
|
||||
browserNetworkIsolated: false,
|
||||
serverNetworkIsolated: false,
|
||||
imageArchitecture: "arm64",
|
||||
lcpObservationMs: 1_000,
|
||||
lcpSampleCount: 1,
|
||||
nodeArchitecture: "x64",
|
||||
pageSampleCount: 1,
|
||||
requireImplementationMarkers: false,
|
||||
revision: null,
|
||||
sourceFingerprint: null,
|
||||
settleMs: 0,
|
||||
settleSampleCount: 0,
|
||||
widgetDataQuietMs: 100,
|
||||
widgetDataSampleCount: 1,
|
||||
}),
|
||||
).toEqual([
|
||||
"image architecture arm64 is not host amd64",
|
||||
"image revision label is missing",
|
||||
"image source fingerprint label is missing",
|
||||
"browser external network isolation is not active",
|
||||
"server external network isolation is not active",
|
||||
"widget implementation markers are not required",
|
||||
"page sample count 1 is below 20",
|
||||
"LCP observation 1000ms is below 5000ms",
|
||||
"LCP sample count 1 is below 20",
|
||||
"widget data quiet window 100ms is below 500ms",
|
||||
"widget data sample count 1 is below 20",
|
||||
"settle duration 0ms is below 600000ms",
|
||||
"settle sample count 0 is below 20",
|
||||
]);
|
||||
});
|
||||
|
||||
it("scopes repeatable, within-run, and single-sample metrics separately", () => {
|
||||
const eligibility = getRuntimeMetricEligibility({
|
||||
expectedSha: "abc123",
|
||||
expectedSourceFingerprint: "source-hash",
|
||||
browserNetworkIsolated: true,
|
||||
serverNetworkIsolated: true,
|
||||
imageArchitecture: "amd64",
|
||||
lcpObservationMs: 5_000,
|
||||
lcpSampleCount: 20,
|
||||
nodeArchitecture: "x64",
|
||||
pageSampleCount: 20,
|
||||
requireImplementationMarkers: true,
|
||||
revision: "abc123",
|
||||
sourceFingerprint: "source-hash",
|
||||
settleMs: 10 * 60 * 1_000,
|
||||
settleSampleCount: 20,
|
||||
widgetDataQuietMs: 500,
|
||||
widgetDataSampleCount: 20,
|
||||
});
|
||||
|
||||
expect(eligibility.pageLoads).toMatchObject({
|
||||
claimEligible: true,
|
||||
kind: "comparative",
|
||||
requiredSampleCount: 20,
|
||||
sampleCount: 20,
|
||||
});
|
||||
expect(eligibility.largestContentfulPaint).toMatchObject({
|
||||
claimEligible: true,
|
||||
kind: "comparative",
|
||||
requiredSampleCount: 20,
|
||||
sampleCount: 20,
|
||||
});
|
||||
expect(eligibility.widgetDataSettled).toMatchObject({
|
||||
claimEligible: true,
|
||||
kind: "comparative",
|
||||
requiredSampleCount: 20,
|
||||
sampleCount: 20,
|
||||
});
|
||||
expect(eligibility.settleStability).toMatchObject({
|
||||
claimEligible: true,
|
||||
kind: "within-run",
|
||||
requiredSampleCount: 20,
|
||||
sampleCount: 20,
|
||||
});
|
||||
expect(eligibility.settleStability.limitations).toContain("one container instance");
|
||||
expect(eligibility.startup).toMatchObject({ claimEligible: false, kind: "diagnostic", sampleCount: 1 });
|
||||
expect(eligibility.memoryLevels).toMatchObject({ claimEligible: false, kind: "diagnostic", sampleCount: 1 });
|
||||
expect(eligibility.coldSpotlightInteraction).toMatchObject({
|
||||
claimEligible: false,
|
||||
kind: "diagnostic",
|
||||
sampleCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes cgroup, process, and Redis values", () => {
|
||||
expect(parseRuntimeMemorySnapshot("startup", snapshot(1_000, 2_000))).toMatchObject({
|
||||
name: "startup",
|
||||
container: { currentBytes: 1_000, peakBytes: 2_000, anonymousBytes: 100, fileBytes: 20 },
|
||||
cpu: { usageUsec: 1_000, userUsec: 700, systemUsec: 300, nrThrottled: 2, throttledUsec: 50 },
|
||||
processes: [
|
||||
{ pid: 42, parentPid: 1, rssBytes: 200 * 1024, pssBytes: 180 * 1024, command: "next-server" },
|
||||
{ pid: 7, parentPid: 1, rssBytes: 10 * 1024, pssBytes: 8 * 1024, command: "redis-server" },
|
||||
],
|
||||
redis: { usedMemoryBytes: 30, peakMemoryBytes: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a failed Redis collection absent instead of reporting zero bytes", () => {
|
||||
const withoutRedis = snapshot(1_000, 2_000)
|
||||
.split("\n")
|
||||
.filter((line) => !line.startsWith("redis_"))
|
||||
.join("\n");
|
||||
|
||||
expect(parseRuntimeMemorySnapshot("startup", withoutRedis).redis).toBeNull();
|
||||
});
|
||||
|
||||
it("reports workload-to-settled growth and the largest sample", () => {
|
||||
const checkpoints = [
|
||||
parseRuntimeMemorySnapshot("startup", snapshot(1_000, 2_000)),
|
||||
parseRuntimeMemorySnapshot("workload", snapshot(1_100, 2_500, 1_500)),
|
||||
parseRuntimeMemorySnapshot("settle-1", snapshot(1_200, 2_500, 1_800)),
|
||||
];
|
||||
checkpoints.forEach((checkpoint, index) => {
|
||||
checkpoint.capturedAt = new Date(Date.UTC(2026, 0, 1, 0, index)).toISOString();
|
||||
});
|
||||
|
||||
expect(summarizeRuntimeMemory(checkpoints)).toEqual({
|
||||
startupBytes: 1_000,
|
||||
workloadBytes: 1_100,
|
||||
settledBytes: 1_200,
|
||||
maximumSampleBytes: 1_200,
|
||||
peakBytes: 2_500,
|
||||
growthBytes: 100,
|
||||
growthPercent: 100 / 11,
|
||||
settle: {
|
||||
count: 1,
|
||||
minBytes: 1_200,
|
||||
medianBytes: 1_200,
|
||||
p95Bytes: 1_200,
|
||||
maxBytes: 1_200,
|
||||
standardDeviationBytes: 0,
|
||||
trendBytesPerHour: null,
|
||||
anonymous: {
|
||||
count: 1,
|
||||
minBytes: 100,
|
||||
medianBytes: 100,
|
||||
p95Bytes: 100,
|
||||
maxBytes: 100,
|
||||
standardDeviationBytes: 0,
|
||||
trendBytesPerHour: null,
|
||||
},
|
||||
file: {
|
||||
count: 1,
|
||||
minBytes: 20,
|
||||
medianBytes: 20,
|
||||
p95Bytes: 20,
|
||||
maxBytes: 20,
|
||||
standardDeviationBytes: 0,
|
||||
trendBytesPerHour: null,
|
||||
},
|
||||
},
|
||||
cpu: {
|
||||
startupToWorkload: { usageUsec: 500, userUsec: 500, systemUsec: 0, nrThrottled: 0, throttledUsec: 0 },
|
||||
workloadToSettled: { usageUsec: 300, userUsec: 300, systemUsec: 0, nrThrottled: 0, throttledUsec: 0 },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("derives memory trends from observed checkpoint timestamps", () => {
|
||||
const first = parseRuntimeMemorySnapshot("settle-1", snapshot(1_000, 2_000));
|
||||
const second = parseRuntimeMemorySnapshot("settle-2", snapshot(1_100, 2_000));
|
||||
first.capturedAt = "2026-01-01T00:00:00.000Z";
|
||||
second.capturedAt = "2026-01-01T02:00:00.000Z";
|
||||
|
||||
expect(summarizeRuntimeMemory([first, second]).settle?.trendBytesPerHour).toBe(50);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,107 @@
|
||||
import { createIntegrationRequestHandler } from "../../packages/request-handler/src/lib/integration-request-handler.ts";
|
||||
import { createRequestHandler } from "../../packages/request-handler/src/lib/request-handler.ts";
|
||||
|
||||
const allocations = Number(process.env.REQUEST_MEMORY_ALLOCATIONS ?? 64);
|
||||
const allocationBytes = Number(process.env.REQUEST_MEMORY_ALLOCATION_BYTES ?? 1024 * 1024);
|
||||
const maxRetainedBytes = Number(process.env.REQUEST_MEMORY_MAX_RETAINED_BYTES ?? 16 * 1024 * 1024);
|
||||
const positiveTtlMs = Number(process.env.REQUEST_MEMORY_POSITIVE_TTL_MS ?? 25);
|
||||
|
||||
if (!globalThis.gc) throw new Error("Run this benchmark with node --expose-gc");
|
||||
if (!Number.isInteger(allocations) || allocations < 2) throw new Error("REQUEST_MEMORY_ALLOCATIONS must be >= 2");
|
||||
if (!Number.isInteger(allocationBytes) || allocationBytes < 1) {
|
||||
throw new Error("REQUEST_MEMORY_ALLOCATION_BYTES must be positive");
|
||||
}
|
||||
|
||||
const collectGarbage = async () => {
|
||||
for (let index = 0; index < 4; index++) {
|
||||
globalThis.gc?.();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
};
|
||||
|
||||
const retainedRoots: unknown[] = [];
|
||||
const measureArrayBufferGrowthAsync = async (workload: () => Promise<unknown>) => {
|
||||
await collectGarbage();
|
||||
const before = process.memoryUsage().arrayBuffers;
|
||||
const retainedRoot = await workload();
|
||||
retainedRoots.push(retainedRoot);
|
||||
await collectGarbage();
|
||||
const growth = process.memoryUsage().arrayBuffers - before;
|
||||
retainedRoots.pop();
|
||||
return growth;
|
||||
};
|
||||
|
||||
const zeroTtlGrowthBytes = await measureArrayBufferGrowthAsync(async () => {
|
||||
const handler = createRequestHandler<Uint8Array, { id: number }>({
|
||||
cacheTtlMs: 0,
|
||||
requestAsync: async () => new Uint8Array(allocationBytes),
|
||||
});
|
||||
for (let id = 0; id < allocations; id++) {
|
||||
await handler.handler({ id }).getDataAsync();
|
||||
}
|
||||
return handler;
|
||||
});
|
||||
|
||||
const integrationGrowthBytes = await measureArrayBufferGrowthAsync(async () => {
|
||||
const handler = createIntegrationRequestHandler<number, "mock", Record<string, never>>({
|
||||
cacheTtlMs: 0,
|
||||
requestAsync: async (integration) =>
|
||||
(integration as typeof integration & { benchmarkPayload: Uint8Array }).benchmarkPayload.byteLength,
|
||||
});
|
||||
for (let id = 0; id < allocations; id++) {
|
||||
await handler
|
||||
.handler(
|
||||
{
|
||||
id: `benchmark-${id}`,
|
||||
kind: "mock",
|
||||
name: "benchmark",
|
||||
url: "http://benchmark.invalid",
|
||||
externalUrl: null,
|
||||
decryptedSecrets: [],
|
||||
benchmarkPayload: new Uint8Array(allocationBytes),
|
||||
} as never,
|
||||
{},
|
||||
)
|
||||
.getDataAsync();
|
||||
}
|
||||
return handler;
|
||||
});
|
||||
|
||||
const expiredPositiveTtlGrowthBytes = await measureArrayBufferGrowthAsync(async () => {
|
||||
const handler = createRequestHandler<Uint8Array, { id: number }>({
|
||||
cacheTtlMs: positiveTtlMs,
|
||||
requestAsync: async () => new Uint8Array(allocationBytes),
|
||||
});
|
||||
for (let id = 0; id < allocations; id++) {
|
||||
await handler.handler({ id }).getDataAsync();
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, positiveTtlMs + 25));
|
||||
return handler;
|
||||
});
|
||||
|
||||
const positiveControlGrowthBytes = await measureArrayBufferGrowthAsync(async () =>
|
||||
Array.from({ length: allocations }, () => new Uint8Array(allocationBytes)),
|
||||
);
|
||||
const positiveControlMinimumBytes = allocations * allocationBytes * 0.8;
|
||||
|
||||
const result = {
|
||||
allocations,
|
||||
allocationBytes,
|
||||
allocatedPerScenarioBytes: allocations * allocationBytes,
|
||||
maxRetainedBytes,
|
||||
positiveTtlMs,
|
||||
zeroTtlGrowthBytes,
|
||||
integrationGrowthBytes,
|
||||
expiredPositiveTtlGrowthBytes,
|
||||
positiveControlGrowthBytes,
|
||||
positiveControlMinimumBytes,
|
||||
zeroTtlPass: zeroTtlGrowthBytes <= maxRetainedBytes,
|
||||
integrationPass: integrationGrowthBytes <= maxRetainedBytes,
|
||||
expiredPositiveTtlPass: expiredPositiveTtlGrowthBytes <= maxRetainedBytes,
|
||||
positiveControlPass: positiveControlGrowthBytes >= positiveControlMinimumBytes,
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (!result.zeroTtlPass || !result.integrationPass || !result.expiredPositiveTtlPass || !result.positiveControlPass) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in new issue
Block a user