test(performance): add authenticated local benchmarks

This commit is contained in:
ajnart committed 2026-07-29 06:47:07 +02:00
1 parent f8d8a09c56
commit 440626ba00
8 files changed
+389 -2

No files matched your search

+67
View File
@@ -0,0 +1,67 @@
export type NumericSummary = {
count: number;
medianMs: number | null;
p95Ms: number | null;
standardDeviationMs: number | null;
outliersMs: number[];
};
export const normalizeTrpcPath = (requestUrl: string) => {
try {
const url = new URL(requestUrl, "http://benchmark.invalid");
url.searchParams.delete("input");
return url.pathname + url.search;
} catch {
return requestUrl;
}
};
export const countDuplicateTrpcPaths = (requestUrls: string[]) => {
const paths = requestUrls.map(normalizeTrpcPath);
return Math.max(0, paths.length - new Set(paths).size);
};
const values = (numbers: Array<number | null | undefined>) =>
numbers.filter((value): value is number => typeof value === "number" && value >= 0).toSorted((a, b) => a - b);
export const summarize = (numbers: Array<number | null | undefined>): NumericSummary => {
const sorted = values(numbers);
if (sorted.length === 0) {
return { count: 0, medianMs: null, p95Ms: null, standardDeviationMs: null, outliersMs: [] };
}
const medianIndex = Math.floor(sorted.length / 2);
const p95Index = Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1);
const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length;
const variance = sorted.reduce((total, value) => total + (value - mean) ** 2, 0) / sorted.length;
const firstQuartile = sorted[Math.floor((sorted.length - 1) * 0.25)] ?? 0;
const thirdQuartile = sorted[Math.floor((sorted.length - 1) * 0.75)] ?? 0;
const iqr = thirdQuartile - firstQuartile;
const outlierLimit = thirdQuartile + Math.max(1, iqr * 1.5);
return {
count: sorted.length,
medianMs: sorted[medianIndex] ?? 0,
p95Ms: sorted[p95Index] ?? 0,
standardDeviationMs: Math.round(Math.sqrt(variance)),
outliersMs: sorted.filter((value) => value > outlierLimit),
};
};
export const validateBrowserMeasurement = (measurement: {
responseStatus: number | null;
domContentLoadedMs: number;
boardHydratedMs: number | null;
firstInteractionMs: number | null;
error?: string;
}) => {
const errors: string[] = [];
if (measurement.error) errors.push(measurement.error);
if (measurement.responseStatus === null) errors.push("browser navigation returned no response");
else if (measurement.responseStatus < 200 || measurement.responseStatus >= 300)
errors.push(`browser navigation returned HTTP ${measurement.responseStatus}`);
if (measurement.domContentLoadedMs < 0) errors.push("DOMContentLoaded timing is missing");
if (measurement.boardHydratedMs === null) errors.push("board hydration marker is missing");
if (measurement.firstInteractionMs === null) errors.push("first interaction timing is missing");
return errors;
};
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import {
countDuplicateTrpcPaths,
normalizeTrpcPath,
summarize,
validateBrowserMeasurement,
} from "./dev-benchmark-lib.mts";
describe("development benchmark measurements", () => {
it("normalizes tRPC input payloads before counting duplicates", () => {
const first = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A1%7D";
const second = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A2%7D";
expect(normalizeTrpcPath(first)).toBe("/api/trpc/board.get?batch=1");
expect(countDuplicateTrpcPaths([first, second, "/api/trpc/user.get"])).toBe(1);
});
it("reports median, p95, deviation, and high outliers", () => {
expect(summarize([1, 2, 3, 4, 5, 6, 100])).toEqual({
count: 7,
medianMs: 4,
p95Ms: 100,
standardDeviationMs: 34,
outliersMs: [100],
});
});
it("requires a successful authenticated board hydration and interaction", () => {
expect(
validateBrowserMeasurement({
responseStatus: 307,
domContentLoadedMs: 120,
boardHydratedMs: null,
firstInteractionMs: null,
}),
).toEqual([
"browser navigation returned HTTP 307",
"board hydration marker is missing",
"first interaction timing is missing",
]);
});
});
+159
View File
@@ -0,0 +1,159 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
import type { Page } from "@playwright/test";
import { chromium } from "@playwright/test";
import { countDuplicateTrpcPaths, summarize, validateBrowserMeasurement } from "./dev-benchmark-lib.mts";
type BoardMeasurement = {
boardReadyMs: number;
domContentLoadedMs: number;
firstInteractionMs: number;
pageErrors: string[];
responseStatus: number | null;
trpcDuplicateRequestCount: number;
trpcRequestCount: number;
};
const root = path.resolve(import.meta.dirname, "../..");
const argumentsSet = new Set(process.argv.slice(2));
const smokeOnly = argumentsSet.has("--smoke");
const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 7);
const outputDirectory = path.resolve(
process.env.DEV_BENCHMARK_OUTPUT_DIR ?? path.join(root, "benchmark-results", "dev"),
);
const boardUrl = process.env.DEV_BENCHMARK_BOARD_URL;
const storageStatePath = process.env.DEV_BENCHMARK_STORAGE_STATE;
const readinessUrl = process.env.DEV_BENCHMARK_READINESS_URL ?? "http://127.0.0.1:3000/api/health/ready";
const boardSelector = "[data-homarr-dev-benchmark-board]";
const interactionSelector = '[data-homarr-dev-benchmark-interaction="search"]';
const spotlightSelector = "[data-homarr-dev-benchmark-spotlight]";
if (!Number.isInteger(iterations) || iterations < 7) {
throw new Error("DEV_BENCHMARK_ITERATIONS must be an integer of at least 7");
}
await fs.mkdir(outputDirectory, { recursive: true });
if (smokeOnly) {
const measurements: number[] = [];
for (let iteration = 0; iteration < iterations; iteration++) {
const startedAt = performance.now();
const response = await fetch(readinessUrl);
await response.arrayBuffer();
if (!response.ok) throw new Error(`Readiness smoke received HTTP ${response.status}`);
measurements.push(Math.round(performance.now() - startedAt));
}
const result = {
kind: "readiness-smoke",
claimEligible: false,
url: readinessUrl,
iterations,
summary: summarize(measurements),
};
const outputPath = path.join(outputDirectory, `smoke-${Date.now()}.json`);
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
console.log(JSON.stringify({ outputPath, result }, null, 2));
process.exit(0);
}
if (!boardUrl || !storageStatePath) {
throw new Error(
"Authenticated claim mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " +
"Use --smoke only for the non-claimable readiness check.",
);
}
await fs.access(storageStatePath);
const browser = await chromium.launch();
const cold: BoardMeasurement[] = [];
const warm: BoardMeasurement[] = [];
const measureBoardAsync = async (page: Page) => {
const trpcRequests: string[] = [];
const pageErrors: string[] = [];
page.on("request", (request) => {
if (request.url().includes("/api/trpc/")) trpcRequests.push(request.url());
});
page.on("pageerror", (error) => pageErrors.push(error.message));
const startedAt = performance.now();
const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
const domContentLoadedMs = Math.round(performance.now() - startedAt);
await page.locator(boardSelector).waitFor({ state: "visible" });
const boardReadyMs = Math.round(performance.now() - startedAt);
const interactionStartedAt = performance.now();
await page.locator(interactionSelector).first().click();
await page.locator(spotlightSelector).waitFor({ state: "visible" });
const firstInteractionMs = Math.round(performance.now() - interactionStartedAt);
const measurement: BoardMeasurement = {
responseStatus: response?.status() ?? null,
domContentLoadedMs,
boardReadyMs,
firstInteractionMs,
trpcRequestCount: trpcRequests.length,
trpcDuplicateRequestCount: countDuplicateTrpcPaths(trpcRequests),
pageErrors,
};
const errors = validateBrowserMeasurement({
responseStatus: measurement.responseStatus,
domContentLoadedMs,
boardHydratedMs: boardReadyMs,
firstInteractionMs,
error: pageErrors.length > 0 ? pageErrors.join("; ") : undefined,
});
if (errors.length > 0) throw new Error(errors.map((error) => `${boardUrl}: ${error}`).join("; "));
return measurement;
};
try {
for (let iteration = 0; iteration < iterations; iteration++) {
const context = await browser.newContext({ storageState: storageStatePath });
const coldPage = await context.newPage();
cold.push(await measureBoardAsync(coldPage));
await coldPage.close();
const warmPage = await context.newPage();
warm.push(await measureBoardAsync(warmPage));
await context.close();
}
} finally {
await browser.close();
}
const summarizeMeasurements = (measurements: BoardMeasurement[]) => ({
boardReady: summarize(measurements.map(({ boardReadyMs }) => boardReadyMs)),
domContentLoaded: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)),
firstInteraction: summarize(measurements.map(({ firstInteractionMs }) => firstInteractionMs)),
trpcRequests: summarize(measurements.map(({ trpcRequestCount }) => trpcRequestCount)),
duplicateTrpcRequests: summarize(measurements.map(({ trpcDuplicateRequestCount }) => trpcDuplicateRequestCount)),
});
const result = {
kind: "authenticated-board",
claimEligible: true,
boardUrl,
createdAt: new Date().toISOString(),
environment: {
architecture: process.arch,
cpus: os.cpus().map(({ model }) => model),
node: process.version,
platform: `${process.platform} ${os.release()}`,
sha: process.env.DEV_BENCHMARK_SHA ?? null,
},
iterations,
measurements: { cold, warm },
summary: {
cold: summarizeMeasurements(cold),
warm: summarizeMeasurements(warm),
},
};
const outputPath = path.join(outputDirectory, `board-${Date.now()}.json`);
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
console.log(JSON.stringify({ outputPath, summary: result.summary }, null, 2));