test(performance): add authenticated local benchmarks

This commit is contained in:
ajnart committed 2026-07-29 06:47:07 +02:00
1 parent f8d8a09c56
commit 440626ba00
8 files changed
+389 -2

No files matched your search

+1
View File
@@ -78,3 +78,4 @@ skills-lock.json
apps/docs/.docusaurus
apps/docs/build
.pnpm-store
benchmark-results
@@ -17,7 +17,7 @@ export const ClientBoard = () => {
.sort((sectionA, sectionB) => sectionA.yOffset - sectionB.yOffset);
return (
<Box h="100%" pos="relative">
<Box h="100%" pos="relative" data-homarr-dev-benchmark-board>
<BoardBackgroundVideo />
<Stack h="100%">
{fullWidthSortedSections.map((section) =>
@@ -14,6 +14,7 @@ export const DesktopSearchInput = () => {
return (
<TextInput
data-homarr-dev-benchmark-interaction="search"
component={UnstyledButton}
className={classes.desktopSearch}
w={400}
@@ -29,7 +30,11 @@ export const DesktopSearchInput = () => {
export const MobileSearchButton = () => {
return (
<HeaderButton onClick={openSpotlight} className={classes.mobileSearch}>
<HeaderButton
data-homarr-dev-benchmark-interaction="search"
onClick={openSpotlight}
className={classes.mobileSearch}
>
<IconSearch size={20} stroke={1.5} />
</HeaderButton>
);
+111
View File
@@ -0,0 +1,111 @@
import { createId } from "@paralleldrive/cuid2";
import { chromium, expect } from "@playwright/test";
import { eq } from "drizzle-orm";
import { stringify } from "superjson";
import { describe, test } from "vitest";
import * as sqliteSchema from "../packages/db/schema/sqlite";
import { createHomarrContainer } from "./shared/create-homarr-container";
import { createSqliteDbFileAsync } from "./shared/e2e-db";
import { seedAdminUserAsync } from "./shared/seed-admin-user";
const adminCredentials = {
username: "admin",
password: "Comp(exP4sswOrd",
};
describe("lazy widget application graph", () => {
test("renders an authenticated board and preserves first-use UI behavior after reload", async () => {
const { db, localMountPath } = await createSqliteDbFileAsync();
const { userId } = await seedAdminUserAsync(db, adminCredentials);
const boardId = createId();
const layoutId = createId();
const sectionId = createId();
const itemId = createId();
await db.insert(sqliteSchema.boards).values({
id: boardId,
name: "lazy-widget-board",
creatorId: userId,
});
await db.insert(sqliteSchema.layouts).values({
id: layoutId,
name: "Base",
boardId,
columnCount: 12,
});
await db.insert(sqliteSchema.sections).values({
id: sectionId,
boardId,
kind: "empty",
xOffset: 0,
yOffset: 0,
});
await db.insert(sqliteSchema.items).values({
id: itemId,
boardId,
kind: "clock",
options: stringify({ showSeconds: true }),
});
await db.insert(sqliteSchema.itemLayouts).values({
itemId,
sectionId,
layoutId,
xOffset: 0,
yOffset: 0,
width: 4,
height: 2,
});
await db.update(sqliteSchema.users).set({ homeBoardId: boardId }).where(eq(sqliteSchema.users.id, userId));
const homarrContainer = await createHomarrContainer({
environment: {
AUTH_PROVIDERS: "credentials",
},
mounts: {
"/appdata": localMountPath,
},
}).start();
const baseUrl = `http://${homarrContainer.getHost()}:${homarrContainer.getMappedPort(7575)}`;
const browser = await chromium.launch();
const context = await browser.newContext();
const page = await context.newPage();
const pageErrors: Error[] = [];
page.on("pageerror", (error) => pageErrors.push(error));
try {
await page.goto(`${baseUrl}/auth/login`);
await page.getByLabel("Username").fill(adminCredentials.username);
await page.locator("#password").fill(adminCredentials.password);
await page.locator("css=button[type='submit']").click();
await page.waitForURL(baseUrl, { timeout: 15_000 });
await expect(page.locator("[data-homarr-dev-benchmark-board]")).toBeVisible();
await expect(page.locator(`[data-id="${itemId}"] .clock-widget-container`)).toBeVisible({ timeout: 15_000 });
await page.locator('[data-homarr-dev-benchmark-interaction="search"]').first().click();
await expect(page.locator("[data-homarr-dev-benchmark-spotlight]")).toBeVisible();
await page.keyboard.press("Escape");
await expect(page.locator("[data-homarr-dev-benchmark-spotlight]")).not.toBeVisible();
await expect
.poll(
async () =>
await page.evaluate(async () => {
const registrations = await navigator.serviceWorker.getRegistrations();
return registrations[0]?.scope ?? null;
}),
{ timeout: 15_000 },
)
.toBe(`${baseUrl}/`);
await page.reload();
await expect(page.locator(`[data-id="${itemId}"] .clock-widget-container`)).toBeVisible({ timeout: 15_000 });
expect(pageErrors).toEqual([]);
} finally {
await browser.close();
await homarrContainer.stop();
}
}, 120_000);
});
@@ -134,6 +134,7 @@ const SpotlightWithActiveMode = ({ modeState, queryState, activeMode }: Spotligh
store={spotlightStore}
>
<MantineSpotlight.Search
data-homarr-dev-benchmark-spotlight
placeholder={`${t("search.placeholder")}...`}
ref={inputRef}
leftSectionWidth={hasModeCharacter ? 80 : 48}
+67
View File
@@ -0,0 +1,67 @@
export type NumericSummary = {
count: number;
medianMs: number | null;
p95Ms: number | null;
standardDeviationMs: number | null;
outliersMs: number[];
};
export const normalizeTrpcPath = (requestUrl: string) => {
try {
const url = new URL(requestUrl, "http://benchmark.invalid");
url.searchParams.delete("input");
return url.pathname + url.search;
} catch {
return requestUrl;
}
};
export const countDuplicateTrpcPaths = (requestUrls: string[]) => {
const paths = requestUrls.map(normalizeTrpcPath);
return Math.max(0, paths.length - new Set(paths).size);
};
const values = (numbers: Array<number | null | undefined>) =>
numbers.filter((value): value is number => typeof value === "number" && value >= 0).toSorted((a, b) => a - b);
export const summarize = (numbers: Array<number | null | undefined>): NumericSummary => {
const sorted = values(numbers);
if (sorted.length === 0) {
return { count: 0, medianMs: null, p95Ms: null, standardDeviationMs: null, outliersMs: [] };
}
const medianIndex = Math.floor(sorted.length / 2);
const p95Index = Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1);
const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length;
const variance = sorted.reduce((total, value) => total + (value - mean) ** 2, 0) / sorted.length;
const firstQuartile = sorted[Math.floor((sorted.length - 1) * 0.25)] ?? 0;
const thirdQuartile = sorted[Math.floor((sorted.length - 1) * 0.75)] ?? 0;
const iqr = thirdQuartile - firstQuartile;
const outlierLimit = thirdQuartile + Math.max(1, iqr * 1.5);
return {
count: sorted.length,
medianMs: sorted[medianIndex] ?? 0,
p95Ms: sorted[p95Index] ?? 0,
standardDeviationMs: Math.round(Math.sqrt(variance)),
outliersMs: sorted.filter((value) => value > outlierLimit),
};
};
export const validateBrowserMeasurement = (measurement: {
responseStatus: number | null;
domContentLoadedMs: number;
boardHydratedMs: number | null;
firstInteractionMs: number | null;
error?: string;
}) => {
const errors: string[] = [];
if (measurement.error) errors.push(measurement.error);
if (measurement.responseStatus === null) errors.push("browser navigation returned no response");
else if (measurement.responseStatus < 200 || measurement.responseStatus >= 300)
errors.push(`browser navigation returned HTTP ${measurement.responseStatus}`);
if (measurement.domContentLoadedMs < 0) errors.push("DOMContentLoaded timing is missing");
if (measurement.boardHydratedMs === null) errors.push("board hydration marker is missing");
if (measurement.firstInteractionMs === null) errors.push("first interaction timing is missing");
return errors;
};
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import {
countDuplicateTrpcPaths,
normalizeTrpcPath,
summarize,
validateBrowserMeasurement,
} from "./dev-benchmark-lib.mts";
describe("development benchmark measurements", () => {
it("normalizes tRPC input payloads before counting duplicates", () => {
const first = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A1%7D";
const second = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A2%7D";
expect(normalizeTrpcPath(first)).toBe("/api/trpc/board.get?batch=1");
expect(countDuplicateTrpcPaths([first, second, "/api/trpc/user.get"])).toBe(1);
});
it("reports median, p95, deviation, and high outliers", () => {
expect(summarize([1, 2, 3, 4, 5, 6, 100])).toEqual({
count: 7,
medianMs: 4,
p95Ms: 100,
standardDeviationMs: 34,
outliersMs: [100],
});
});
it("requires a successful authenticated board hydration and interaction", () => {
expect(
validateBrowserMeasurement({
responseStatus: 307,
domContentLoadedMs: 120,
boardHydratedMs: null,
firstInteractionMs: null,
}),
).toEqual([
"browser navigation returned HTTP 307",
"board hydration marker is missing",
"first interaction timing is missing",
]);
});
});
+159
View File
@@ -0,0 +1,159 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
import type { Page } from "@playwright/test";
import { chromium } from "@playwright/test";
import { countDuplicateTrpcPaths, summarize, validateBrowserMeasurement } from "./dev-benchmark-lib.mts";
type BoardMeasurement = {
boardReadyMs: number;
domContentLoadedMs: number;
firstInteractionMs: number;
pageErrors: string[];
responseStatus: number | null;
trpcDuplicateRequestCount: number;
trpcRequestCount: number;
};
const root = path.resolve(import.meta.dirname, "../..");
const argumentsSet = new Set(process.argv.slice(2));
const smokeOnly = argumentsSet.has("--smoke");
const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 7);
const outputDirectory = path.resolve(
process.env.DEV_BENCHMARK_OUTPUT_DIR ?? path.join(root, "benchmark-results", "dev"),
);
const boardUrl = process.env.DEV_BENCHMARK_BOARD_URL;
const storageStatePath = process.env.DEV_BENCHMARK_STORAGE_STATE;
const readinessUrl = process.env.DEV_BENCHMARK_READINESS_URL ?? "http://127.0.0.1:3000/api/health/ready";
const boardSelector = "[data-homarr-dev-benchmark-board]";
const interactionSelector = '[data-homarr-dev-benchmark-interaction="search"]';
const spotlightSelector = "[data-homarr-dev-benchmark-spotlight]";
if (!Number.isInteger(iterations) || iterations < 7) {
throw new Error("DEV_BENCHMARK_ITERATIONS must be an integer of at least 7");
}
await fs.mkdir(outputDirectory, { recursive: true });
if (smokeOnly) {
const measurements: number[] = [];
for (let iteration = 0; iteration < iterations; iteration++) {
const startedAt = performance.now();
const response = await fetch(readinessUrl);
await response.arrayBuffer();
if (!response.ok) throw new Error(`Readiness smoke received HTTP ${response.status}`);
measurements.push(Math.round(performance.now() - startedAt));
}
const result = {
kind: "readiness-smoke",
claimEligible: false,
url: readinessUrl,
iterations,
summary: summarize(measurements),
};
const outputPath = path.join(outputDirectory, `smoke-${Date.now()}.json`);
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
console.log(JSON.stringify({ outputPath, result }, null, 2));
process.exit(0);
}
if (!boardUrl || !storageStatePath) {
throw new Error(
"Authenticated claim mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " +
"Use --smoke only for the non-claimable readiness check.",
);
}
await fs.access(storageStatePath);
const browser = await chromium.launch();
const cold: BoardMeasurement[] = [];
const warm: BoardMeasurement[] = [];
const measureBoardAsync = async (page: Page) => {
const trpcRequests: string[] = [];
const pageErrors: string[] = [];
page.on("request", (request) => {
if (request.url().includes("/api/trpc/")) trpcRequests.push(request.url());
});
page.on("pageerror", (error) => pageErrors.push(error.message));
const startedAt = performance.now();
const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
const domContentLoadedMs = Math.round(performance.now() - startedAt);
await page.locator(boardSelector).waitFor({ state: "visible" });
const boardReadyMs = Math.round(performance.now() - startedAt);
const interactionStartedAt = performance.now();
await page.locator(interactionSelector).first().click();
await page.locator(spotlightSelector).waitFor({ state: "visible" });
const firstInteractionMs = Math.round(performance.now() - interactionStartedAt);
const measurement: BoardMeasurement = {
responseStatus: response?.status() ?? null,
domContentLoadedMs,
boardReadyMs,
firstInteractionMs,
trpcRequestCount: trpcRequests.length,
trpcDuplicateRequestCount: countDuplicateTrpcPaths(trpcRequests),
pageErrors,
};
const errors = validateBrowserMeasurement({
responseStatus: measurement.responseStatus,
domContentLoadedMs,
boardHydratedMs: boardReadyMs,
firstInteractionMs,
error: pageErrors.length > 0 ? pageErrors.join("; ") : undefined,
});
if (errors.length > 0) throw new Error(errors.map((error) => `${boardUrl}: ${error}`).join("; "));
return measurement;
};
try {
for (let iteration = 0; iteration < iterations; iteration++) {
const context = await browser.newContext({ storageState: storageStatePath });
const coldPage = await context.newPage();
cold.push(await measureBoardAsync(coldPage));
await coldPage.close();
const warmPage = await context.newPage();
warm.push(await measureBoardAsync(warmPage));
await context.close();
}
} finally {
await browser.close();
}
const summarizeMeasurements = (measurements: BoardMeasurement[]) => ({
boardReady: summarize(measurements.map(({ boardReadyMs }) => boardReadyMs)),
domContentLoaded: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)),
firstInteraction: summarize(measurements.map(({ firstInteractionMs }) => firstInteractionMs)),
trpcRequests: summarize(measurements.map(({ trpcRequestCount }) => trpcRequestCount)),
duplicateTrpcRequests: summarize(measurements.map(({ trpcDuplicateRequestCount }) => trpcDuplicateRequestCount)),
});
const result = {
kind: "authenticated-board",
claimEligible: true,
boardUrl,
createdAt: new Date().toISOString(),
environment: {
architecture: process.arch,
cpus: os.cpus().map(({ model }) => model),
node: process.version,
platform: `${process.platform} ${os.release()}`,
sha: process.env.DEV_BENCHMARK_SHA ?? null,
},
iterations,
measurements: { cold, warm },
summary: {
cold: summarizeMeasurements(cold),
warm: summarizeMeasurements(warm),
},
};
const outputPath = path.join(outputDirectory, `board-${Date.now()}.json`);
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
console.log(JSON.stringify({ outputPath, summary: result.summary }, null, 2));