test(performance): add authenticated local benchmarks
This commit is contained in:
1 parent
f8d8a09c56
commit
440626ba00
8 files changed
+389
-2
No files matched your search
@@ -78,3 +78,4 @@ skills-lock.json
|
||||
apps/docs/.docusaurus
|
||||
apps/docs/build
|
||||
.pnpm-store
|
||||
benchmark-results
|
||||
@@ -17,7 +17,7 @@ export const ClientBoard = () => {
|
||||
.sort((sectionA, sectionB) => sectionA.yOffset - sectionB.yOffset);
|
||||
|
||||
return (
|
||||
<Box h="100%" pos="relative">
|
||||
<Box h="100%" pos="relative" data-homarr-dev-benchmark-board>
|
||||
<BoardBackgroundVideo />
|
||||
<Stack h="100%">
|
||||
{fullWidthSortedSections.map((section) =>
|
||||
|
||||
@@ -14,6 +14,7 @@ export const DesktopSearchInput = () => {
|
||||
|
||||
return (
|
||||
<TextInput
|
||||
data-homarr-dev-benchmark-interaction="search"
|
||||
component={UnstyledButton}
|
||||
className={classes.desktopSearch}
|
||||
w={400}
|
||||
@@ -29,7 +30,11 @@ export const DesktopSearchInput = () => {
|
||||
|
||||
export const MobileSearchButton = () => {
|
||||
return (
|
||||
<HeaderButton onClick={openSpotlight} className={classes.mobileSearch}>
|
||||
<HeaderButton
|
||||
data-homarr-dev-benchmark-interaction="search"
|
||||
onClick={openSpotlight}
|
||||
className={classes.mobileSearch}
|
||||
>
|
||||
<IconSearch size={20} stroke={1.5} />
|
||||
</HeaderButton>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { createId } from "@paralleldrive/cuid2";
|
||||
import { chromium, expect } from "@playwright/test";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { stringify } from "superjson";
|
||||
import { describe, test } from "vitest";
|
||||
|
||||
import * as sqliteSchema from "../packages/db/schema/sqlite";
|
||||
import { createHomarrContainer } from "./shared/create-homarr-container";
|
||||
import { createSqliteDbFileAsync } from "./shared/e2e-db";
|
||||
import { seedAdminUserAsync } from "./shared/seed-admin-user";
|
||||
|
||||
const adminCredentials = {
|
||||
username: "admin",
|
||||
password: "Comp(exP4sswOrd",
|
||||
};
|
||||
|
||||
describe("lazy widget application graph", () => {
|
||||
test("renders an authenticated board and preserves first-use UI behavior after reload", async () => {
|
||||
const { db, localMountPath } = await createSqliteDbFileAsync();
|
||||
const { userId } = await seedAdminUserAsync(db, adminCredentials);
|
||||
const boardId = createId();
|
||||
const layoutId = createId();
|
||||
const sectionId = createId();
|
||||
const itemId = createId();
|
||||
|
||||
await db.insert(sqliteSchema.boards).values({
|
||||
id: boardId,
|
||||
name: "lazy-widget-board",
|
||||
creatorId: userId,
|
||||
});
|
||||
await db.insert(sqliteSchema.layouts).values({
|
||||
id: layoutId,
|
||||
name: "Base",
|
||||
boardId,
|
||||
columnCount: 12,
|
||||
});
|
||||
await db.insert(sqliteSchema.sections).values({
|
||||
id: sectionId,
|
||||
boardId,
|
||||
kind: "empty",
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
});
|
||||
await db.insert(sqliteSchema.items).values({
|
||||
id: itemId,
|
||||
boardId,
|
||||
kind: "clock",
|
||||
options: stringify({ showSeconds: true }),
|
||||
});
|
||||
await db.insert(sqliteSchema.itemLayouts).values({
|
||||
itemId,
|
||||
sectionId,
|
||||
layoutId,
|
||||
xOffset: 0,
|
||||
yOffset: 0,
|
||||
width: 4,
|
||||
height: 2,
|
||||
});
|
||||
await db.update(sqliteSchema.users).set({ homeBoardId: boardId }).where(eq(sqliteSchema.users.id, userId));
|
||||
|
||||
const homarrContainer = await createHomarrContainer({
|
||||
environment: {
|
||||
AUTH_PROVIDERS: "credentials",
|
||||
},
|
||||
mounts: {
|
||||
"/appdata": localMountPath,
|
||||
},
|
||||
}).start();
|
||||
|
||||
const baseUrl = `http://${homarrContainer.getHost()}:${homarrContainer.getMappedPort(7575)}`;
|
||||
const browser = await chromium.launch();
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
const pageErrors: Error[] = [];
|
||||
page.on("pageerror", (error) => pageErrors.push(error));
|
||||
|
||||
try {
|
||||
await page.goto(`${baseUrl}/auth/login`);
|
||||
await page.getByLabel("Username").fill(adminCredentials.username);
|
||||
await page.locator("#password").fill(adminCredentials.password);
|
||||
await page.locator("css=button[type='submit']").click();
|
||||
await page.waitForURL(baseUrl, { timeout: 15_000 });
|
||||
|
||||
await expect(page.locator("[data-homarr-dev-benchmark-board]")).toBeVisible();
|
||||
await expect(page.locator(`[data-id="${itemId}"] .clock-widget-container`)).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await page.locator('[data-homarr-dev-benchmark-interaction="search"]').first().click();
|
||||
await expect(page.locator("[data-homarr-dev-benchmark-spotlight]")).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(page.locator("[data-homarr-dev-benchmark-spotlight]")).not.toBeVisible();
|
||||
|
||||
await expect
|
||||
.poll(
|
||||
async () =>
|
||||
await page.evaluate(async () => {
|
||||
const registrations = await navigator.serviceWorker.getRegistrations();
|
||||
return registrations[0]?.scope ?? null;
|
||||
}),
|
||||
{ timeout: 15_000 },
|
||||
)
|
||||
.toBe(`${baseUrl}/`);
|
||||
|
||||
await page.reload();
|
||||
await expect(page.locator(`[data-id="${itemId}"] .clock-widget-container`)).toBeVisible({ timeout: 15_000 });
|
||||
expect(pageErrors).toEqual([]);
|
||||
} finally {
|
||||
await browser.close();
|
||||
await homarrContainer.stop();
|
||||
}
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -134,6 +134,7 @@ const SpotlightWithActiveMode = ({ modeState, queryState, activeMode }: Spotligh
|
||||
store={spotlightStore}
|
||||
>
|
||||
<MantineSpotlight.Search
|
||||
data-homarr-dev-benchmark-spotlight
|
||||
placeholder={`${t("search.placeholder")}...`}
|
||||
ref={inputRef}
|
||||
leftSectionWidth={hasModeCharacter ? 80 : 48}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
export type NumericSummary = {
|
||||
count: number;
|
||||
medianMs: number | null;
|
||||
p95Ms: number | null;
|
||||
standardDeviationMs: number | null;
|
||||
outliersMs: number[];
|
||||
};
|
||||
|
||||
export const normalizeTrpcPath = (requestUrl: string) => {
|
||||
try {
|
||||
const url = new URL(requestUrl, "http://benchmark.invalid");
|
||||
url.searchParams.delete("input");
|
||||
return url.pathname + url.search;
|
||||
} catch {
|
||||
return requestUrl;
|
||||
}
|
||||
};
|
||||
|
||||
export const countDuplicateTrpcPaths = (requestUrls: string[]) => {
|
||||
const paths = requestUrls.map(normalizeTrpcPath);
|
||||
return Math.max(0, paths.length - new Set(paths).size);
|
||||
};
|
||||
|
||||
const values = (numbers: Array<number | null | undefined>) =>
|
||||
numbers.filter((value): value is number => typeof value === "number" && value >= 0).toSorted((a, b) => a - b);
|
||||
|
||||
export const summarize = (numbers: Array<number | null | undefined>): NumericSummary => {
|
||||
const sorted = values(numbers);
|
||||
if (sorted.length === 0) {
|
||||
return { count: 0, medianMs: null, p95Ms: null, standardDeviationMs: null, outliersMs: [] };
|
||||
}
|
||||
|
||||
const medianIndex = Math.floor(sorted.length / 2);
|
||||
const p95Index = Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1);
|
||||
const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length;
|
||||
const variance = sorted.reduce((total, value) => total + (value - mean) ** 2, 0) / sorted.length;
|
||||
const firstQuartile = sorted[Math.floor((sorted.length - 1) * 0.25)] ?? 0;
|
||||
const thirdQuartile = sorted[Math.floor((sorted.length - 1) * 0.75)] ?? 0;
|
||||
const iqr = thirdQuartile - firstQuartile;
|
||||
const outlierLimit = thirdQuartile + Math.max(1, iqr * 1.5);
|
||||
|
||||
return {
|
||||
count: sorted.length,
|
||||
medianMs: sorted[medianIndex] ?? 0,
|
||||
p95Ms: sorted[p95Index] ?? 0,
|
||||
standardDeviationMs: Math.round(Math.sqrt(variance)),
|
||||
outliersMs: sorted.filter((value) => value > outlierLimit),
|
||||
};
|
||||
};
|
||||
|
||||
export const validateBrowserMeasurement = (measurement: {
|
||||
responseStatus: number | null;
|
||||
domContentLoadedMs: number;
|
||||
boardHydratedMs: number | null;
|
||||
firstInteractionMs: number | null;
|
||||
error?: string;
|
||||
}) => {
|
||||
const errors: string[] = [];
|
||||
if (measurement.error) errors.push(measurement.error);
|
||||
if (measurement.responseStatus === null) errors.push("browser navigation returned no response");
|
||||
else if (measurement.responseStatus < 200 || measurement.responseStatus >= 300)
|
||||
errors.push(`browser navigation returned HTTP ${measurement.responseStatus}`);
|
||||
if (measurement.domContentLoadedMs < 0) errors.push("DOMContentLoaded timing is missing");
|
||||
if (measurement.boardHydratedMs === null) errors.push("board hydration marker is missing");
|
||||
if (measurement.firstInteractionMs === null) errors.push("first interaction timing is missing");
|
||||
return errors;
|
||||
};
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
countDuplicateTrpcPaths,
|
||||
normalizeTrpcPath,
|
||||
summarize,
|
||||
validateBrowserMeasurement,
|
||||
} from "./dev-benchmark-lib.mts";
|
||||
|
||||
describe("development benchmark measurements", () => {
|
||||
it("normalizes tRPC input payloads before counting duplicates", () => {
|
||||
const first = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A1%7D";
|
||||
const second = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A2%7D";
|
||||
|
||||
expect(normalizeTrpcPath(first)).toBe("/api/trpc/board.get?batch=1");
|
||||
expect(countDuplicateTrpcPaths([first, second, "/api/trpc/user.get"])).toBe(1);
|
||||
});
|
||||
|
||||
it("reports median, p95, deviation, and high outliers", () => {
|
||||
expect(summarize([1, 2, 3, 4, 5, 6, 100])).toEqual({
|
||||
count: 7,
|
||||
medianMs: 4,
|
||||
p95Ms: 100,
|
||||
standardDeviationMs: 34,
|
||||
outliersMs: [100],
|
||||
});
|
||||
});
|
||||
|
||||
it("requires a successful authenticated board hydration and interaction", () => {
|
||||
expect(
|
||||
validateBrowserMeasurement({
|
||||
responseStatus: 307,
|
||||
domContentLoadedMs: 120,
|
||||
boardHydratedMs: null,
|
||||
firstInteractionMs: null,
|
||||
}),
|
||||
).toEqual([
|
||||
"browser navigation returned HTTP 307",
|
||||
"board hydration marker is missing",
|
||||
"first interaction timing is missing",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import type { Page } from "@playwright/test";
|
||||
import { chromium } from "@playwright/test";
|
||||
|
||||
import { countDuplicateTrpcPaths, summarize, validateBrowserMeasurement } from "./dev-benchmark-lib.mts";
|
||||
|
||||
type BoardMeasurement = {
|
||||
boardReadyMs: number;
|
||||
domContentLoadedMs: number;
|
||||
firstInteractionMs: number;
|
||||
pageErrors: string[];
|
||||
responseStatus: number | null;
|
||||
trpcDuplicateRequestCount: number;
|
||||
trpcRequestCount: number;
|
||||
};
|
||||
|
||||
const root = path.resolve(import.meta.dirname, "../..");
|
||||
const argumentsSet = new Set(process.argv.slice(2));
|
||||
const smokeOnly = argumentsSet.has("--smoke");
|
||||
const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 7);
|
||||
const outputDirectory = path.resolve(
|
||||
process.env.DEV_BENCHMARK_OUTPUT_DIR ?? path.join(root, "benchmark-results", "dev"),
|
||||
);
|
||||
const boardUrl = process.env.DEV_BENCHMARK_BOARD_URL;
|
||||
const storageStatePath = process.env.DEV_BENCHMARK_STORAGE_STATE;
|
||||
const readinessUrl = process.env.DEV_BENCHMARK_READINESS_URL ?? "http://127.0.0.1:3000/api/health/ready";
|
||||
const boardSelector = "[data-homarr-dev-benchmark-board]";
|
||||
const interactionSelector = '[data-homarr-dev-benchmark-interaction="search"]';
|
||||
const spotlightSelector = "[data-homarr-dev-benchmark-spotlight]";
|
||||
|
||||
if (!Number.isInteger(iterations) || iterations < 7) {
|
||||
throw new Error("DEV_BENCHMARK_ITERATIONS must be an integer of at least 7");
|
||||
}
|
||||
|
||||
await fs.mkdir(outputDirectory, { recursive: true });
|
||||
|
||||
if (smokeOnly) {
|
||||
const measurements: number[] = [];
|
||||
for (let iteration = 0; iteration < iterations; iteration++) {
|
||||
const startedAt = performance.now();
|
||||
const response = await fetch(readinessUrl);
|
||||
await response.arrayBuffer();
|
||||
if (!response.ok) throw new Error(`Readiness smoke received HTTP ${response.status}`);
|
||||
measurements.push(Math.round(performance.now() - startedAt));
|
||||
}
|
||||
|
||||
const result = {
|
||||
kind: "readiness-smoke",
|
||||
claimEligible: false,
|
||||
url: readinessUrl,
|
||||
iterations,
|
||||
summary: summarize(measurements),
|
||||
};
|
||||
const outputPath = path.join(outputDirectory, `smoke-${Date.now()}.json`);
|
||||
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
|
||||
console.log(JSON.stringify({ outputPath, result }, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (!boardUrl || !storageStatePath) {
|
||||
throw new Error(
|
||||
"Authenticated claim mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " +
|
||||
"Use --smoke only for the non-claimable readiness check.",
|
||||
);
|
||||
}
|
||||
|
||||
await fs.access(storageStatePath);
|
||||
|
||||
const browser = await chromium.launch();
|
||||
const cold: BoardMeasurement[] = [];
|
||||
const warm: BoardMeasurement[] = [];
|
||||
|
||||
const measureBoardAsync = async (page: Page) => {
|
||||
const trpcRequests: string[] = [];
|
||||
const pageErrors: string[] = [];
|
||||
page.on("request", (request) => {
|
||||
if (request.url().includes("/api/trpc/")) trpcRequests.push(request.url());
|
||||
});
|
||||
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||
|
||||
const startedAt = performance.now();
|
||||
const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
|
||||
const domContentLoadedMs = Math.round(performance.now() - startedAt);
|
||||
await page.locator(boardSelector).waitFor({ state: "visible" });
|
||||
const boardReadyMs = Math.round(performance.now() - startedAt);
|
||||
|
||||
const interactionStartedAt = performance.now();
|
||||
await page.locator(interactionSelector).first().click();
|
||||
await page.locator(spotlightSelector).waitFor({ state: "visible" });
|
||||
const firstInteractionMs = Math.round(performance.now() - interactionStartedAt);
|
||||
|
||||
const measurement: BoardMeasurement = {
|
||||
responseStatus: response?.status() ?? null,
|
||||
domContentLoadedMs,
|
||||
boardReadyMs,
|
||||
firstInteractionMs,
|
||||
trpcRequestCount: trpcRequests.length,
|
||||
trpcDuplicateRequestCount: countDuplicateTrpcPaths(trpcRequests),
|
||||
pageErrors,
|
||||
};
|
||||
const errors = validateBrowserMeasurement({
|
||||
responseStatus: measurement.responseStatus,
|
||||
domContentLoadedMs,
|
||||
boardHydratedMs: boardReadyMs,
|
||||
firstInteractionMs,
|
||||
error: pageErrors.length > 0 ? pageErrors.join("; ") : undefined,
|
||||
});
|
||||
if (errors.length > 0) throw new Error(errors.map((error) => `${boardUrl}: ${error}`).join("; "));
|
||||
|
||||
return measurement;
|
||||
};
|
||||
|
||||
try {
|
||||
for (let iteration = 0; iteration < iterations; iteration++) {
|
||||
const context = await browser.newContext({ storageState: storageStatePath });
|
||||
const coldPage = await context.newPage();
|
||||
cold.push(await measureBoardAsync(coldPage));
|
||||
await coldPage.close();
|
||||
const warmPage = await context.newPage();
|
||||
warm.push(await measureBoardAsync(warmPage));
|
||||
await context.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
const summarizeMeasurements = (measurements: BoardMeasurement[]) => ({
|
||||
boardReady: summarize(measurements.map(({ boardReadyMs }) => boardReadyMs)),
|
||||
domContentLoaded: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)),
|
||||
firstInteraction: summarize(measurements.map(({ firstInteractionMs }) => firstInteractionMs)),
|
||||
trpcRequests: summarize(measurements.map(({ trpcRequestCount }) => trpcRequestCount)),
|
||||
duplicateTrpcRequests: summarize(measurements.map(({ trpcDuplicateRequestCount }) => trpcDuplicateRequestCount)),
|
||||
});
|
||||
|
||||
const result = {
|
||||
kind: "authenticated-board",
|
||||
claimEligible: true,
|
||||
boardUrl,
|
||||
createdAt: new Date().toISOString(),
|
||||
environment: {
|
||||
architecture: process.arch,
|
||||
cpus: os.cpus().map(({ model }) => model),
|
||||
node: process.version,
|
||||
platform: `${process.platform} ${os.release()}`,
|
||||
sha: process.env.DEV_BENCHMARK_SHA ?? null,
|
||||
},
|
||||
iterations,
|
||||
measurements: { cold, warm },
|
||||
summary: {
|
||||
cold: summarizeMeasurements(cold),
|
||||
warm: summarizeMeasurements(warm),
|
||||
},
|
||||
};
|
||||
const outputPath = path.join(outputDirectory, `board-${Date.now()}.json`);
|
||||
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
|
||||
console.log(JSON.stringify({ outputPath, summary: result.summary }, null, 2));
|
||||
Reference in new issue
Block a user