From 440626ba006063893f39957471678b7caff4cbe0 Mon Sep 17 00:00:00 2001 From: ajnart Date: Tue, 28 Jul 2026 17:43:04 +0200 Subject: [PATCH] test(performance): add authenticated local benchmarks --- .gitignore | 1 + .../app/[locale]/boards/(content)/_client.tsx | 2 +- .../src/components/layout/header/search.tsx | 7 +- e2e/lazy-widgets.spec.ts | 111 ++++++++++++ .../spotlight/src/components/spotlight.tsx | 1 + scripts/benchmarks/dev-benchmark-lib.mts | 67 ++++++++ scripts/benchmarks/dev-benchmark-lib.spec.ts | 43 +++++ scripts/benchmarks/dev-board.mts | 159 ++++++++++++++++++ 8 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 e2e/lazy-widgets.spec.ts create mode 100644 scripts/benchmarks/dev-benchmark-lib.mts create mode 100644 scripts/benchmarks/dev-benchmark-lib.spec.ts create mode 100644 scripts/benchmarks/dev-board.mts diff --git a/.gitignore b/.gitignore index c22262f6d..71a27829a 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,4 @@ skills-lock.json apps/docs/.docusaurus apps/docs/build .pnpm-store +benchmark-results diff --git a/apps/nextjs/src/app/[locale]/boards/(content)/_client.tsx b/apps/nextjs/src/app/[locale]/boards/(content)/_client.tsx index 0ba502a9a..1d453b514 100644 --- a/apps/nextjs/src/app/[locale]/boards/(content)/_client.tsx +++ b/apps/nextjs/src/app/[locale]/boards/(content)/_client.tsx @@ -17,7 +17,7 @@ export const ClientBoard = () => { .sort((sectionA, sectionB) => sectionA.yOffset - sectionB.yOffset); return ( - + {fullWidthSortedSections.map((section) => diff --git a/apps/nextjs/src/components/layout/header/search.tsx b/apps/nextjs/src/components/layout/header/search.tsx index 8f6e4f908..699b08bfa 100644 --- a/apps/nextjs/src/components/layout/header/search.tsx +++ b/apps/nextjs/src/components/layout/header/search.tsx @@ -14,6 +14,7 @@ export const DesktopSearchInput = () => { return ( { export const MobileSearchButton = () => { return ( - + ); diff --git a/e2e/lazy-widgets.spec.ts b/e2e/lazy-widgets.spec.ts new file mode 100644 index 000000000..dc7b252c8 --- /dev/null +++ b/e2e/lazy-widgets.spec.ts @@ -0,0 +1,111 @@ +import { createId } from "@paralleldrive/cuid2"; +import { chromium, expect } from "@playwright/test"; +import { eq } from "drizzle-orm"; +import { stringify } from "superjson"; +import { describe, test } from "vitest"; + +import * as sqliteSchema from "../packages/db/schema/sqlite"; +import { createHomarrContainer } from "./shared/create-homarr-container"; +import { createSqliteDbFileAsync } from "./shared/e2e-db"; +import { seedAdminUserAsync } from "./shared/seed-admin-user"; + +const adminCredentials = { + username: "admin", + password: "Comp(exP4sswOrd", +}; + +describe("lazy widget application graph", () => { + test("renders an authenticated board and preserves first-use UI behavior after reload", async () => { + const { db, localMountPath } = await createSqliteDbFileAsync(); + const { userId } = await seedAdminUserAsync(db, adminCredentials); + const boardId = createId(); + const layoutId = createId(); + const sectionId = createId(); + const itemId = createId(); + + await db.insert(sqliteSchema.boards).values({ + id: boardId, + name: "lazy-widget-board", + creatorId: userId, + }); + await db.insert(sqliteSchema.layouts).values({ + id: layoutId, + name: "Base", + boardId, + columnCount: 12, + }); + await db.insert(sqliteSchema.sections).values({ + id: sectionId, + boardId, + kind: "empty", + xOffset: 0, + yOffset: 0, + }); + await db.insert(sqliteSchema.items).values({ + id: itemId, + boardId, + kind: "clock", + options: stringify({ showSeconds: true }), + }); + await db.insert(sqliteSchema.itemLayouts).values({ + itemId, + sectionId, + layoutId, + xOffset: 0, + yOffset: 0, + width: 4, + height: 2, + }); + await db.update(sqliteSchema.users).set({ homeBoardId: boardId }).where(eq(sqliteSchema.users.id, userId)); + + const homarrContainer = await createHomarrContainer({ + environment: { + AUTH_PROVIDERS: "credentials", + }, + mounts: { + "/appdata": localMountPath, + }, + }).start(); + + const baseUrl = `http://${homarrContainer.getHost()}:${homarrContainer.getMappedPort(7575)}`; + const browser = await chromium.launch(); + const context = await browser.newContext(); + const page = await context.newPage(); + const pageErrors: Error[] = []; + page.on("pageerror", (error) => pageErrors.push(error)); + + try { + await page.goto(`${baseUrl}/auth/login`); + await page.getByLabel("Username").fill(adminCredentials.username); + await page.locator("#password").fill(adminCredentials.password); + await page.locator("css=button[type='submit']").click(); + await page.waitForURL(baseUrl, { timeout: 15_000 }); + + await expect(page.locator("[data-homarr-dev-benchmark-board]")).toBeVisible(); + await expect(page.locator(`[data-id="${itemId}"] .clock-widget-container`)).toBeVisible({ timeout: 15_000 }); + + await page.locator('[data-homarr-dev-benchmark-interaction="search"]').first().click(); + await expect(page.locator("[data-homarr-dev-benchmark-spotlight]")).toBeVisible(); + await page.keyboard.press("Escape"); + await expect(page.locator("[data-homarr-dev-benchmark-spotlight]")).not.toBeVisible(); + + await expect + .poll( + async () => + await page.evaluate(async () => { + const registrations = await navigator.serviceWorker.getRegistrations(); + return registrations[0]?.scope ?? null; + }), + { timeout: 15_000 }, + ) + .toBe(`${baseUrl}/`); + + await page.reload(); + await expect(page.locator(`[data-id="${itemId}"] .clock-widget-container`)).toBeVisible({ timeout: 15_000 }); + expect(pageErrors).toEqual([]); + } finally { + await browser.close(); + await homarrContainer.stop(); + } + }, 120_000); +}); diff --git a/packages/spotlight/src/components/spotlight.tsx b/packages/spotlight/src/components/spotlight.tsx index 89b1b0d63..4a2680b73 100644 --- a/packages/spotlight/src/components/spotlight.tsx +++ b/packages/spotlight/src/components/spotlight.tsx @@ -134,6 +134,7 @@ const SpotlightWithActiveMode = ({ modeState, queryState, activeMode }: Spotligh store={spotlightStore} > { + try { + const url = new URL(requestUrl, "http://benchmark.invalid"); + url.searchParams.delete("input"); + return url.pathname + url.search; + } catch { + return requestUrl; + } +}; + +export const countDuplicateTrpcPaths = (requestUrls: string[]) => { + const paths = requestUrls.map(normalizeTrpcPath); + return Math.max(0, paths.length - new Set(paths).size); +}; + +const values = (numbers: Array) => + numbers.filter((value): value is number => typeof value === "number" && value >= 0).toSorted((a, b) => a - b); + +export const summarize = (numbers: Array): NumericSummary => { + const sorted = values(numbers); + if (sorted.length === 0) { + return { count: 0, medianMs: null, p95Ms: null, standardDeviationMs: null, outliersMs: [] }; + } + + const medianIndex = Math.floor(sorted.length / 2); + const p95Index = Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1); + const mean = sorted.reduce((total, value) => total + value, 0) / sorted.length; + const variance = sorted.reduce((total, value) => total + (value - mean) ** 2, 0) / sorted.length; + const firstQuartile = sorted[Math.floor((sorted.length - 1) * 0.25)] ?? 0; + const thirdQuartile = sorted[Math.floor((sorted.length - 1) * 0.75)] ?? 0; + const iqr = thirdQuartile - firstQuartile; + const outlierLimit = thirdQuartile + Math.max(1, iqr * 1.5); + + return { + count: sorted.length, + medianMs: sorted[medianIndex] ?? 0, + p95Ms: sorted[p95Index] ?? 0, + standardDeviationMs: Math.round(Math.sqrt(variance)), + outliersMs: sorted.filter((value) => value > outlierLimit), + }; +}; + +export const validateBrowserMeasurement = (measurement: { + responseStatus: number | null; + domContentLoadedMs: number; + boardHydratedMs: number | null; + firstInteractionMs: number | null; + error?: string; +}) => { + const errors: string[] = []; + if (measurement.error) errors.push(measurement.error); + if (measurement.responseStatus === null) errors.push("browser navigation returned no response"); + else if (measurement.responseStatus < 200 || measurement.responseStatus >= 300) + errors.push(`browser navigation returned HTTP ${measurement.responseStatus}`); + if (measurement.domContentLoadedMs < 0) errors.push("DOMContentLoaded timing is missing"); + if (measurement.boardHydratedMs === null) errors.push("board hydration marker is missing"); + if (measurement.firstInteractionMs === null) errors.push("first interaction timing is missing"); + return errors; +}; diff --git a/scripts/benchmarks/dev-benchmark-lib.spec.ts b/scripts/benchmarks/dev-benchmark-lib.spec.ts new file mode 100644 index 000000000..f1e113376 --- /dev/null +++ b/scripts/benchmarks/dev-benchmark-lib.spec.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; + +import { + countDuplicateTrpcPaths, + normalizeTrpcPath, + summarize, + validateBrowserMeasurement, +} from "./dev-benchmark-lib.mts"; + +describe("development benchmark measurements", () => { + it("normalizes tRPC input payloads before counting duplicates", () => { + const first = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A1%7D"; + const second = "/api/trpc/board.get?batch=1&input=%7B%22json%22%3A2%7D"; + + expect(normalizeTrpcPath(first)).toBe("/api/trpc/board.get?batch=1"); + expect(countDuplicateTrpcPaths([first, second, "/api/trpc/user.get"])).toBe(1); + }); + + it("reports median, p95, deviation, and high outliers", () => { + expect(summarize([1, 2, 3, 4, 5, 6, 100])).toEqual({ + count: 7, + medianMs: 4, + p95Ms: 100, + standardDeviationMs: 34, + outliersMs: [100], + }); + }); + + it("requires a successful authenticated board hydration and interaction", () => { + expect( + validateBrowserMeasurement({ + responseStatus: 307, + domContentLoadedMs: 120, + boardHydratedMs: null, + firstInteractionMs: null, + }), + ).toEqual([ + "browser navigation returned HTTP 307", + "board hydration marker is missing", + "first interaction timing is missing", + ]); + }); +}); diff --git a/scripts/benchmarks/dev-board.mts b/scripts/benchmarks/dev-board.mts new file mode 100644 index 000000000..7b400ab7f --- /dev/null +++ b/scripts/benchmarks/dev-board.mts @@ -0,0 +1,159 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import type { Page } from "@playwright/test"; +import { chromium } from "@playwright/test"; + +import { countDuplicateTrpcPaths, summarize, validateBrowserMeasurement } from "./dev-benchmark-lib.mts"; + +type BoardMeasurement = { + boardReadyMs: number; + domContentLoadedMs: number; + firstInteractionMs: number; + pageErrors: string[]; + responseStatus: number | null; + trpcDuplicateRequestCount: number; + trpcRequestCount: number; +}; + +const root = path.resolve(import.meta.dirname, "../.."); +const argumentsSet = new Set(process.argv.slice(2)); +const smokeOnly = argumentsSet.has("--smoke"); +const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 7); +const outputDirectory = path.resolve( + process.env.DEV_BENCHMARK_OUTPUT_DIR ?? path.join(root, "benchmark-results", "dev"), +); +const boardUrl = process.env.DEV_BENCHMARK_BOARD_URL; +const storageStatePath = process.env.DEV_BENCHMARK_STORAGE_STATE; +const readinessUrl = process.env.DEV_BENCHMARK_READINESS_URL ?? "http://127.0.0.1:3000/api/health/ready"; +const boardSelector = "[data-homarr-dev-benchmark-board]"; +const interactionSelector = '[data-homarr-dev-benchmark-interaction="search"]'; +const spotlightSelector = "[data-homarr-dev-benchmark-spotlight]"; + +if (!Number.isInteger(iterations) || iterations < 7) { + throw new Error("DEV_BENCHMARK_ITERATIONS must be an integer of at least 7"); +} + +await fs.mkdir(outputDirectory, { recursive: true }); + +if (smokeOnly) { + const measurements: number[] = []; + for (let iteration = 0; iteration < iterations; iteration++) { + const startedAt = performance.now(); + const response = await fetch(readinessUrl); + await response.arrayBuffer(); + if (!response.ok) throw new Error(`Readiness smoke received HTTP ${response.status}`); + measurements.push(Math.round(performance.now() - startedAt)); + } + + const result = { + kind: "readiness-smoke", + claimEligible: false, + url: readinessUrl, + iterations, + summary: summarize(measurements), + }; + const outputPath = path.join(outputDirectory, `smoke-${Date.now()}.json`); + await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`); + console.log(JSON.stringify({ outputPath, result }, null, 2)); + process.exit(0); +} + +if (!boardUrl || !storageStatePath) { + throw new Error( + "Authenticated claim mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " + + "Use --smoke only for the non-claimable readiness check.", + ); +} + +await fs.access(storageStatePath); + +const browser = await chromium.launch(); +const cold: BoardMeasurement[] = []; +const warm: BoardMeasurement[] = []; + +const measureBoardAsync = async (page: Page) => { + const trpcRequests: string[] = []; + const pageErrors: string[] = []; + page.on("request", (request) => { + if (request.url().includes("/api/trpc/")) trpcRequests.push(request.url()); + }); + page.on("pageerror", (error) => pageErrors.push(error.message)); + + const startedAt = performance.now(); + const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" }); + const domContentLoadedMs = Math.round(performance.now() - startedAt); + await page.locator(boardSelector).waitFor({ state: "visible" }); + const boardReadyMs = Math.round(performance.now() - startedAt); + + const interactionStartedAt = performance.now(); + await page.locator(interactionSelector).first().click(); + await page.locator(spotlightSelector).waitFor({ state: "visible" }); + const firstInteractionMs = Math.round(performance.now() - interactionStartedAt); + + const measurement: BoardMeasurement = { + responseStatus: response?.status() ?? null, + domContentLoadedMs, + boardReadyMs, + firstInteractionMs, + trpcRequestCount: trpcRequests.length, + trpcDuplicateRequestCount: countDuplicateTrpcPaths(trpcRequests), + pageErrors, + }; + const errors = validateBrowserMeasurement({ + responseStatus: measurement.responseStatus, + domContentLoadedMs, + boardHydratedMs: boardReadyMs, + firstInteractionMs, + error: pageErrors.length > 0 ? pageErrors.join("; ") : undefined, + }); + if (errors.length > 0) throw new Error(errors.map((error) => `${boardUrl}: ${error}`).join("; ")); + + return measurement; +}; + +try { + for (let iteration = 0; iteration < iterations; iteration++) { + const context = await browser.newContext({ storageState: storageStatePath }); + const coldPage = await context.newPage(); + cold.push(await measureBoardAsync(coldPage)); + await coldPage.close(); + const warmPage = await context.newPage(); + warm.push(await measureBoardAsync(warmPage)); + await context.close(); + } +} finally { + await browser.close(); +} + +const summarizeMeasurements = (measurements: BoardMeasurement[]) => ({ + boardReady: summarize(measurements.map(({ boardReadyMs }) => boardReadyMs)), + domContentLoaded: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)), + firstInteraction: summarize(measurements.map(({ firstInteractionMs }) => firstInteractionMs)), + trpcRequests: summarize(measurements.map(({ trpcRequestCount }) => trpcRequestCount)), + duplicateTrpcRequests: summarize(measurements.map(({ trpcDuplicateRequestCount }) => trpcDuplicateRequestCount)), +}); + +const result = { + kind: "authenticated-board", + claimEligible: true, + boardUrl, + createdAt: new Date().toISOString(), + environment: { + architecture: process.arch, + cpus: os.cpus().map(({ model }) => model), + node: process.version, + platform: `${process.platform} ${os.release()}`, + sha: process.env.DEV_BENCHMARK_SHA ?? null, + }, + iterations, + measurements: { cold, warm }, + summary: { + cold: summarizeMeasurements(cold), + warm: summarizeMeasurements(warm), + }, +}; +const outputPath = path.join(outputDirectory, `board-${Date.now()}.json`); +await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`); +console.log(JSON.stringify({ outputPath, summary: result.summary }, null, 2));