homarr/scripts/benchmarks/dev-board.mts

import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
import type { BrowserContext, Page } from "@playwright/test";
import { chromium } from "@playwright/test";

import { countDuplicateTrpcPaths, summarize, validateBrowserMeasurement } from "./dev-benchmark-lib.mts";

type BoardMeasurement = {
  boardMountedMs: number;
  domContentLoadedMs: number;
  postIdleWarmInteractionMs: number;
  widgetCount: number;
  widgetsReadyMs: number;
  pageErrors: string[];
  responseStatus: number | null;
  trpcDuplicateRequestCount: number;
  trpcRequestCount: number;
  webSocketCount: number;
  performance: {
    cls: number;
    decodedScriptBytes: number;
    jsHeapBytes: number | null;
    lcpMs: number | null;
    longTaskCount: number;
    longTaskTotalMs: number;
    resourceCount: number;
    transferBytes: number;
  };
};

const root = path.resolve(import.meta.dirname, "../..");
const argumentsSet = new Set(process.argv.slice(2));
const smokeOnly = argumentsSet.has("--smoke");
const iterations = Number(process.env.DEV_BENCHMARK_ITERATIONS ?? 10);
const spotlightIdleMs = Number(process.env.DEV_BENCHMARK_SPOTLIGHT_IDLE_MS ?? 2_500);
const outputDirectory = path.resolve(
  process.env.DEV_BENCHMARK_OUTPUT_DIR ?? path.join(root, "benchmark-results", "dev"),
);
const boardUrl = process.env.DEV_BENCHMARK_BOARD_URL;
const storageStatePath = process.env.DEV_BENCHMARK_STORAGE_STATE;
const readinessUrl = process.env.DEV_BENCHMARK_READINESS_URL ?? "http://127.0.0.1:3000/api/health/ready";
const boardSelector = "[data-homarr-dev-benchmark-board]";
const interactionSelector = '[data-homarr-dev-benchmark-interaction="search"]';
const spotlightSelector = "[data-homarr-dev-benchmark-spotlight]";
const spotlightFeedbackSelector =
  "[data-homarr-dev-benchmark-spotlight-feedback], [data-homarr-dev-benchmark-spotlight]";

if (!Number.isInteger(iterations) || iterations < 7) {
  throw new Error("DEV_BENCHMARK_ITERATIONS must be an integer of at least 7");
}
if (!Number.isFinite(spotlightIdleMs) || spotlightIdleMs < 0) {
  throw new Error("DEV_BENCHMARK_SPOTLIGHT_IDLE_MS must be non-negative");
}

await fs.mkdir(outputDirectory, { recursive: true });

if (smokeOnly) {
  const measurements: number[] = [];
  for (let iteration = 0; iteration < iterations; iteration++) {
    const startedAt = performance.now();
    const response = await fetch(readinessUrl);
    await response.arrayBuffer();
    if (!response.ok) throw new Error(`Readiness smoke received HTTP ${response.status}`);
    measurements.push(Math.round(performance.now() - startedAt));
  }

  const result = {
    kind: "readiness-smoke",
    claimEligible: false,
    reason: "Readiness checks do not exercise the browser or dashboard graph",
    url: readinessUrl,
    iterations,
    summary: summarize(measurements),
  };
  const outputPath = path.join(outputDirectory, `smoke-${Date.now()}.json`);
  await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
  console.log(JSON.stringify({ outputPath, result }, null, 2));
  process.exit(0);
}

if (!boardUrl || !storageStatePath) {
  throw new Error(
    "Authenticated mode requires DEV_BENCHMARK_BOARD_URL and DEV_BENCHMARK_STORAGE_STATE. " +
      "Use --smoke only for the non-claimable readiness check.",
  );
}

await fs.access(storageStatePath);

const installPerformanceObservers = () => {
  const state = { cls: 0, lcpMs: null as number | null, longTaskCount: 0, longTaskTotalMs: 0 };
  Object.defineProperty(window, "homarrBenchmarkVitals", { value: state, configurable: true });

  if (PerformanceObserver.supportedEntryTypes.includes("largest-contentful-paint")) {
    new PerformanceObserver((list) => {
      const latest = list.getEntries().at(-1);
      if (latest) state.lcpMs = latest.startTime;
    }).observe({ type: "largest-contentful-paint", buffered: true });
  }
  if (PerformanceObserver.supportedEntryTypes.includes("layout-shift")) {
    new PerformanceObserver((list) => {
      for (const entry of list.getEntries()) {
        const shift = entry as PerformanceEntry & { hadRecentInput?: boolean; value?: number };
        if (!shift.hadRecentInput) state.cls += shift.value ?? 0;
      }
    }).observe({ type: "layout-shift", buffered: true });
  }
  if (PerformanceObserver.supportedEntryTypes.includes("longtask")) {
    new PerformanceObserver((list) => {
      for (const entry of list.getEntries()) {
        state.longTaskCount += 1;
        state.longTaskTotalMs += entry.duration;
      }
    }).observe({ type: "longtask", buffered: true });
  }
};

const createContextAsync = async (): Promise<BrowserContext> => {
  const context = await browser.newContext({ storageState: storageStatePath });
  await context.addInitScript(installPerformanceObservers);
  return context;
};

const browser = await chromium.launch();
const freshContext: BoardMeasurement[] = [];
const warmPage: BoardMeasurement[] = [];
const coldInteractionSamples: Array<{ feedbackMs: number; readyMs: number }> = [];

const measureBoardAsync = async (page: Page) => {
  const trpcRequests: string[] = [];
  const pageErrors: string[] = [];
  let webSocketCount = 0;

  page.on("request", (request) => {
    if (request.url().includes("/api/trpc/")) trpcRequests.push(request.url());
  });
  page.on("websocket", () => {
    webSocketCount += 1;
  });
  page.on("pageerror", (error) => pageErrors.push(error.message));

  const startedAt = performance.now();
  const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
  const domContentLoadedMs = Math.round(performance.now() - startedAt);
  await page.locator(boardSelector).waitFor({ state: "visible" });
  const boardMountedMs = Math.round(performance.now() - startedAt);

  await page.waitForFunction(
    (selector) => {
      const board = document.querySelector(selector);
      if (!board) return false;
      const widgetCount = board.querySelectorAll('[data-type="item"]').length;
      return widgetCount > 0 && board.querySelectorAll("[data-homarr-widget-ready]").length === widgetCount;
    },
    boardSelector,
    { timeout: 30_000 },
  );
  const widgetsReadyMs = Math.round(performance.now() - startedAt);
  const widgetCount = await page.locator(`${boardSelector} [data-type="item"]`).count();

  await page.waitForTimeout(spotlightIdleMs);
  const warmInteractionStartedAt = performance.now();
  await page.locator(`${interactionSelector}:visible`).click();
  await page.locator(spotlightSelector).waitFor({ state: "visible" });
  const postIdleWarmInteractionMs = Math.round(performance.now() - warmInteractionStartedAt);
  await page.keyboard.press("Escape");
  await page.locator(spotlightSelector).waitFor({ state: "hidden" });

  const browserPerformance = await page.evaluate(() => {
    const resources = performance.getEntriesByType("resource") as PerformanceResourceTiming[];
    const vitals = (
      window as Window & {
        homarrBenchmarkVitals?: {
          cls: number;
          lcpMs: number | null;
          longTaskCount: number;
          longTaskTotalMs: number;
        };
      }
    ).homarrBenchmarkVitals;
    const memory = performance as Performance & { memory?: { usedJSHeapSize: number } };
    return {
      cls: Number((vitals?.cls ?? 0).toFixed(4)),
      decodedScriptBytes: resources
        .filter((resource) => resource.initiatorType === "script")
        .reduce((total, resource) => total + resource.decodedBodySize, 0),
      jsHeapBytes: memory.memory?.usedJSHeapSize ?? null,
      lcpMs: vitals?.lcpMs === null || vitals?.lcpMs === undefined ? null : Math.round(vitals.lcpMs),
      longTaskCount: vitals?.longTaskCount ?? 0,
      longTaskTotalMs: Math.round(vitals?.longTaskTotalMs ?? 0),
      resourceCount: resources.length,
      transferBytes: resources.reduce((total, resource) => total + resource.transferSize, 0),
    };
  });

  const measurement: BoardMeasurement = {
    responseStatus: response?.status() ?? null,
    domContentLoadedMs,
    boardMountedMs,
    widgetsReadyMs,
    widgetCount,
    postIdleWarmInteractionMs,
    trpcRequestCount: trpcRequests.length,
    trpcDuplicateRequestCount: countDuplicateTrpcPaths(trpcRequests),
    webSocketCount,
    performance: browserPerformance,
    pageErrors,
  };
  const errors = validateBrowserMeasurement({
    responseStatus: measurement.responseStatus,
    domContentLoadedMs,
    boardHydratedMs: boardMountedMs,
    widgetsReadyMs,
    widgetCount,
    postIdleWarmInteractionMs,
    error: pageErrors.length > 0 ? pageErrors.join("; ") : undefined,
  });
  if (errors.length > 0) throw new Error(errors.map((error) => `${boardUrl}: ${error}`).join("; "));

  return measurement;
};

const measureColdInteractionAsync = async (page: Page) => {
  const pageErrors: string[] = [];
  page.on("pageerror", (error) => pageErrors.push(error.message));

  const response = await page.goto(boardUrl, { waitUntil: "domcontentloaded" });
  if (!response?.ok()) {
    throw new Error(`Cold-interaction board returned HTTP ${String(response?.status() ?? "none")}`);
  }
  await page.locator(boardSelector).waitFor({ state: "visible" });
  const interactionStartedAt = performance.now();
  await page.locator(`${interactionSelector}:visible`).click();
  await page.locator(spotlightFeedbackSelector).waitFor({ state: "visible" });
  const feedbackMs = Math.round(performance.now() - interactionStartedAt);
  await page.locator(spotlightSelector).waitFor({ state: "visible" });
  const readyMs = Math.round(performance.now() - interactionStartedAt);
  if (pageErrors.length > 0) throw new Error(`Cold-interaction page errors: ${pageErrors.join("; ")}`);
  return { feedbackMs, readyMs };
};

try {
  for (let iteration = 0; iteration < iterations; iteration++) {
    const context = await createContextAsync();
    const firstPage = await context.newPage();
    freshContext.push(await measureBoardAsync(firstPage));
    await firstPage.close();
    const secondPage = await context.newPage();
    warmPage.push(await measureBoardAsync(secondPage));
    await context.close();

    const coldContext = await createContextAsync();
    await coldContext.addInitScript(() => {
      // Measure the actual first-open path instead of racing the intentional
      // requestIdleCallback prefetch used during normal idle time.
      Object.defineProperty(window, "requestIdleCallback", { value: () => 0, configurable: true });
      Object.defineProperty(window, "cancelIdleCallback", { value: () => undefined, configurable: true });
    });
    const coldPage = await coldContext.newPage();
    coldInteractionSamples.push(await measureColdInteractionAsync(coldPage));
    await coldContext.close();
  }
} finally {
  await browser.close();
}

const summarizeMeasurements = (measurements: BoardMeasurement[]) => ({
  boardMountedMs: summarize(measurements.map(({ boardMountedMs }) => boardMountedMs)),
  widgetsReadyMs: summarize(measurements.map(({ widgetsReadyMs }) => widgetsReadyMs)),
  domContentLoadedMs: summarize(measurements.map(({ domContentLoadedMs }) => domContentLoadedMs)),
  postIdleWarmInteractionMs: summarize(measurements.map(({ postIdleWarmInteractionMs }) => postIdleWarmInteractionMs)),
  lcpMs: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.lcpMs)),
  cls: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.cls)),
  longTaskTotalMs: summarize(
    measurements.map(({ performance: browserPerformance }) => browserPerformance.longTaskTotalMs),
  ),
  decodedScriptBytes: summarize(
    measurements.map(({ performance: browserPerformance }) => browserPerformance.decodedScriptBytes),
  ),
  transferBytes: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.transferBytes)),
  jsHeapBytes: summarize(measurements.map(({ performance: browserPerformance }) => browserPerformance.jsHeapBytes)),
  trpcRequests: summarize(measurements.map(({ trpcRequestCount }) => trpcRequestCount)),
  duplicateTrpcRequests: summarize(measurements.map(({ trpcDuplicateRequestCount }) => trpcDuplicateRequestCount)),
});

const sha = process.env.DEV_BENCHMARK_SHA ?? null;
const image = process.env.DEV_BENCHMARK_IMAGE ?? null;
const runtime = process.env.DEV_BENCHMARK_RUNTIME ?? "development";
const claimEligible = false;
const result = {
  kind: "authenticated-board",
  claimEligible,
  claimIneligibleReasons: [
    "served revision and production runtime are self-reported, not independently verified",
    ...(runtime === "production" ? [] : ["runtime is not production"]),
    ...(sha ? [] : ["DEV_BENCHMARK_SHA is missing"]),
    ...(image ? [] : ["DEV_BENCHMARK_IMAGE is missing"]),
  ],
  boardUrl,
  createdAt: new Date().toISOString(),
  environment: {
    architecture: process.arch,
    cpus: os.cpus().map(({ model }) => model),
    image,
    node: process.version,
    platform: `${process.platform} ${os.release()}`,
    runtime,
    sha,
  },
  iterations,
  spotlightIdleMs,
  measurements: { freshContext, warmPage, coldInteractionSamples },
  summary: {
    coldInteractionFeedbackMs: summarize(coldInteractionSamples.map(({ feedbackMs }) => feedbackMs)),
    coldInteractionReadyMs: summarize(coldInteractionSamples.map(({ readyMs }) => readyMs)),
    freshContext: summarizeMeasurements(freshContext),
    warmPage: summarizeMeasurements(warmPage),
  },
};
const outputPath = path.join(outputDirectory, `board-${Date.now()}.json`);
await fs.writeFile(outputPath, `${JSON.stringify(result, null, 2)}\n`);
console.log(JSON.stringify({ outputPath, claimEligible, summary: result.summary }, null, 2));