From 4766653979ddd3a0cd8f9502def025b39b5958a5 Mon Sep 17 00:00:00 2001 From: Thomas Camlong Date: Sun, 9 Aug 2026 21:38:49 +0200 Subject: [PATCH] chore(benchmarks): record the server require graph, and use it to attribute boot-time code A flat module cache says 24 MiB of source is resident before the first request, but not what pulled any of it in. Recording parent -> child as the CommonJS loader resolves it answers that, and the answer redirects the remaining 'ship less server JS' work: - Next loads all 74 app route bundles at startup, not lazily. - One 16.10 MiB / 460-file subtree is reached by the tRPC/API routes AND by instrumentation, so instrumentation's exclusive cost is 0 bytes - deferring it would free nothing, because Next preloads /api/[...trpc] anyway. - Only 1.10 MiB is common to every route, so this is the API surface pulling in every integration, not a universal barrel. --- PHASE2-FINDINGS.md | 31 ++++++ scripts/benchmarks/analyze-require-graph.mjs | 109 +++++++++++++++++++ scripts/benchmarks/memory-probe.cjs | 31 +++++- 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 scripts/benchmarks/analyze-require-graph.mjs diff --git a/PHASE2-FINDINGS.md b/PHASE2-FINDINGS.md index 4bf2a6626..65b46020b 100644 --- a/PHASE2-FINDINGS.md +++ b/PHASE2-FINDINGS.md @@ -588,6 +588,37 @@ A prediction worth recording, because it confirms the mechanism rather than just outcome: at `pageSize=1000` the largest allocation measured 0.7 MiB, and at 5,000 it measured 2.4 MiB — 5× the page size, 3.4× the bytes, exactly as a per-page bound implies. +## Why 24 MiB of code is resident before the first request + +The module cache is a flat list with no parentage, so "what pulled this in" needed the +require graph recorded as the loader resolves it (`HOMARR_PROBE_TRACK_REQUIRES=1`, analysed +by `scripts/benchmarks/analyze-require-graph.mjs`). At boot idle, having served only +`/api/health/ready`: **2,004 edges over 1,121 nodes, 23.97 MiB of source**. + +| importer | subtree | +| --- | --- | +| `apps/nextjs/server.js` (everything) | 23.83 MiB / 1,087 files | +| `next/dist/server/load-components.js` | 21.75 MiB / 796 files | +| **`[root-of-the-server]__0g06wx7._.js`** | **16.10 MiB / 460 files** | +| next's own framework code, exclusively | 2.16 MiB / 315 files | + +Two things follow, and the second is the one that matters: + +1. **Next loads all 74 app route bundles at startup**, not on first request. That is + framework behaviour, not something this codebase sets. +2. **One subtree of 16.10 MiB / 460 files is reached by the tRPC/API routes *and* by + `instrumentation`** — the cron tasks and websocket server. Exclusive attribution for + instrumentation is therefore **0 bytes**: everything it touches, an API route touches + too. Only 1.10 MiB / 119 files is common to *every* route, so this is not one universal + barrel — it is specifically the API surface pulling in every integration and widget + request handler. + +That kills the obvious fix. Deferring `instrumentation` would free nothing, because Next +preloads `/api/[...trpc]` at boot and that route reaches the same graph. Cutting this +requires the tRPC router to stop referencing every integration eagerly — the same lever the +integrations barrel diet started, with roughly 16 MiB of source still behind it. Recorded +as measured and understood, not attempted here. + ## Measured but not pursued diff --git a/scripts/benchmarks/analyze-require-graph.mjs b/scripts/benchmarks/analyze-require-graph.mjs new file mode 100644 index 000000000..c1f6ad328 --- /dev/null +++ b/scripts/benchmarks/analyze-require-graph.mjs @@ -0,0 +1,109 @@ +// Attributes loaded server code to whatever pulled it in. +// +// node scripts/benchmarks/analyze-require-graph.mjs +// +// `graph.json` is the probe's /requires output (parent → child edges) and `sizes.json` is a +// map of file → byte size. Together they answer the question a flat module list cannot: +// 24 MiB of source is resident before the first request, and this says which entry point is +// responsible for it — boot-time instrumentation, or code a route actually needed. +// +// Bytes are attributed by *exclusive reachability*: a file counts against an entry only if +// every path to it goes through that entry. Anything reachable from more than one entry is +// reported as shared, because removing either entry would not free it. +import { readFileSync } from "node:fs"; + +const [, , graphPath, sizesPath] = process.argv; +if (!graphPath || !sizesPath) throw new Error("usage: analyze-require-graph.mjs "); + +const edges = JSON.parse(readFileSync(graphPath, "utf8")); +const sizes = JSON.parse(readFileSync(sizesPath, "utf8")); + +const children = new Map(); +const allNodes = new Set(); +for (const [parent, child] of edges) { + allNodes.add(parent); + allNodes.add(child); + const list = children.get(parent) ?? []; + list.push(child); + children.set(parent, list); +} + +const sizeOf = (file) => sizes[file] ?? 0; +const MiB = (bytes) => (bytes / 1048576).toFixed(2); + +const reachableFrom = (roots) => { + const seen = new Set(); + const stack = [...roots]; + while (stack.length) { + const node = stack.pop(); + if (seen.has(node)) continue; + seen.add(node); + for (const child of children.get(node) ?? []) if (!seen.has(child)) stack.push(child); + } + return seen; +}; + +/** + * Boot-time entries. Next runs `instrumentation` before serving anything, and Homarr uses it + * to register cron tasks and the websocket server — the two subsystems that can drag the + * whole integration and widget graph in before a single request arrives. + */ +const classify = (file) => { + if (/instrumentation/.test(file)) return "instrumentation (cron tasks, websocket)"; + if (/\/\.next\/server\/app\//.test(file)) return "route bundles"; + if (/next\/dist/.test(file)) return "next framework"; + return null; +}; + +const entries = new Map(); +for (const node of allNodes) { + const kind = classify(node); + if (!kind) continue; + const list = entries.get(kind) ?? []; + list.push(node); + entries.set(kind, list); +} + +console.log(`edges: ${edges.length.toLocaleString()} nodes: ${allNodes.size.toLocaleString()}`); +const totalBytes = [...allNodes].reduce((sum, node) => sum + sizeOf(node), 0); +console.log(`total loaded source in the graph: ${MiB(totalBytes)} MiB\n`); + +const reach = new Map(); +for (const [kind, roots] of entries) reach.set(kind, reachableFrom(roots)); + +console.log("=== exclusive attribution (bytes only this entry can reach) ==="); +for (const [kind, reachable] of reach) { + let exclusive = 0; + let exclusiveCount = 0; + for (const node of reachable) { + let onlyHere = true; + for (const [otherKind, otherReachable] of reach) { + if (otherKind !== kind && otherReachable.has(node)) { + onlyHere = false; + break; + } + } + if (onlyHere) { + exclusive += sizeOf(node); + exclusiveCount++; + } + } + console.log( + ` ${MiB(exclusive).padStart(8)} MiB ${String(exclusiveCount).padStart(5)} files ${kind} (reaches ${reachable.size} nodes total)`, + ); +} + +// The heaviest single importers: a parent that alone drags in several MiB is the actionable +// unit, because that is the import a developer can make lazy. +console.log(`\n=== heaviest direct importers (own subtree, may overlap) ===`); +const subtreeSize = new Map(); +for (const parent of children.keys()) { + const reachable = reachableFrom([parent]); + let bytes = 0; + for (const node of reachable) bytes += sizeOf(node); + subtreeSize.set(parent, { bytes, count: reachable.size }); +} +const short = (file) => file.replace(/^.*\/(\.next|node_modules)\//, "$1/"); +for (const [file, info] of [...subtreeSize.entries()].sort((a, b) => b[1].bytes - a[1].bytes).slice(0, 18)) { + console.log(` ${MiB(info.bytes).padStart(8)} MiB ${String(info.count).padStart(5)} files ${short(file).slice(0, 90)}`); +} diff --git a/scripts/benchmarks/memory-probe.cjs b/scripts/benchmarks/memory-probe.cjs index e58f12670..772360097 100644 --- a/scripts/benchmarks/memory-probe.cjs +++ b/scripts/benchmarks/memory-probe.cjs @@ -105,6 +105,33 @@ globalThis.__homarrProbe = { moduleLoadList: () => process.moduleLoadList.slice(), }; +/** + * Opt-in recording of the CommonJS require graph. + * + * Knowing that 24 MiB of source is resident at boot does not say *why* any of it is there. + * The module cache is a flat list with no parentage, so the only way to answer "what pulled + * this chunk in" is to record parent → child as the loader resolves it. The preload runs + * before Next, so the graph is complete from the entry point down. + * + * Enable with HOMARR_PROBE_TRACK_REQUIRES=1. + */ +if (process.env.HOMARR_PROBE_TRACK_REQUIRES === "1") { + const edges = new Set(); + const originalLoad = Module._load; + Module._load = function (request, parent, isMain) { + const result = originalLoad.call(this, request, parent, isMain); + try { + const resolved = Module._resolveFilename(request, parent, isMain); + // Builtins resolve to their bare name and carry no file cost, so they only add noise. + if (resolved.includes("/")) edges.add(`${parent ? parent.filename : "(entry)"}\t${resolved}`); + } catch { + // An unresolvable request is the caller's problem, not the probe's. + } + return result; + }; + globalThis.__homarrProbe.requireGraph = () => [...edges].map((edge) => edge.split("\t")); +} + /** * Opt-in tracking of large Buffer allocations, by allocation site. * @@ -269,7 +296,9 @@ if (probePort > 0) { ? JSON.stringify(probe.buffers ? probe.buffers() : { disabled: true }) : request.url === "/fetches" ? JSON.stringify(probe.fetches ? probe.fetches() : { disabled: true }) - : JSON.stringify({ + : request.url === "/requires" + ? JSON.stringify(probe.requireGraph ? probe.requireGraph() : { disabled: true }) + : JSON.stringify({ usage: probe.usage(), spaces: probe.spaces(), heap: probe.heap(),