chore(benchmarks): record the server require graph, and use it to attribute boot-time code
A flat module cache says 24 MiB of source is resident before the first request, but not
what pulled any of it in. Recording parent -> child as the CommonJS loader resolves it
answers that, and the answer redirects the remaining 'ship less server JS' work:
- Next loads all 74 app route bundles at startup, not lazily.
- One 16.10 MiB / 460-file subtree is reached by the tRPC/API routes AND by
instrumentation, so instrumentation's exclusive cost is 0 bytes - deferring it would
free nothing, because Next preloads /api/[...trpc] anyway.
- Only 1.10 MiB is common to every route, so this is the API surface pulling in every
integration, not a universal barrel.
This commit is contained in:
3 files changed
+170
-1
No files matched your search
@@ -588,6 +588,37 @@ A prediction worth recording, because it confirms the mechanism rather than just
|
|||||||
outcome: at `pageSize=1000` the largest allocation measured 0.7 MiB, and at 5,000 it
|
outcome: at `pageSize=1000` the largest allocation measured 0.7 MiB, and at 5,000 it
|
||||||
measured 2.4 MiB — 5× the page size, 3.4× the bytes, exactly as a per-page bound implies.
|
measured 2.4 MiB — 5× the page size, 3.4× the bytes, exactly as a per-page bound implies.
|
||||||
|
|
||||||
|
## Why 24 MiB of code is resident before the first request
|
||||||
|
|
||||||
|
The module cache is a flat list with no parentage, so "what pulled this in" needed the
|
||||||
|
require graph recorded as the loader resolves it (`HOMARR_PROBE_TRACK_REQUIRES=1`, analysed
|
||||||
|
by `scripts/benchmarks/analyze-require-graph.mjs`). At boot idle, having served only
|
||||||
|
`/api/health/ready`: **2,004 edges over 1,121 nodes, 23.97 MiB of source**.
|
||||||
|
|
||||||
|
| importer | subtree |
|
||||||
|
| --- | --- |
|
||||||
|
| `apps/nextjs/server.js` (everything) | 23.83 MiB / 1,087 files |
|
||||||
|
| `next/dist/server/load-components.js` | 21.75 MiB / 796 files |
|
||||||
|
| **`[root-of-the-server]__0g06wx7._.js`** | **16.10 MiB / 460 files** |
|
||||||
|
| next's own framework code, exclusively | 2.16 MiB / 315 files |
|
||||||
|
|
||||||
|
Two things follow, and the second is the one that matters:
|
||||||
|
|
||||||
|
1. **Next loads all 74 app route bundles at startup**, not on first request. That is
|
||||||
|
framework behaviour, not something this codebase sets.
|
||||||
|
2. **One subtree of 16.10 MiB / 460 files is reached by the tRPC/API routes *and* by
|
||||||
|
`instrumentation`** — the cron tasks and websocket server. Exclusive attribution for
|
||||||
|
instrumentation is therefore **0 bytes**: everything it touches, an API route touches
|
||||||
|
too. Only 1.10 MiB / 119 files is common to *every* route, so this is not one universal
|
||||||
|
barrel — it is specifically the API surface pulling in every integration and widget
|
||||||
|
request handler.
|
||||||
|
|
||||||
|
That kills the obvious fix. Deferring `instrumentation` would free nothing, because Next
|
||||||
|
preloads `/api/[...trpc]` at boot and that route reaches the same graph. Cutting this
|
||||||
|
requires the tRPC router to stop referencing every integration eagerly — the same lever the
|
||||||
|
integrations barrel diet started, with roughly 16 MiB of source still behind it. Recorded
|
||||||
|
as measured and understood, not attempted here.
|
||||||
|
|
||||||
## Measured but not pursued
|
## Measured but not pursued
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
// Attributes loaded server code to whatever pulled it in.
|
||||||
|
//
|
||||||
|
// node scripts/benchmarks/analyze-require-graph.mjs <graph.json> <sizes.json>
|
||||||
|
//
|
||||||
|
// `graph.json` is the probe's /requires output (parent → child edges) and `sizes.json` is a
|
||||||
|
// map of file → byte size. Together they answer the question a flat module list cannot:
|
||||||
|
// 24 MiB of source is resident before the first request, and this says which entry point is
|
||||||
|
// responsible for it — boot-time instrumentation, or code a route actually needed.
|
||||||
|
//
|
||||||
|
// Bytes are attributed by *exclusive reachability*: a file counts against an entry only if
|
||||||
|
// every path to it goes through that entry. Anything reachable from more than one entry is
|
||||||
|
// reported as shared, because removing either entry would not free it.
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
|
||||||
|
const [, , graphPath, sizesPath] = process.argv;
|
||||||
|
if (!graphPath || !sizesPath) throw new Error("usage: analyze-require-graph.mjs <graph.json> <sizes.json>");
|
||||||
|
|
||||||
|
const edges = JSON.parse(readFileSync(graphPath, "utf8"));
|
||||||
|
const sizes = JSON.parse(readFileSync(sizesPath, "utf8"));
|
||||||
|
|
||||||
|
const children = new Map();
|
||||||
|
const allNodes = new Set();
|
||||||
|
for (const [parent, child] of edges) {
|
||||||
|
allNodes.add(parent);
|
||||||
|
allNodes.add(child);
|
||||||
|
const list = children.get(parent) ?? [];
|
||||||
|
list.push(child);
|
||||||
|
children.set(parent, list);
|
||||||
|
}
|
||||||
|
|
||||||
|
const sizeOf = (file) => sizes[file] ?? 0;
|
||||||
|
const MiB = (bytes) => (bytes / 1048576).toFixed(2);
|
||||||
|
|
||||||
|
const reachableFrom = (roots) => {
|
||||||
|
const seen = new Set();
|
||||||
|
const stack = [...roots];
|
||||||
|
while (stack.length) {
|
||||||
|
const node = stack.pop();
|
||||||
|
if (seen.has(node)) continue;
|
||||||
|
seen.add(node);
|
||||||
|
for (const child of children.get(node) ?? []) if (!seen.has(child)) stack.push(child);
|
||||||
|
}
|
||||||
|
return seen;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Boot-time entries. Next runs `instrumentation` before serving anything, and Homarr uses it
|
||||||
|
* to register cron tasks and the websocket server — the two subsystems that can drag the
|
||||||
|
* whole integration and widget graph in before a single request arrives.
|
||||||
|
*/
|
||||||
|
const classify = (file) => {
|
||||||
|
if (/instrumentation/.test(file)) return "instrumentation (cron tasks, websocket)";
|
||||||
|
if (/\/\.next\/server\/app\//.test(file)) return "route bundles";
|
||||||
|
if (/next\/dist/.test(file)) return "next framework";
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const entries = new Map();
|
||||||
|
for (const node of allNodes) {
|
||||||
|
const kind = classify(node);
|
||||||
|
if (!kind) continue;
|
||||||
|
const list = entries.get(kind) ?? [];
|
||||||
|
list.push(node);
|
||||||
|
entries.set(kind, list);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`edges: ${edges.length.toLocaleString()} nodes: ${allNodes.size.toLocaleString()}`);
|
||||||
|
const totalBytes = [...allNodes].reduce((sum, node) => sum + sizeOf(node), 0);
|
||||||
|
console.log(`total loaded source in the graph: ${MiB(totalBytes)} MiB\n`);
|
||||||
|
|
||||||
|
const reach = new Map();
|
||||||
|
for (const [kind, roots] of entries) reach.set(kind, reachableFrom(roots));
|
||||||
|
|
||||||
|
console.log("=== exclusive attribution (bytes only this entry can reach) ===");
|
||||||
|
for (const [kind, reachable] of reach) {
|
||||||
|
let exclusive = 0;
|
||||||
|
let exclusiveCount = 0;
|
||||||
|
for (const node of reachable) {
|
||||||
|
let onlyHere = true;
|
||||||
|
for (const [otherKind, otherReachable] of reach) {
|
||||||
|
if (otherKind !== kind && otherReachable.has(node)) {
|
||||||
|
onlyHere = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (onlyHere) {
|
||||||
|
exclusive += sizeOf(node);
|
||||||
|
exclusiveCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(
|
||||||
|
` ${MiB(exclusive).padStart(8)} MiB ${String(exclusiveCount).padStart(5)} files ${kind} (reaches ${reachable.size} nodes total)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The heaviest single importers: a parent that alone drags in several MiB is the actionable
|
||||||
|
// unit, because that is the import a developer can make lazy.
|
||||||
|
console.log(`\n=== heaviest direct importers (own subtree, may overlap) ===`);
|
||||||
|
const subtreeSize = new Map();
|
||||||
|
for (const parent of children.keys()) {
|
||||||
|
const reachable = reachableFrom([parent]);
|
||||||
|
let bytes = 0;
|
||||||
|
for (const node of reachable) bytes += sizeOf(node);
|
||||||
|
subtreeSize.set(parent, { bytes, count: reachable.size });
|
||||||
|
}
|
||||||
|
const short = (file) => file.replace(/^.*\/(\.next|node_modules)\//, "$1/");
|
||||||
|
for (const [file, info] of [...subtreeSize.entries()].sort((a, b) => b[1].bytes - a[1].bytes).slice(0, 18)) {
|
||||||
|
console.log(` ${MiB(info.bytes).padStart(8)} MiB ${String(info.count).padStart(5)} files ${short(file).slice(0, 90)}`);
|
||||||
|
}
|
||||||
@@ -105,6 +105,33 @@ globalThis.__homarrProbe = {
|
|||||||
moduleLoadList: () => process.moduleLoadList.slice(),
|
moduleLoadList: () => process.moduleLoadList.slice(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Opt-in recording of the CommonJS require graph.
|
||||||
|
*
|
||||||
|
* Knowing that 24 MiB of source is resident at boot does not say *why* any of it is there.
|
||||||
|
* The module cache is a flat list with no parentage, so the only way to answer "what pulled
|
||||||
|
* this chunk in" is to record parent → child as the loader resolves it. The preload runs
|
||||||
|
* before Next, so the graph is complete from the entry point down.
|
||||||
|
*
|
||||||
|
* Enable with HOMARR_PROBE_TRACK_REQUIRES=1.
|
||||||
|
*/
|
||||||
|
if (process.env.HOMARR_PROBE_TRACK_REQUIRES === "1") {
|
||||||
|
const edges = new Set();
|
||||||
|
const originalLoad = Module._load;
|
||||||
|
Module._load = function (request, parent, isMain) {
|
||||||
|
const result = originalLoad.call(this, request, parent, isMain);
|
||||||
|
try {
|
||||||
|
const resolved = Module._resolveFilename(request, parent, isMain);
|
||||||
|
// Builtins resolve to their bare name and carry no file cost, so they only add noise.
|
||||||
|
if (resolved.includes("/")) edges.add(`${parent ? parent.filename : "(entry)"}\t${resolved}`);
|
||||||
|
} catch {
|
||||||
|
// An unresolvable request is the caller's problem, not the probe's.
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
globalThis.__homarrProbe.requireGraph = () => [...edges].map((edge) => edge.split("\t"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Opt-in tracking of large Buffer allocations, by allocation site.
|
* Opt-in tracking of large Buffer allocations, by allocation site.
|
||||||
*
|
*
|
||||||
@@ -269,7 +296,9 @@ if (probePort > 0) {
|
|||||||
? JSON.stringify(probe.buffers ? probe.buffers() : { disabled: true })
|
? JSON.stringify(probe.buffers ? probe.buffers() : { disabled: true })
|
||||||
: request.url === "/fetches"
|
: request.url === "/fetches"
|
||||||
? JSON.stringify(probe.fetches ? probe.fetches() : { disabled: true })
|
? JSON.stringify(probe.fetches ? probe.fetches() : { disabled: true })
|
||||||
: JSON.stringify({
|
: request.url === "/requires"
|
||||||
|
? JSON.stringify(probe.requireGraph ? probe.requireGraph() : { disabled: true })
|
||||||
|
: JSON.stringify({
|
||||||
usage: probe.usage(),
|
usage: probe.usage(),
|
||||||
spaces: probe.spaces(),
|
spaces: probe.spaces(),
|
||||||
heap: probe.heap(),
|
heap: probe.heap(),
|
||||||
|
|||||||
Reference in new issue
Block a user