perf(umami): stop buffering 100k raw events per widget refresh
Profiling the container end to end found one allocation site responsible for
most of the peak: a single 46.3 MiB JSON body, assembled from 3,946 socket
chunks, inside undici's fullyReadBody. That function holds the chunk list and
the joined copy at the same instant, so a body of size N costs about 2N.
The source was two Umami calls asking for pageSize=100000, with
getMultiEventTimeSeriesAsync running them for every tracked event name
concurrently via Promise.all. getEventNamesAsync was downloading every raw
event of the last 30 days purely to collect a handful of distinct name strings.
- getEventNamesAsync now reads the aggregated /metrics?type=event endpoint.
- getWebsiteEventTimeSeriesAsync pages 5,000 records at a time, folding each
page into the bucket map before requesting the next.
- getMultiEventTimeSeriesAsync runs series sequentially.
Instances that return aggregated {x,y} buckets still answer in one request, an
instance that accepts pageSize but ignores page is detected rather than
double-counted, and hitting the page ceiling logs instead of silently
undercounting.
Alternated runs, 2 per side, on a restored real backup:
peak container 498.9 -> 398.5 MiB (-20%)
peak node RSS 452.4 -> 353.3 MiB (-22%)
peak during 4 tabs 498.5 -> 356.3 MiB (-29%)
peak arrayBuffers 95.8 -> 6.6 MiB (-93%)
largest allocation 46.4 -> 2.4 MiB (-95%)
Costs +4% CPU from the extra round trips; board load latency unchanged.
Post-soak steady state and boot idle are unchanged within run-to-run spread.
Adds the attribution tooling this needed: attribute-memory.mts reconciles
cgroup -> processes -> address-space mappings -> V8 spaces -> loaded code,
memory-probe.cjs is an injectable diagnostics preload (never shipped) that
tracks buffer and fetch allocation sites, analyze-heap-dominators.mjs computes
real retained sizes, and compare-stress-runs.mjs reports alternated runs with
their spread.
Also corrects an earlier claim in PHASE2-FINDINGS.md: the reported "7.86 MiB of
duplicate chunk sources" was an artefact of V8 truncating string contents to
~1 KiB in heap snapshots. There are zero byte-identical duplicates among all
3,055 server JS files. The real doubling is character width - one codepoint
above U+00FF forces a whole source string to two bytes per character, which
costs 9.45 MiB across 80 loaded files.
Adds 6 tests; this integration had none.
This commit is contained in:
9 files changed
+1808
-96
No files matched your search
@@ -0,0 +1,245 @@
|
||||
// Computes real retained sizes from a .heapsnapshot and reports the biggest retainers.
|
||||
//
|
||||
// node --max-old-space-size=8192 scripts/benchmarks/analyze-heap-dominators.mjs <file>
|
||||
//
|
||||
// Shallow-size histograms answer "what kind of objects exist" and are misleading for
|
||||
// "what is costing memory": 4 MiB of `Object` spread over 95k instances says nothing about
|
||||
// which subsystem is holding them. Retained size — everything that would be freed if a node
|
||||
// were released — is the number that maps onto a cause, and it needs the dominator tree.
|
||||
//
|
||||
// Same algorithm Chrome DevTools uses: postorder the reachable graph, solve dominators by
|
||||
// iterated predecessor intersection, then accumulate self sizes up the dominator tree.
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const file = process.argv[2];
|
||||
if (!file) throw new Error("usage: analyze-heap-dominators.mjs <file.heapsnapshot> [topN]");
|
||||
const topN = Number(process.argv[3] ?? 40);
|
||||
|
||||
const snap = JSON.parse(readFileSync(file, "utf8"));
|
||||
const meta = snap.snapshot.meta;
|
||||
|
||||
const nf = meta.node_fields;
|
||||
const nStride = nf.length;
|
||||
const nType = nf.indexOf("type");
|
||||
const nName = nf.indexOf("name");
|
||||
const nSelf = nf.indexOf("self_size");
|
||||
const nEdgeCount = nf.indexOf("edge_count");
|
||||
const nodeTypes = meta.node_types[0];
|
||||
|
||||
const ef = meta.edge_fields;
|
||||
const eStride = ef.length;
|
||||
const eType = ef.indexOf("type");
|
||||
const eName = ef.indexOf("name_or_index");
|
||||
const eTo = ef.indexOf("to_node");
|
||||
const edgeTypes = meta.edge_types[0];
|
||||
|
||||
const nodes = snap.nodes;
|
||||
const edges = snap.edges;
|
||||
const strings = snap.strings;
|
||||
const N = snap.snapshot.node_count;
|
||||
const E = snap.snapshot.edge_count;
|
||||
|
||||
const nodeName = (o) => strings[nodes[o * nStride + nName]] ?? "";
|
||||
const nodeType = (o) => nodeTypes[nodes[o * nStride + nType]] ?? "?";
|
||||
const selfSize = (o) => nodes[o * nStride + nSelf];
|
||||
|
||||
console.log(`file: ${file}`);
|
||||
console.log(`nodes: ${N.toLocaleString()} edges: ${E.toLocaleString()}`);
|
||||
|
||||
// firstEdge[o] = index of node o's first edge.
|
||||
const firstEdge = new Uint32Array(N + 1);
|
||||
for (let o = 0, acc = 0; o < N; o++) {
|
||||
firstEdge[o] = acc;
|
||||
acc += nodes[o * nStride + nEdgeCount];
|
||||
}
|
||||
firstEdge[N] = E;
|
||||
|
||||
/**
|
||||
* Weak edges do not keep their target alive, so counting them would let a WeakMap or a
|
||||
* weak handle claim retention it does not actually have.
|
||||
*/
|
||||
const isWeak = (e) => edgeTypes[edges[e * eStride + eType]] === "weak";
|
||||
|
||||
// ---------------------------------------------------------------- postorder from the root
|
||||
const postOrder = new Int32Array(N).fill(-1); // node ordinal -> postorder index
|
||||
const postOrderNodes = new Int32Array(N);
|
||||
{
|
||||
const visited = new Uint8Array(N);
|
||||
// Explicit stack: the graph is far too deep for recursion.
|
||||
const stackNode = new Int32Array(N + 1);
|
||||
const stackEdge = new Uint32Array(N + 1);
|
||||
let post = 0;
|
||||
let top = 0;
|
||||
stackNode[0] = 0;
|
||||
stackEdge[0] = firstEdge[0];
|
||||
visited[0] = 1;
|
||||
while (top >= 0) {
|
||||
const node = stackNode[top];
|
||||
let e = stackEdge[top];
|
||||
let descended = false;
|
||||
while (e < firstEdge[node + 1]) {
|
||||
if (isWeak(e)) {
|
||||
e++;
|
||||
continue;
|
||||
}
|
||||
const child = edges[e * eStride + eTo] / nStride;
|
||||
e++;
|
||||
if (!visited[child]) {
|
||||
visited[child] = 1;
|
||||
stackEdge[top] = e;
|
||||
top++;
|
||||
stackNode[top] = child;
|
||||
stackEdge[top] = firstEdge[child];
|
||||
descended = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (descended) continue;
|
||||
stackEdge[top] = e;
|
||||
if (e >= firstEdge[node + 1]) {
|
||||
postOrder[node] = post;
|
||||
postOrderNodes[post] = node;
|
||||
post++;
|
||||
top--;
|
||||
}
|
||||
}
|
||||
console.log(`reachable from root: ${post.toLocaleString()} nodes`);
|
||||
// Unreachable nodes keep postOrder -1 and are excluded from the dominator solve.
|
||||
var reachableCount = post;
|
||||
}
|
||||
|
||||
// ------------------------------------------------- reverse edges, restricted to reachable
|
||||
const inDeg = new Uint32Array(N);
|
||||
for (let src = 0; src < N; src++) {
|
||||
if (postOrder[src] === -1) continue;
|
||||
for (let e = firstEdge[src]; e < firstEdge[src + 1]; e++) {
|
||||
if (isWeak(e)) continue;
|
||||
const dst = edges[e * eStride + eTo] / nStride;
|
||||
if (postOrder[dst] !== -1) inDeg[dst]++;
|
||||
}
|
||||
}
|
||||
const revStart = new Uint32Array(N + 1);
|
||||
{
|
||||
let acc = 0;
|
||||
for (let o = 0; o < N; o++) {
|
||||
revStart[o] = acc;
|
||||
acc += inDeg[o];
|
||||
}
|
||||
revStart[N] = acc;
|
||||
}
|
||||
const revTo = new Uint32Array(revStart[N]);
|
||||
{
|
||||
const cursor = Uint32Array.from(revStart.subarray(0, N));
|
||||
for (let src = 0; src < N; src++) {
|
||||
if (postOrder[src] === -1) continue;
|
||||
for (let e = firstEdge[src]; e < firstEdge[src + 1]; e++) {
|
||||
if (isWeak(e)) continue;
|
||||
const dst = edges[e * eStride + eTo] / nStride;
|
||||
if (postOrder[dst] !== -1) revTo[cursor[dst]++] = src;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- dominators
|
||||
const rootPost = postOrder[0];
|
||||
const dom = new Int32Array(reachableCount).fill(-1);
|
||||
dom[rootPost] = rootPost;
|
||||
{
|
||||
const intersect = (a, b) => {
|
||||
// Higher postorder index is closer to the root, so walking the smaller one upward
|
||||
// converges on the nearest common dominator.
|
||||
while (a !== b) {
|
||||
if (a < b) a = dom[a];
|
||||
else b = dom[b];
|
||||
if (a === -1 || b === -1) return -1;
|
||||
}
|
||||
return a;
|
||||
};
|
||||
let changed = true;
|
||||
let sweeps = 0;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
sweeps++;
|
||||
for (let i = reachableCount - 1; i >= 0; i--) {
|
||||
if (i === rootPost) continue;
|
||||
const node = postOrderNodes[i];
|
||||
let newDom = -1;
|
||||
for (let r = revStart[node]; r < revStart[node + 1]; r++) {
|
||||
const predPost = postOrder[revTo[r]];
|
||||
if (predPost === -1 || dom[predPost] === -1) continue;
|
||||
newDom = newDom === -1 ? predPost : intersect(newDom, predPost);
|
||||
if (newDom === -1) break;
|
||||
}
|
||||
if (newDom !== -1 && dom[i] !== newDom) {
|
||||
dom[i] = newDom;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`dominator solve converged in ${sweeps} sweeps`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- retained sizes
|
||||
const retained = new Float64Array(reachableCount);
|
||||
for (let i = 0; i < reachableCount; i++) retained[i] = selfSize(postOrderNodes[i]);
|
||||
for (let i = 0; i < reachableCount; i++) {
|
||||
if (i === rootPost) continue;
|
||||
const d = dom[i];
|
||||
if (d !== -1 && d !== i) retained[d] += retained[i];
|
||||
}
|
||||
|
||||
const MiB = (bytes) => (bytes / 1048576).toFixed(2);
|
||||
|
||||
// A node whose dominator is the root is a top-level owner: its retained size is memory that
|
||||
// only it holds. Those are the entries worth naming.
|
||||
const label = (o) => {
|
||||
const type = nodeType(o);
|
||||
const name = nodeName(o) || "(anonymous)";
|
||||
return `${type}:${name}`;
|
||||
};
|
||||
|
||||
const candidates = [];
|
||||
for (let i = 0; i < reachableCount; i++) {
|
||||
if (i === rootPost) continue;
|
||||
if (retained[i] < 262144) continue;
|
||||
candidates.push(i);
|
||||
}
|
||||
candidates.sort((a, b) => retained[b] - retained[a]);
|
||||
|
||||
console.log(`\n=== largest retainers (retained size = freed if this went away) ===`);
|
||||
console.log(`${"retained".padStart(10)} ${"self".padStart(9)} what`);
|
||||
const shown = [];
|
||||
for (const i of candidates) {
|
||||
const node = postOrderNodes[i];
|
||||
// Skip a node whose dominator is already listed: its bytes are counted in that ancestor,
|
||||
// so printing both would double-report the same memory.
|
||||
let ancestor = dom[i];
|
||||
let covered = false;
|
||||
for (let guard = 0; guard < 200 && ancestor !== -1 && ancestor !== rootPost; guard++) {
|
||||
if (shown.includes(ancestor)) {
|
||||
covered = true;
|
||||
break;
|
||||
}
|
||||
ancestor = dom[ancestor];
|
||||
}
|
||||
if (covered) continue;
|
||||
shown.push(i);
|
||||
console.log(`${MiB(retained[i]).padStart(10)} ${MiB(selfSize(node)).padStart(9)} ${label(node).slice(0, 96)}`);
|
||||
if (shown.length >= topN) break;
|
||||
}
|
||||
|
||||
// Grouping by constructor over retained size shows which *kind* of thing dominates, which
|
||||
// is what a shallow histogram cannot express.
|
||||
const byName = new Map();
|
||||
for (let i = 0; i < reachableCount; i++) {
|
||||
const node = postOrderNodes[i];
|
||||
const key = label(node);
|
||||
const acc = byName.get(key) ?? { retained: 0, self: 0, count: 0 };
|
||||
acc.self += selfSize(node);
|
||||
acc.count++;
|
||||
byName.set(key, acc);
|
||||
}
|
||||
console.log(`\n=== self size by constructor, top 15 (for cross-checking the above) ===`);
|
||||
for (const [name, acc] of [...byName.entries()].sort((a, b) => b[1].self - a[1].self).slice(0, 15)) {
|
||||
console.log(`${MiB(acc.self).padStart(10)} MiB ${String(acc.count).padStart(8)}x ${name.slice(0, 80)}`);
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Turns the per-stage probe captures of a stress run into one table, so growth can be
|
||||
// attributed to a specific part of the process rather than to "memory".
|
||||
//
|
||||
// node scripts/benchmarks/analyze-probe-stages.mjs .bench/stress/<label>
|
||||
//
|
||||
// The interesting column is never the total: it is which sub-total moved. Old-space
|
||||
// growth is retained application state; new-space is churn; external is Buffers; and
|
||||
// loaded-source growth means the server pulled in more code to serve a route. Those have
|
||||
// completely different fixes, and a single RSS number cannot tell them apart.
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const directory = process.argv[2];
|
||||
if (!directory) throw new Error("usage: analyze-probe-stages.mjs <stress output dir>");
|
||||
|
||||
const stageFiles = readdirSync(directory)
|
||||
.filter((name) => name.endsWith(".probe.json"))
|
||||
.sort();
|
||||
if (!stageFiles.length) throw new Error(`no *.probe.json in ${directory} (was HOMARR_PROBE_PORT set?)`);
|
||||
|
||||
/** Stage files are named so lexical order is chronological, except the numbered prefixes interleave. */
|
||||
const ORDER = ["01-boot", "02-post-restore", "03-post-login", "04-board", "nav-", "07-multi-tab", "05-after", "soak-", "06-final"];
|
||||
const rank = (name) => {
|
||||
const index = ORDER.findIndex((prefix) => name.startsWith(prefix));
|
||||
return index === -1 ? ORDER.length : index;
|
||||
};
|
||||
stageFiles.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
|
||||
|
||||
const MiB = (bytes) => (bytes / 1048576).toFixed(1);
|
||||
const space = (stage, name) => stage.spaces?.find((entry) => entry.space_name === name)?.space_used_size ?? 0;
|
||||
|
||||
const stages = [];
|
||||
for (const file of stageFiles) {
|
||||
const raw = JSON.parse(readFileSync(path.join(directory, file), "utf8"));
|
||||
if (raw.error) continue;
|
||||
stages.push({ name: file.replace(".probe.json", ""), ...raw });
|
||||
}
|
||||
|
||||
const columns = [
|
||||
["rss", (s) => s.usage.rss],
|
||||
["heapTotal", (s) => s.usage.heapTotal],
|
||||
["heapUsed", (s) => s.usage.heapUsed],
|
||||
["old", (s) => space(s, "old_space")],
|
||||
["large obj", (s) => space(s, "large_object_space")],
|
||||
["new", (s) => space(s, "new_space")],
|
||||
["code", (s) => space(s, "code_space") + space(s, "code_large_object_space")],
|
||||
["trusted", (s) => space(s, "trusted_space")],
|
||||
["external", (s) => s.usage.external],
|
||||
["arrayBuf", (s) => s.usage.arrayBuffers],
|
||||
["src loaded", (s) => s.modules?.totalBytes ?? 0],
|
||||
];
|
||||
|
||||
const header = ["stage".padEnd(16), ...columns.map(([name]) => name.padStart(10)), "files".padStart(7)].join(" ");
|
||||
console.log(`\n=== per-stage server memory, MiB (${stages.length} stages) ===`);
|
||||
console.log(header);
|
||||
console.log("-".repeat(header.length));
|
||||
for (const stage of stages) {
|
||||
const cells = columns.map(([, get]) => MiB(get(stage)).padStart(10));
|
||||
console.log([stage.name.padEnd(16), ...cells, String(stage.modules?.fileCount ?? 0).padStart(7)].join(" "));
|
||||
}
|
||||
|
||||
const first = stages[0];
|
||||
const last = stages.at(-1);
|
||||
const peak = (get) => Math.max(...stages.map(get));
|
||||
console.log(`\n=== growth from "${first.name}" to "${last.name}", and the worst stage ===`);
|
||||
console.log(`${"component".padEnd(34)} ${"start".padStart(9)} ${"end".padStart(9)} ${"delta".padStart(9)} ${"peak".padStart(9)}`);
|
||||
for (const [name, get] of columns) {
|
||||
const delta = get(last) - get(first);
|
||||
console.log(
|
||||
`${name.padEnd(34)} ${MiB(get(first)).padStart(9)} ${MiB(get(last)).padStart(9)} ${(delta >= 0 ? "+" : "") + MiB(delta)}`.padEnd(65) +
|
||||
MiB(peak(get)).padStart(9),
|
||||
);
|
||||
}
|
||||
|
||||
// Which code was pulled in later: a route that loads 5 MiB of chunks on first hit is a
|
||||
// different (and more fixable) problem than one that retains 5 MiB of data.
|
||||
const groupsOf = (stage) => new Map((stage.modules?.groups ?? []).map((group) => [group.name, group.bytes]));
|
||||
const firstGroups = groupsOf(first);
|
||||
const lastGroups = groupsOf(last);
|
||||
const names = new Set([...firstGroups.keys(), ...lastGroups.keys()]);
|
||||
const codeDeltas = [...names]
|
||||
.map((name) => ({ name, delta: (lastGroups.get(name) ?? 0) - (firstGroups.get(name) ?? 0), end: lastGroups.get(name) ?? 0 }))
|
||||
.filter((entry) => Math.abs(entry.delta) > 65536)
|
||||
.sort((a, b) => b.delta - a.delta);
|
||||
if (codeDeltas.length) {
|
||||
console.log(`\n=== code loaded after boot (source bytes, MiB) ===`);
|
||||
for (const entry of codeDeltas) {
|
||||
console.log(` ${(entry.delta >= 0 ? "+" : "") + MiB(entry.delta).padStart(7)} -> ${MiB(entry.end).padStart(7)} total ${entry.name}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
/**
|
||||
* Full RAM attribution for a running Homarr container, top-down:
|
||||
*
|
||||
* cgroup -> processes -> address-space mappings -> V8 spaces -> JS objects
|
||||
*
|
||||
* Every layer is measured, and each layer's total is reconciled against the one
|
||||
* above it so nothing hides in a rounding gap. The point is to answer "what takes
|
||||
* what" with numbers that add up, rather than quoting one aggregate (`memory.current`,
|
||||
* `heapUsed`) that silently omits most of the footprint.
|
||||
*
|
||||
* node --experimental-strip-types scripts/benchmarks/attribute-memory.mts <container>
|
||||
*
|
||||
* Notes on the layers, because each one measures something different:
|
||||
* - cgroup `anon` is the number that counts against a container limit and cannot be
|
||||
* reclaimed. `memory.current` also includes page cache, which swings by tens of MiB
|
||||
* for reasons unrelated to the app.
|
||||
* - `smaps` is the only place the split between JS heap, JIT code, native malloc and
|
||||
* mapped shared libraries is visible. V8's own numbers cover just its heap.
|
||||
* - `heapUsed` is live JS objects. `heapTotal` is what V8 has committed, including
|
||||
* free space it is holding for reuse. The gap between them is real RSS.
|
||||
*/
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const container = process.argv[2];
|
||||
if (!container) throw new Error("usage: attribute-memory.mts <container> [--snapshot <file>] [--label <name>]");
|
||||
const snapshotIndex = process.argv.indexOf("--snapshot");
|
||||
const snapshotPath = snapshotIndex === -1 ? undefined : process.argv[snapshotIndex + 1];
|
||||
const labelIndex = process.argv.indexOf("--label");
|
||||
const label = labelIndex === -1 ? "current" : (process.argv[labelIndex + 1] ?? "current");
|
||||
|
||||
const MiB = (bytes: number) => Math.round((bytes / 1048576) * 10) / 10;
|
||||
const pad = (value: string | number, width: number) => String(value).padStart(width);
|
||||
|
||||
const execIn = async (script: string) => {
|
||||
const { stdout } = await execFileAsync("docker", ["exec", container, "sh", "-c", script], {
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
return stdout;
|
||||
};
|
||||
|
||||
/**
|
||||
* Piping to stdin avoids the escaping problems of `node -e "..."` through two shells.
|
||||
* NODE_OPTIONS is cleared because the container may carry a `--require` preload meant only
|
||||
* for the long-lived server; inheriting it into every helper is wasted work at best.
|
||||
*/
|
||||
const execNodeIn = (script: string) =>
|
||||
new Promise<string>((resolve, reject) => {
|
||||
const proc = spawn("docker", ["exec", "-e", "NODE_OPTIONS=", "-i", container, "node"], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
proc.stdout.on("data", (chunk) => (stdout += chunk));
|
||||
proc.stderr.on("data", (chunk) => (stderr += chunk));
|
||||
proc.on("error", reject);
|
||||
proc.on("close", (code) => (code === 0 ? resolve(stdout) : reject(new Error(stderr.trim() || `node exited ${code}`))));
|
||||
proc.stdin.end(script);
|
||||
});
|
||||
|
||||
/** comm is truncated to 15 chars and Next retitles itself, so match by prefix. */
|
||||
const findServerPidAsync = async () => {
|
||||
const out = await execIn(
|
||||
`for p in /proc/[0-9]*; do n=$(cat "$p/comm" 2>/dev/null);` +
|
||||
` case "$n" in node*|next-server*|bun*) echo "$(basename "$p") $n";; esac; done`,
|
||||
);
|
||||
const line = out.trim().split("\n").filter(Boolean)[0];
|
||||
if (!line) throw new Error("no node/next-server/bun process found in the container");
|
||||
const [pid, comm] = line.split(/\s+/);
|
||||
return { pid: Number(pid), comm: comm ?? "?" };
|
||||
};
|
||||
|
||||
const { pid, comm } = await findServerPidAsync();
|
||||
|
||||
// ---------------------------------------------------------------- layer 1: cgroup
|
||||
const cgroupRaw = await execIn(
|
||||
`cat /sys/fs/cgroup/memory.current; echo ---; cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0; echo ---; cat /sys/fs/cgroup/memory.stat`,
|
||||
);
|
||||
const [currentRaw, peakRaw, statRaw] = cgroupRaw.split("---");
|
||||
const cgroupStat = new Map<string, number>();
|
||||
for (const line of (statRaw ?? "").trim().split("\n")) {
|
||||
const [key, value] = line.trim().split(/\s+/);
|
||||
if (key && value) cgroupStat.set(key, Number(value));
|
||||
}
|
||||
const cgroupCurrent = Number((currentRaw ?? "0").trim());
|
||||
const cgroupPeak = Number((peakRaw ?? "0").trim());
|
||||
|
||||
// ---------------------------------------------------------- layer 2: per-process RSS
|
||||
const perProcessRaw = await execIn(
|
||||
`for p in /proc/[0-9]*/status; do awk '/^Name:/{n=$2} /^VmRSS:/{print n" "$2}' "$p" 2>/dev/null; done`,
|
||||
);
|
||||
const processes = perProcessRaw
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name, kib] = line.trim().split(/\s+/);
|
||||
return { name: name ?? "?", bytes: Number(kib) * 1024 };
|
||||
})
|
||||
.sort((a, b) => b.bytes - a.bytes);
|
||||
|
||||
// ------------------------------------------------- layer 3: address space via smaps
|
||||
/**
|
||||
* Buckets every mapping of the server process. The categories are chosen so each maps
|
||||
* onto a distinct cause a developer can act on:
|
||||
* - `.so` / node binary text: the runtime and native addons, mostly shared and
|
||||
* file-backed, so cheap and not something app code controls.
|
||||
* - JIT code: anonymous executable pages, i.e. what V8 compiled at runtime. Grows
|
||||
* with how much JavaScript actually ran, not with how much shipped.
|
||||
* - large anonymous: V8's heap reservations and big malloc'd blocks.
|
||||
* - small anonymous: malloc arenas and thread stacks.
|
||||
*/
|
||||
const smapsRaw = await execIn(`cat /proc/${pid}/smaps`);
|
||||
type Mapping = { start: number; end: number; size: number; rss: number; anonymous: number; perms: string; path: string };
|
||||
const mappings: Mapping[] = [];
|
||||
{
|
||||
let current: Mapping | undefined;
|
||||
for (const line of smapsRaw.split("\n")) {
|
||||
const header = /^([0-9a-f]+)-([0-9a-f]+)\s+(\S{4})\s+\S+\s+\S+\s+\S+\s*(.*)$/.exec(line);
|
||||
if (header) {
|
||||
if (current) mappings.push(current);
|
||||
const start = Number.parseInt(header[1]!, 16);
|
||||
const end = Number.parseInt(header[2]!, 16);
|
||||
current = { start, end, size: end - start, rss: 0, anonymous: 0, perms: header[3]!, path: (header[4] ?? "").trim() };
|
||||
continue;
|
||||
}
|
||||
if (!current) continue;
|
||||
const rss = /^Rss:\s+(\d+) kB/.exec(line);
|
||||
if (rss) current.rss = Number(rss[1]) * 1024;
|
||||
const anon = /^Anonymous:\s+(\d+) kB/.exec(line);
|
||||
if (anon) current.anonymous = Number(anon[1]) * 1024;
|
||||
}
|
||||
if (current) mappings.push(current);
|
||||
}
|
||||
|
||||
/**
|
||||
* Anonymous mappings cannot be told apart by size alone: V8 carves its heap out of one
|
||||
* huge reservation in ~256 KiB pages, so /proc shows hundreds of small `rw-p` mappings
|
||||
* that look exactly like malloc arenas. Grouping address-adjacent anonymous mappings
|
||||
* back into the reservation they came from restores the distinction — a region reserving
|
||||
* gigabytes of address space is V8's cage, and committed RSS inside it is the JS heap.
|
||||
* Everything anonymous outside those regions is native: malloc, Buffers, thread stacks.
|
||||
*/
|
||||
const anonymousMappings = mappings.filter((mapping) => !mapping.path).sort((a, b) => a.start - b.start);
|
||||
type Region = { start: number; end: number; reserved: number; rss: number; exec: boolean; maps: number };
|
||||
const regions: Region[] = [];
|
||||
for (const mapping of anonymousMappings) {
|
||||
const last = regions.at(-1);
|
||||
if (last && mapping.start === last.end) {
|
||||
last.end = mapping.end;
|
||||
last.reserved += mapping.size;
|
||||
last.rss += mapping.rss;
|
||||
last.exec ||= mapping.perms.includes("x");
|
||||
last.maps++;
|
||||
continue;
|
||||
}
|
||||
regions.push({
|
||||
start: mapping.start,
|
||||
end: mapping.end,
|
||||
reserved: mapping.size,
|
||||
rss: mapping.rss,
|
||||
exec: mapping.perms.includes("x"),
|
||||
maps: 1,
|
||||
});
|
||||
}
|
||||
/** Reserving >=512 MiB of address space is only ever V8 claiming its cage up front. */
|
||||
const V8_RESERVATION_BYTES = 512 * 1048576;
|
||||
const v8Regions = regions.filter((region) => region.reserved >= V8_RESERVATION_BYTES);
|
||||
const inV8Region = (address: number) =>
|
||||
v8Regions.some((region) => address >= region.start && address < region.end);
|
||||
|
||||
/**
|
||||
* V8 allocates its heap in 256 KiB pages (`kRegularPageSize`), which is the single most
|
||||
* recognisable signature in the map: a 256 KiB anonymous `rw-p` mapping is a JS heap page
|
||||
* and nothing else in the process allocates at exactly that granularity in bulk. Thread
|
||||
* stacks are the other unmistakable shape — pthread reserves 8 MiB + a guard page, so they
|
||||
* show up as ~8196 KiB reservations that are almost entirely untouched.
|
||||
*/
|
||||
const V8_PAGE_BYTES = 262144;
|
||||
const THREAD_STACK_BYTES = 8196 * 1024;
|
||||
|
||||
const categorise = (mapping: Mapping) => {
|
||||
const { path, perms } = mapping;
|
||||
const executable = perms.includes("x");
|
||||
if (path === "[stack]") return "main thread stack";
|
||||
if (path === "[heap]") return "malloc brk heap";
|
||||
if (path.startsWith("[")) return `kernel ${path}`;
|
||||
if (path) {
|
||||
if (/\.node$/.test(path)) return `native addon ${path.split("/").pop()}`;
|
||||
if (/\/node$|\/bun$/.test(path)) return executable ? "node binary (code)" : "node binary (data)";
|
||||
if (/\.so/.test(path)) return executable ? "shared library (code)" : "shared library (data)";
|
||||
return `file-backed ${path.split("/").pop()}`;
|
||||
}
|
||||
if (executable) return "JIT code (anonymous executable)";
|
||||
if (mapping.size === V8_PAGE_BYTES) return "V8 heap pages (256 KiB regular pages)";
|
||||
if (mapping.size === THREAD_STACK_BYTES) return "thread stacks (8 MiB reserved each)";
|
||||
if (inV8Region(mapping.start)) return "V8 reserved range (code range / trusted space)";
|
||||
if (mapping.size >= 8 * 1048576) return "native: large anonymous mmap (>=8 MiB)";
|
||||
return "native: malloc arenas, Buffers, small V8 spaces";
|
||||
};
|
||||
|
||||
const byCategory = new Map<string, { rss: number; size: number; count: number }>();
|
||||
let smapsRssTotal = 0;
|
||||
let smapsAnonTotal = 0;
|
||||
for (const mapping of mappings) {
|
||||
smapsRssTotal += mapping.rss;
|
||||
smapsAnonTotal += mapping.anonymous;
|
||||
const key = categorise(mapping);
|
||||
const acc = byCategory.get(key) ?? { rss: 0, size: 0, count: 0 };
|
||||
acc.rss += mapping.rss;
|
||||
acc.size += mapping.size;
|
||||
acc.count++;
|
||||
byCategory.set(key, acc);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- layer 4: V8 internals
|
||||
/**
|
||||
* Preferred path: the preloaded probe's localhost endpoint. It needs no signal, no CDP
|
||||
* handshake and no GC pause, so it can be sampled at every stage of a stress run.
|
||||
*/
|
||||
const probeHttp = await execNodeIn(`
|
||||
fetch("http://127.0.0.1:${process.env.HOMARR_PROBE_PORT ?? 9333}/")
|
||||
.then((response) => response.text())
|
||||
.then((text) => console.log(text))
|
||||
.catch((error) => console.log(JSON.stringify({ probeUnavailable: String(error && error.message) })));
|
||||
`)
|
||||
.then((raw) => {
|
||||
try {
|
||||
return JSON.parse(raw.trim());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.catch(() => null);
|
||||
|
||||
const haveProbeHttp = probeHttp !== null && !probeHttp.probeUnavailable;
|
||||
/** The inspector is still needed for a heap snapshot, and as the fallback when no probe is loaded. */
|
||||
const needInspector = Boolean(snapshotPath) || !haveProbeHttp;
|
||||
let webSocketUrl: string | undefined;
|
||||
if (needInspector) {
|
||||
await execIn(`kill -USR1 ${pid}`).catch(() => undefined);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
const listRaw = await execNodeIn(`
|
||||
const http = require("node:http");
|
||||
http.get({ host: "127.0.0.1", port: 9229, path: "/json/list" }, (response) => {
|
||||
let body = "";
|
||||
response.on("data", (chunk) => (body += chunk));
|
||||
response.on("end", () => console.log(body));
|
||||
}).on("error", (error) => console.log("ERR " + error.message));
|
||||
`).catch((error: Error) => `ERR ${error.message}`);
|
||||
webSocketUrl = /"webSocketDebuggerUrl":\s*"([^"]+)"/.exec(listRaw)?.[1];
|
||||
}
|
||||
|
||||
const evaluateInServerAsync = async (expression: string) => {
|
||||
if (!webSocketUrl) return null;
|
||||
const out = await execNodeIn(`
|
||||
const ws = new WebSocket(${JSON.stringify(webSocketUrl)});
|
||||
let nextId = 0;
|
||||
const pending = new Map();
|
||||
const send = (method, params = {}) =>
|
||||
new Promise((resolve) => { const id = ++nextId; pending.set(id, resolve); ws.send(JSON.stringify({ id, method, params })); });
|
||||
ws.addEventListener("message", (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.id && pending.has(message.id)) { pending.get(message.id)(message.result); pending.delete(message.id); }
|
||||
});
|
||||
ws.addEventListener("error", () => { console.error("inspector websocket error"); process.exit(1); });
|
||||
ws.addEventListener("open", async () => {
|
||||
await send("Runtime.enable");
|
||||
const response = await send("Runtime.evaluate", { expression: ${JSON.stringify(expression)}, returnByValue: true, awaitPromise: true });
|
||||
if (response?.exceptionDetails) { console.error(JSON.stringify(response.exceptionDetails).slice(0, 800)); process.exit(1); }
|
||||
console.log(JSON.stringify(response?.result?.value ?? null));
|
||||
ws.close();
|
||||
process.exit(0);
|
||||
});
|
||||
`).catch((error: Error) => `null /* ${error.message} */`);
|
||||
try {
|
||||
return JSON.parse(out.trim());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Fallback when no probe is mounted. The inspected context has no `require`, so the only
|
||||
* route to the module system is `process.mainModule` — which the Next standalone entry
|
||||
* does not set, so per-space statistics are usually unavailable this way. Kept because it
|
||||
* still works on containers started by hand, and reports `process.memoryUsage()` regardless.
|
||||
*/
|
||||
const v8Detail = haveProbeHttp
|
||||
? { ...probeHttp, probe: true }
|
||||
: await evaluateInServerAsync(`(() => {
|
||||
const usage = process.memoryUsage();
|
||||
const out = { usage, spaces: null, heap: null, modules: null, largestModules: null, probe: false,
|
||||
versions: process.versions, execArgv: process.execArgv, uptime: process.uptime() };
|
||||
const probe = globalThis.__homarrProbe;
|
||||
try {
|
||||
if (probe) {
|
||||
out.probe = true;
|
||||
out.spaces = probe.spaces();
|
||||
out.heap = probe.heap();
|
||||
out.modules = probe.modules();
|
||||
out.largestModules = probe.largestModules(25);
|
||||
} else if (process.mainModule) {
|
||||
const req = process.mainModule.require.bind(process.mainModule);
|
||||
const v8 = req("v8");
|
||||
out.spaces = v8.getHeapSpaceStatistics();
|
||||
out.heap = v8.getHeapStatistics();
|
||||
}
|
||||
} catch (error) { out.error = String(error && error.message); }
|
||||
return out;
|
||||
})()`);
|
||||
|
||||
// --------------------------------------------------------------------- report
|
||||
const line = (name: string, bytes: number, of: number, note = "") =>
|
||||
`${pad(MiB(bytes).toFixed(1), 9)} MiB ${pad(((bytes / of) * 100).toFixed(1), 5)}% ${name}${note ? ` ${note}` : ""}`;
|
||||
|
||||
console.log(`\n############ RAM attribution: ${label} ############`);
|
||||
console.log(`container ${container} server pid ${pid} (${comm}) uptime ${Math.round(v8Detail?.uptime ?? 0)}s`);
|
||||
if (v8Detail?.execArgv?.length) console.log(`node flags: ${v8Detail.execArgv.join(" ")}`);
|
||||
|
||||
console.log(`\n=== LAYER 1: cgroup (what the container limit sees) ===`);
|
||||
const anon = cgroupStat.get("anon") ?? 0;
|
||||
console.log(line("anon — non-reclaimable, the real cost", anon, cgroupCurrent));
|
||||
console.log(line("file — page cache, reclaimable under pressure", cgroupStat.get("file") ?? 0, cgroupCurrent));
|
||||
for (const key of ["slab", "kernel_stack", "pagetables", "sock", "shmem"]) {
|
||||
const value = cgroupStat.get(key) ?? 0;
|
||||
if (value > 0) console.log(line(`${key} — kernel bookkeeping`, value, cgroupCurrent));
|
||||
}
|
||||
console.log(`${pad(MiB(cgroupCurrent).toFixed(1), 9)} MiB memory.current (sum of the above)`);
|
||||
console.log(`${pad(MiB(cgroupPeak).toFixed(1), 9)} MiB memory.peak (high-water since boot)`);
|
||||
|
||||
console.log(`\n=== LAYER 2: processes inside the container ===`);
|
||||
const processTotal = processes.reduce((sum, item) => sum + item.bytes, 0);
|
||||
for (const item of processes) console.log(line(item.name, item.bytes, processTotal));
|
||||
console.log(`${pad(MiB(processTotal).toFixed(1), 9)} MiB sum of process RSS`);
|
||||
|
||||
console.log(`\n=== LAYER 3: address space of pid ${pid} (smaps, ${mappings.length} mappings) ===`);
|
||||
for (const [name, acc] of [...byCategory.entries()].sort((a, b) => b[1].rss - a[1].rss)) {
|
||||
if (acc.rss < 65536) continue;
|
||||
console.log(line(name, acc.rss, smapsRssTotal, `(${acc.count} maps, ${MiB(acc.size).toFixed(0)} MiB reserved)`));
|
||||
}
|
||||
console.log(`${pad(MiB(smapsRssTotal).toFixed(1), 9)} MiB total RSS`);
|
||||
console.log(`${pad(MiB(smapsAnonTotal).toFixed(1), 9)} MiB of which anonymous (not backed by a file)`);
|
||||
console.log(
|
||||
`\n address-space reservations >=512 MiB (V8 claims its heap up front; only committed pages cost RAM):`,
|
||||
);
|
||||
for (const region of v8Regions.sort((a, b) => b.reserved - a.reserved)) {
|
||||
console.log(
|
||||
` ${pad((region.reserved / 1073741824).toFixed(1), 6)} GiB reserved -> ${pad(MiB(region.rss).toFixed(1), 7)} MiB committed (${region.maps} mappings)`,
|
||||
);
|
||||
}
|
||||
|
||||
if (v8Detail?.usage) {
|
||||
const usage = v8Detail.usage;
|
||||
console.log(`\n=== LAYER 4: inside V8 ===`);
|
||||
console.log(line("heapTotal — committed to the JS heap", usage.heapTotal, usage.rss));
|
||||
console.log(line(" heapUsed — live JS objects in it", usage.heapUsed, usage.rss));
|
||||
console.log(line(" heap free — committed but unused", usage.heapTotal - usage.heapUsed, usage.rss));
|
||||
console.log(line("external — Buffers/ArrayBuffers off-heap", usage.external, usage.rss));
|
||||
console.log(line(" arrayBuffers — of which ArrayBuffer", usage.arrayBuffers, usage.rss));
|
||||
const rest = usage.rss - usage.heapTotal - usage.external;
|
||||
console.log(line("rest — native malloc, JIT code, .so, stacks", rest, usage.rss));
|
||||
console.log(`${pad(MiB(usage.rss).toFixed(1), 9)} MiB process.memoryUsage().rss`);
|
||||
|
||||
if (v8Detail.spaces) {
|
||||
console.log(`\n--- V8 heap spaces (which part of the JS heap) ---`);
|
||||
const spaceTotal = v8Detail.spaces.reduce((sum: number, s: { space_size: number }) => sum + s.space_size, 0);
|
||||
for (const space of [...v8Detail.spaces].sort((a, b) => b.space_size - a.space_size)) {
|
||||
if (space.space_size < 65536) continue;
|
||||
console.log(
|
||||
line(space.space_name, space.space_size, spaceTotal, `(${MiB(space.space_used_size).toFixed(1)} MiB used)`),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (v8Detail.heap) {
|
||||
console.log(`\n--- V8 heap statistics ---`);
|
||||
console.log(` heap_size_limit ${MiB(v8Detail.heap.heap_size_limit).toFixed(0)} MiB (--max-old-space-size)`);
|
||||
console.log(` malloced_memory ${MiB(v8Detail.heap.malloced_memory).toFixed(1)} MiB`);
|
||||
console.log(` peak_malloced_memory ${MiB(v8Detail.heap.peak_malloced_memory).toFixed(1)} MiB`);
|
||||
console.log(` number_of_native_contexts ${v8Detail.heap.number_of_native_contexts}`);
|
||||
console.log(` number_of_detached_contexts ${v8Detail.heap.number_of_detached_contexts}`);
|
||||
}
|
||||
if (v8Detail.modules) {
|
||||
const modules = v8Detail.modules;
|
||||
console.log(
|
||||
`\n--- LAYER 5: loaded code, by owner (${modules.fileCount} files, ${MiB(modules.totalBytes).toFixed(1)} MiB of source on disk) ---`,
|
||||
);
|
||||
console.log(
|
||||
` V8 keeps a script's source string alive while any function in it is still lazily`,
|
||||
);
|
||||
console.log(` compilable, so this is a floor on what merely *loading* the code costs.`);
|
||||
for (const group of modules.groups) {
|
||||
if (group.bytes < 262144) continue;
|
||||
console.log(line(group.name, group.bytes, modules.totalBytes, `(${group.files} files)`));
|
||||
}
|
||||
console.log(`${pad(MiB(modules.totalBytes).toFixed(1), 9)} MiB total source loaded`);
|
||||
}
|
||||
if (v8Detail.largestModules) {
|
||||
console.log(`\n--- largest individual loaded files ---`);
|
||||
for (const item of v8Detail.largestModules.slice(0, 15)) {
|
||||
const short = item.file.replace(/^.*\/(\.next|node_modules)\//, "$1/");
|
||||
console.log(` ${pad(MiB(item.bytes).toFixed(2), 8)} MiB ${short}`);
|
||||
}
|
||||
}
|
||||
if (!v8Detail.probe) {
|
||||
console.log(
|
||||
`\n (no probe loaded: per-space and per-module attribution unavailable. Start the container with`,
|
||||
);
|
||||
console.log(` NODE_OPTIONS="--require /probe/memory-probe.cjs" and the probe dir mounted.)`);
|
||||
}
|
||||
if (v8Detail.error) console.log(` (v8 detail unavailable: ${v8Detail.error})`);
|
||||
} else {
|
||||
console.log(`\n=== LAYER 4: inside V8 === unavailable (inspector not reachable)`);
|
||||
}
|
||||
|
||||
if (snapshotPath) {
|
||||
const inContainerPath = "/tmp/attribute.heapsnapshot";
|
||||
console.log(`\n=== taking heap snapshot (after a forced GC, so it shows retained not garbage) ===`);
|
||||
console.log(
|
||||
(
|
||||
await execNodeIn(`
|
||||
const fs = require("node:fs");
|
||||
const out = fs.createWriteStream(${JSON.stringify(inContainerPath)});
|
||||
const ws = new WebSocket(${JSON.stringify(webSocketUrl)});
|
||||
ws.addEventListener("message", (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (message.method === "HeapProfiler.addHeapSnapshotChunk") out.write(message.params.chunk);
|
||||
else if (message.id === 3) out.end(() => { console.log("written"); process.exit(0); });
|
||||
});
|
||||
ws.addEventListener("error", () => { console.error("inspector websocket error"); process.exit(1); });
|
||||
ws.addEventListener("open", () => {
|
||||
ws.send(JSON.stringify({ id: 1, method: "HeapProfiler.enable" }));
|
||||
ws.send(JSON.stringify({ id: 2, method: "HeapProfiler.collectGarbage" }));
|
||||
ws.send(JSON.stringify({ id: 3, method: "HeapProfiler.takeHeapSnapshot", params: { reportProgress: false } }));
|
||||
});
|
||||
`)
|
||||
).trim(),
|
||||
);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const out = createWriteStream(snapshotPath);
|
||||
const proc = spawn("docker", ["exec", container, "cat", inContainerPath]);
|
||||
proc.stdout.pipe(out);
|
||||
proc.on("error", reject);
|
||||
proc.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`docker exec cat exited ${code}`))));
|
||||
});
|
||||
await execIn(`rm -f ${inContainerPath}`).catch(() => undefined);
|
||||
console.log(`saved ${snapshotPath}`);
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
// Compares alternated stress runs of two images and reports the medians side by side.
|
||||
//
|
||||
// node scripts/benchmarks/compare-stress-runs.mjs .bench/stress "u-before" "u-after"
|
||||
//
|
||||
// Runs are alternated (before, after, before, after) so machine drift lands on both sides,
|
||||
// and the median of each side is reported rather than a single run: the harness's own
|
||||
// spread between identical runs has been as wide as 50 MiB at peak, so a one-shot
|
||||
// comparison cannot distinguish a real change from noise.
|
||||
import { readFileSync, existsSync, readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
const root = process.argv[2] ?? ".bench/stress";
|
||||
const beforePrefix = process.argv[3] ?? "u-before";
|
||||
const afterPrefix = process.argv[4] ?? "u-after";
|
||||
|
||||
const loadSide = (prefix) =>
|
||||
readdirSync(root)
|
||||
.filter((name) => name.startsWith(prefix))
|
||||
.map((name) => path.join(root, name))
|
||||
.filter((dir) => existsSync(path.join(dir, "summary.json")))
|
||||
.map((dir) => ({ dir, summary: JSON.parse(readFileSync(path.join(dir, "summary.json"), "utf8")) }));
|
||||
|
||||
const median = (values) => {
|
||||
const sorted = values.filter((value) => typeof value === "number" && Number.isFinite(value)).sort((a, b) => a - b);
|
||||
if (!sorted.length) return null;
|
||||
const middle = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
||||
};
|
||||
|
||||
/** Peak arrayBuffers across a run's stages: the transient spike never shows in a summary total. */
|
||||
const peakArrayBuffers = (dir) => {
|
||||
let peak = 0;
|
||||
for (const name of readdirSync(dir).filter((file) => file.endsWith(".probe.json"))) {
|
||||
try {
|
||||
const stage = JSON.parse(readFileSync(path.join(dir, name), "utf8"));
|
||||
peak = Math.max(peak, (stage.usage?.arrayBuffers ?? 0) / 1048576);
|
||||
} catch {
|
||||
// A stage capture can be missing if the probe was not wired up; skip it.
|
||||
}
|
||||
}
|
||||
return peak || null;
|
||||
};
|
||||
|
||||
/** Largest single tracked Buffer allocation, which is what the umami paging change targets. */
|
||||
const largestAllocation = (dir) => {
|
||||
let largest = 0;
|
||||
for (const name of readdirSync(dir).filter((file) => file.endsWith(".buffers.json"))) {
|
||||
try {
|
||||
const capture = JSON.parse(readFileSync(path.join(dir, name), "utf8"));
|
||||
for (const site of capture.sites ?? []) largest = Math.max(largest, (site.max ?? 0) / 1048576);
|
||||
} catch {
|
||||
// Same: buffer tracking is opt-in.
|
||||
}
|
||||
}
|
||||
return largest || null;
|
||||
};
|
||||
|
||||
/** Stage samples are keyed by stage name, so the peak is the max across them. */
|
||||
const peakStage = (stages) => {
|
||||
const values = Object.values(stages ?? {}).filter((value) => typeof value === "number");
|
||||
return values.length ? Math.max(...values) : null;
|
||||
};
|
||||
|
||||
const METRICS = [
|
||||
["peak container MiB", (run) => peakStage(run.summary.containerMiB)],
|
||||
["peak node RSS MiB", (run) => peakStage(run.summary.nodeRssMiB)],
|
||||
["peak node anon MiB", (run) => peakStage(run.summary.nodeAnonMiB)],
|
||||
["multi-tab container MiB", (run) => run.summary.containerMiB?.["07-multi-tab"]],
|
||||
["final (post-soak) MiB", (run) => run.summary.containerMiB?.["06-final"]],
|
||||
["boot idle MiB", (run) => run.summary.containerMiB?.["01-boot-idle"]],
|
||||
["peak arrayBuffers MiB", (run) => peakArrayBuffers(run.dir)],
|
||||
["largest single alloc MiB", (run) => largestAllocation(run.dir)],
|
||||
["client JS heap MiB", (run) => run.summary.client?.jsHeapUsedMiB],
|
||||
["median board load ms", (run) => run.summary.latency?.medianBoardLoadMs],
|
||||
["CPU seconds", (run) => run.summary.cpuSeconds],
|
||||
];
|
||||
|
||||
const before = loadSide(beforePrefix);
|
||||
const after = loadSide(afterPrefix);
|
||||
console.log(`before "${beforePrefix}": ${before.length} runs after "${afterPrefix}": ${after.length} runs`);
|
||||
if (!before.length || !after.length) throw new Error("need at least one run on each side");
|
||||
|
||||
console.log(`\n${"metric".padEnd(26)} ${"before".padStart(10)} ${"after".padStart(10)} ${"change".padStart(12)}`);
|
||||
console.log("-".repeat(62));
|
||||
for (const [name, get] of METRICS) {
|
||||
const beforeValue = median(before.map(get));
|
||||
const afterValue = median(after.map(get));
|
||||
if (beforeValue === null || afterValue === null) continue;
|
||||
const delta = afterValue - beforeValue;
|
||||
const percent = beforeValue === 0 ? "" : ` (${delta >= 0 ? "+" : ""}${((delta / beforeValue) * 100).toFixed(0)}%)`;
|
||||
console.log(
|
||||
`${name.padEnd(26)} ${beforeValue.toFixed(1).padStart(10)} ${afterValue.toFixed(1).padStart(10)} ${((delta >= 0 ? "+" : "") + delta.toFixed(1) + percent).padStart(12)}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`\nper-run values (to judge spread):`);
|
||||
for (const [name, get] of METRICS) {
|
||||
const beforeValues = before.map(get).filter((value) => typeof value === "number");
|
||||
const afterValues = after.map(get).filter((value) => typeof value === "number");
|
||||
if (!beforeValues.length || !afterValues.length) continue;
|
||||
console.log(
|
||||
` ${name.padEnd(26)} before [${beforeValues.map((value) => value.toFixed(1)).join(", ")}] after [${afterValues.map((value) => value.toFixed(1)).join(", ")}]`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
/**
|
||||
* Preloaded into the server with `NODE_OPTIONS=--require /probe/memory-probe.cjs` so
|
||||
* memory attribution can reach things the inspector cannot.
|
||||
*
|
||||
* Node's inspector evaluates in the main context, where `require` is a module-scoped
|
||||
* variable rather than a global — and the Next standalone entry does not set
|
||||
* `process.mainModule` — so `v8.getHeapSpaceStatistics()` and the CommonJS module cache
|
||||
* are both unreachable from a plain `Runtime.evaluate`. A preload runs as a real module,
|
||||
* so it can park what is needed on `globalThis`, which the inspector *can* see.
|
||||
*
|
||||
* Diagnostics only. Never part of a shipped image: it is mounted in by the benchmark
|
||||
* harness and costs a few hundred KiB of retained bookkeeping.
|
||||
*/
|
||||
const v8 = require("v8");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const Module = require("module");
|
||||
|
||||
/** Cache stat() results: the same chunk is asked about at every capture stage. */
|
||||
const sizeCache = new Map();
|
||||
const fileSize = (file) => {
|
||||
if (sizeCache.has(file)) return sizeCache.get(file);
|
||||
let size = 0;
|
||||
try {
|
||||
size = fs.statSync(file).size;
|
||||
} catch {
|
||||
size = 0;
|
||||
}
|
||||
sizeCache.set(file, size);
|
||||
return size;
|
||||
};
|
||||
|
||||
/**
|
||||
* Which package a loaded file belongs to. Server bundle chunks live under
|
||||
* `.next/server/chunks`, so they are grouped separately from real node_modules
|
||||
* packages — a chunk's cost belongs to "what Next inlined", not to one dependency.
|
||||
*/
|
||||
const attributeFile = (file) => {
|
||||
const nodeModules = file.lastIndexOf("/node_modules/");
|
||||
if (nodeModules !== -1) {
|
||||
const rest = file.slice(nodeModules + "/node_modules/".length);
|
||||
const parts = rest.split("/");
|
||||
const pkg = parts[0].startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0];
|
||||
if (pkg === "next") {
|
||||
if (rest.includes("/dist/compiled/")) return "next/dist/compiled (vendored deps)";
|
||||
return "next/dist (framework runtime)";
|
||||
}
|
||||
return `node_modules: ${pkg}`;
|
||||
}
|
||||
if (file.includes("/.next/server/chunks/")) return ".next/server/chunks (bundled app code)";
|
||||
if (file.includes("/.next/server/app/")) return ".next/server/app (route bundles)";
|
||||
if (file.includes("/.next/server/pages/")) return ".next/server/pages";
|
||||
if (file.includes("/.next/server/")) return ".next/server (other)";
|
||||
if (file.includes("/.next/")) return ".next (other)";
|
||||
return `app: ${path.dirname(file).split("/").slice(-2).join("/")}`;
|
||||
};
|
||||
|
||||
globalThis.__homarrProbe = {
|
||||
/** Per-space heap statistics: separates live JS objects from compiled code. */
|
||||
spaces: () => v8.getHeapSpaceStatistics(),
|
||||
heap: () => v8.getHeapStatistics(),
|
||||
usage: () => process.memoryUsage(),
|
||||
|
||||
/**
|
||||
* Every file the CommonJS loader has resolved, with its on-disk byte size grouped by
|
||||
* owner. V8 keeps a script's source string alive for as long as any function in it
|
||||
* might still need lazy compilation, so these sizes are a direct lower bound on how
|
||||
* much heap the loaded code costs before a single object is allocated.
|
||||
*/
|
||||
modules: () => {
|
||||
const cache = Module._cache ?? {};
|
||||
const groups = new Map();
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (const file of Object.keys(cache)) {
|
||||
const size = fileSize(file);
|
||||
total += size;
|
||||
count++;
|
||||
const key = attributeFile(file);
|
||||
const acc = groups.get(key) ?? { bytes: 0, files: 0 };
|
||||
acc.bytes += size;
|
||||
acc.files++;
|
||||
groups.set(key, acc);
|
||||
}
|
||||
return {
|
||||
totalBytes: total,
|
||||
fileCount: count,
|
||||
groups: [...groups.entries()]
|
||||
.map(([name, acc]) => ({ name, ...acc }))
|
||||
.sort((a, b) => b.bytes - a.bytes),
|
||||
};
|
||||
},
|
||||
|
||||
/** The individual loaded files, largest first — for naming specific offenders. */
|
||||
largestModules: (limit = 40) =>
|
||||
Object.keys(Module._cache ?? {})
|
||||
.map((file) => ({ file, bytes: fileSize(file) }))
|
||||
.sort((a, b) => b.bytes - a.bytes)
|
||||
.slice(0, limit),
|
||||
|
||||
/** Raw list of resolved files, so two capture stages can be diffed. */
|
||||
moduleFiles: () => Object.keys(Module._cache ?? {}),
|
||||
|
||||
/** Native bindings pulled in by the process, in load order. */
|
||||
moduleLoadList: () => process.moduleLoadList.slice(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Opt-in tracking of large Buffer allocations, by allocation site.
|
||||
*
|
||||
* `process.memoryUsage().arrayBuffers` can show ~100 MiB without any hint of who asked for
|
||||
* it, and a heap snapshot does not help: an ArrayBuffer's backing store lives outside the
|
||||
* JS heap, so the snapshot shows a small wrapper object and none of the bytes. Wrapping the
|
||||
* allocators and keeping a stack per call is the only way to get from "94 MiB of buffers"
|
||||
* to a file and line.
|
||||
*
|
||||
* Only allocations at or above the threshold are recorded, so the common small-buffer path
|
||||
* keeps its normal cost. Enable with HOMARR_PROBE_TRACK_BUFFERS=<bytes>.
|
||||
*/
|
||||
const trackThreshold = Number(process.env.HOMARR_PROBE_TRACK_BUFFERS ?? 0);
|
||||
const bufferSites = new Map();
|
||||
let trackedBytes = 0;
|
||||
let trackedCount = 0;
|
||||
if (trackThreshold > 0) {
|
||||
const record = (size) => {
|
||||
if (size < trackThreshold) return;
|
||||
trackedBytes += size;
|
||||
trackedCount++;
|
||||
// Frames 0-2 are Error/this wrapper, so skip them to land on the real caller.
|
||||
const stack = (new Error().stack ?? "").split("\n").slice(3, 9).join("\n");
|
||||
const acc = bufferSites.get(stack) ?? { bytes: 0, count: 0, max: 0 };
|
||||
acc.bytes += size;
|
||||
acc.count++;
|
||||
if (size > acc.max) acc.max = size;
|
||||
bufferSites.set(stack, acc);
|
||||
};
|
||||
|
||||
for (const name of ["alloc", "allocUnsafe", "allocUnsafeSlow"]) {
|
||||
const original = Buffer[name];
|
||||
if (typeof original !== "function") continue;
|
||||
Buffer[name] = function (size, ...rest) {
|
||||
record(typeof size === "number" ? size : 0);
|
||||
return original.call(this, size, ...rest);
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Buffers big enough to matter get their first bytes recorded. A stack says undici
|
||||
* buffered a body; only the bytes say whether that body was JSON, an image or a video
|
||||
* stream, and that is what decides whether the fix is a cap, a stream or a different URL.
|
||||
*/
|
||||
const PREVIEW_THRESHOLD = 4 * 1048576;
|
||||
const previews = [];
|
||||
const originalConcat = Buffer.concat;
|
||||
Buffer.concat = function (list, totalLength) {
|
||||
const size = typeof totalLength === "number" ? totalLength : list.reduce((sum, item) => sum + item.length, 0);
|
||||
record(size);
|
||||
const result = originalConcat.call(this, list, totalLength);
|
||||
if (size >= PREVIEW_THRESHOLD && previews.length < 20) {
|
||||
previews.push({
|
||||
size,
|
||||
chunks: list.length,
|
||||
hex: result.subarray(0, 24).toString("hex"),
|
||||
text: result.subarray(0, 220).toString("latin1").replace(/[^\x20-\x7e]/g, "."),
|
||||
stack: (new Error().stack ?? "").split("\n").slice(2, 12).join("\n"),
|
||||
});
|
||||
}
|
||||
return result;
|
||||
};
|
||||
globalThis.__homarrProbe.previews = () => previews;
|
||||
|
||||
// Default of 10 frames loses the caller behind undici's microtask hops.
|
||||
Error.stackTraceLimit = 30;
|
||||
|
||||
globalThis.__homarrProbe.buffers = () => ({
|
||||
threshold: trackThreshold,
|
||||
previews: globalThis.__homarrProbe.previews ? globalThis.__homarrProbe.previews() : [],
|
||||
totalBytes: trackedBytes,
|
||||
count: trackedCount,
|
||||
sites: [...bufferSites.entries()]
|
||||
.map(([stack, acc]) => ({ stack, ...acc }))
|
||||
.sort((a, b) => b.bytes - a.bytes)
|
||||
.slice(0, 25),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Opt-in tracking of outbound fetch responses, by URL.
|
||||
*
|
||||
* Buffer-allocation stacks say *how* memory was allocated (undici buffering a whole body)
|
||||
* but not *which request* did it, because the URL is not on the stack. Wrapping fetch before
|
||||
* Next loads means every caller gets the instrumented version, so a single oversized
|
||||
* integration response can be named. Enable with HOMARR_PROBE_TRACK_FETCH=1.
|
||||
*/
|
||||
if (process.env.HOMARR_PROBE_TRACK_FETCH === "1" && typeof globalThis.fetch === "function") {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const fetchStats = new Map();
|
||||
/** Group by origin+path with the query string dropped: tokens and ids would fragment the table. */
|
||||
const keyFor = (url) => {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return `${parsed.origin}${parsed.pathname}`;
|
||||
} catch {
|
||||
return String(url).slice(0, 120);
|
||||
}
|
||||
};
|
||||
|
||||
globalThis.fetch = async function (resource, options) {
|
||||
const url = typeof resource === "string" ? resource : (resource?.url ?? String(resource));
|
||||
const response = await originalFetch.call(this, resource, options);
|
||||
const key = keyFor(url);
|
||||
const acc = fetchStats.get(key) ?? { calls: 0, declaredBytes: 0, readBytes: 0, maxRead: 0, unknownLength: 0 };
|
||||
acc.calls++;
|
||||
const declared = Number(response.headers.get("content-length") ?? 0);
|
||||
if (declared > 0) acc.declaredBytes += declared;
|
||||
else acc.unknownLength++;
|
||||
fetchStats.set(key, acc);
|
||||
|
||||
// Body readers are wrapped rather than trusting content-length, because chunked and
|
||||
// gzipped responses declare nothing and those are exactly the large ones.
|
||||
for (const method of ["json", "text", "arrayBuffer"]) {
|
||||
const originalMethod = response[method];
|
||||
if (typeof originalMethod !== "function") continue;
|
||||
response[method] = async function (...rest) {
|
||||
const before = process.memoryUsage().arrayBuffers;
|
||||
const value = await originalMethod.apply(this, rest);
|
||||
const size =
|
||||
method === "text" && typeof value === "string"
|
||||
? value.length
|
||||
: method === "arrayBuffer" && value?.byteLength
|
||||
? value.byteLength
|
||||
: Math.max(0, process.memoryUsage().arrayBuffers - before);
|
||||
acc.readBytes += size;
|
||||
if (size > acc.maxRead) acc.maxRead = size;
|
||||
return value;
|
||||
};
|
||||
}
|
||||
return response;
|
||||
};
|
||||
|
||||
globalThis.__homarrProbe.fetches = () => ({
|
||||
total: [...fetchStats.values()].reduce((sum, acc) => sum + acc.readBytes, 0),
|
||||
sites: [...fetchStats.entries()]
|
||||
.map(([url, acc]) => ({ url, ...acc }))
|
||||
.sort((a, b) => Math.max(b.readBytes, b.declaredBytes) - Math.max(a.readBytes, a.declaredBytes))
|
||||
.slice(0, 30),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* A localhost-only JSON endpoint for the same data. Sampling over the inspector needs
|
||||
* SIGUSR1 and a CDP handshake per capture, which is too heavy to run at every stage of a
|
||||
* stress run; this is a plain fetch. Only started when HOMARR_PROBE_PORT is set, so the
|
||||
* preload does nothing observable unless the harness asks for it.
|
||||
*/
|
||||
const probePort = Number(process.env.HOMARR_PROBE_PORT ?? 0);
|
||||
if (probePort > 0) {
|
||||
const probe = globalThis.__homarrProbe;
|
||||
require("http")
|
||||
.createServer((request, response) => {
|
||||
let body;
|
||||
try {
|
||||
// The raw file list is served separately: it is long, and every stage of a stress
|
||||
// run fetches the summary, so keeping it out of the default payload keeps those
|
||||
// captures small while still allowing a boot-vs-loaded diff on demand.
|
||||
body =
|
||||
request.url === "/files"
|
||||
? JSON.stringify(probe.moduleFiles())
|
||||
: request.url === "/buffers"
|
||||
? JSON.stringify(probe.buffers ? probe.buffers() : { disabled: true })
|
||||
: request.url === "/fetches"
|
||||
? JSON.stringify(probe.fetches ? probe.fetches() : { disabled: true })
|
||||
: JSON.stringify({
|
||||
usage: probe.usage(),
|
||||
spaces: probe.spaces(),
|
||||
heap: probe.heap(),
|
||||
modules: probe.modules(),
|
||||
largestModules: probe.largestModules(25),
|
||||
uptime: process.uptime(),
|
||||
execArgv: process.execArgv,
|
||||
});
|
||||
} catch (error) {
|
||||
response.writeHead(500, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify({ error: String(error && error.message) }));
|
||||
return;
|
||||
}
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.end(body);
|
||||
})
|
||||
// NODE_OPTIONS applies to every node process in the container, including the DB
|
||||
// migration step and any `docker exec node` the harness runs, so all of them load this
|
||||
// preload and race for the port. Only the long-lived server needs to serve; the rest
|
||||
// must carry on silently rather than dying on EADDRINUSE.
|
||||
.on("error", (error) => {
|
||||
if (error.code !== "EADDRINUSE") throw error;
|
||||
})
|
||||
.listen(probePort, "127.0.0.1", () => console.log(`[memory-probe] listening on 127.0.0.1:${probePort}`));
|
||||
}
|
||||
@@ -46,6 +46,22 @@ const soakIntervalMs = Number(process.env.STRESS_SOAK_INTERVAL_MS ?? 20_000);
|
||||
const keepContainer = process.env.STRESS_KEEP_CONTAINER === "true";
|
||||
/** Dump a browser heap snapshot here (analyse with analyze-react-fibers.mjs). */
|
||||
const clientSnapshotPath = process.env.STRESS_CLIENT_SNAPSHOT;
|
||||
/**
|
||||
* Extra `docker run` wiring, so a diagnostics-only preload can be injected into a
|
||||
* shipped image without rebuilding it. Used to mount scripts/benchmarks at /probe and
|
||||
* point NODE_OPTIONS at memory-probe.cjs, which is what makes per-V8-space and
|
||||
* per-module attribution possible at every capture stage.
|
||||
* STRESS_MOUNT="hostPath:containerPath:ro" (comma-separated for several)
|
||||
* STRESS_ENV="KEY=value,KEY2=value2"
|
||||
*/
|
||||
const extraMounts = (process.env.STRESS_MOUNT ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const extraEnv = (process.env.STRESS_ENV ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const outputDirectory = path.resolve(process.env.STRESS_OUTPUT ?? `.bench/stress/${label}`);
|
||||
|
||||
const password = process.env.STRESS_PASSWORD ?? "demodemo";
|
||||
@@ -68,11 +84,45 @@ let startedAt = Date.now();
|
||||
|
||||
const log = (message: string) => console.log(`[stress:${label}] ${message}`);
|
||||
|
||||
/** Port the injected memory probe serves JSON on, if one was wired up via STRESS_ENV. */
|
||||
const probePort = /HOMARR_PROBE_PORT=(\d+)/.exec(extraEnv.join(","))?.[1];
|
||||
|
||||
/**
|
||||
* Pulls the probe's V8-space and loaded-module attribution alongside the coarse memory
|
||||
* numbers. Saved per stage so the *delta* between stages is attributable — "the 90 MiB
|
||||
* that appeared at sign-in was old-space objects, not code" is a different problem from
|
||||
* "the code got bigger", and only per-stage capture can tell them apart.
|
||||
*/
|
||||
const fetchProbeAsync = async (probePath: string) => {
|
||||
const script = `fetch("http://127.0.0.1:${probePort}${probePath}").then((r) => r.text()).then((t) => console.log(t)).catch((e) => console.log(JSON.stringify({ error: String(e && e.message) })));`;
|
||||
// NODE_OPTIONS is cleared so this helper does not also run the server's --require preload.
|
||||
const { stdout } = await execFileAsync("docker", ["exec", "-e", "NODE_OPTIONS=", containerName, "node", "-e", script], {
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
}).catch((error: Error) => ({ stdout: JSON.stringify({ error: error.message }) }));
|
||||
return stdout;
|
||||
};
|
||||
|
||||
const captureProbeAsync = async (name: string) => {
|
||||
if (!probePort) return;
|
||||
await writeFile(path.join(outputDirectory, `${name}.probe.json`), await fetchProbeAsync("/"));
|
||||
// Buffer allocation sites, when tracking is enabled. Cumulative, so the useful reading is
|
||||
// the difference between two stages: that isolates what one workload allocated.
|
||||
const buffers = await fetchProbeAsync("/buffers");
|
||||
if (!buffers.includes('"disabled"')) {
|
||||
await writeFile(path.join(outputDirectory, `${name}.buffers.json`), buffers);
|
||||
}
|
||||
const fetches = await fetchProbeAsync("/fetches");
|
||||
if (!fetches.includes('"disabled"')) {
|
||||
await writeFile(path.join(outputDirectory, `${name}.fetches.json`), fetches);
|
||||
}
|
||||
};
|
||||
|
||||
const captureAsync = async (name: string) => {
|
||||
const { stdout } = await execFileAsync("docker", ["exec", containerName, "sh", "-c", stressMemoryScript], {
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
await writeFile(path.join(outputDirectory, `${name}.txt`), stdout);
|
||||
await captureProbeAsync(name);
|
||||
const checkpoint = parseStressSnapshot(name, Date.now() - startedAt, stdout);
|
||||
checkpoints.push(checkpoint);
|
||||
log(`${name}: container=${toMiB(checkpoint.container.currentBytes)} MiB`);
|
||||
@@ -386,6 +436,8 @@ const main = async () => {
|
||||
// Escape hatch for A/B-ing V8 flags against an image whose run.sh does not read
|
||||
// HOMARR_MAX_OLD_SPACE_SIZE (e.g. measuring the heap cap against an older image).
|
||||
if (nodeOptions) runArgs.push("-e", `NODE_OPTIONS=${nodeOptions}`);
|
||||
for (const mount of extraMounts) runArgs.push("-v", mount);
|
||||
for (const entry of extraEnv) runArgs.push("-e", entry);
|
||||
runArgs.push(image);
|
||||
|
||||
await execFileAsync("docker", runArgs);
|
||||
|
||||
Reference in new issue
Block a user