perf(umami): stop buffering 100k raw events per widget refresh
Profiling the container end to end found one allocation site responsible for
most of the peak: a single 46.3 MiB JSON body, assembled from 3,946 socket
chunks, inside undici's fullyReadBody. That function holds the chunk list and
the joined copy at the same instant, so a body of size N costs about 2N.
The source was two Umami calls asking for pageSize=100000, with
getMultiEventTimeSeriesAsync running them for every tracked event name
concurrently via Promise.all. getEventNamesAsync was downloading every raw
event of the last 30 days purely to collect a handful of distinct name strings.
- getEventNamesAsync now reads the aggregated /metrics?type=event endpoint.
- getWebsiteEventTimeSeriesAsync pages 5,000 records at a time, folding each
page into the bucket map before requesting the next.
- getMultiEventTimeSeriesAsync runs series sequentially.
Instances that return aggregated {x,y} buckets still answer in one request, an
instance that accepts pageSize but ignores page is detected rather than
double-counted, and hitting the page ceiling logs instead of silently
undercounting.
Alternated runs, 2 per side, on a restored real backup:
peak container 498.9 -> 398.5 MiB (-20%)
peak node RSS 452.4 -> 353.3 MiB (-22%)
peak during 4 tabs 498.5 -> 356.3 MiB (-29%)
peak arrayBuffers 95.8 -> 6.6 MiB (-93%)
largest allocation 46.4 -> 2.4 MiB (-95%)
Costs +4% CPU from the extra round trips; board load latency unchanged.
Post-soak steady state and boot idle are unchanged within run-to-run spread.
Adds the attribution tooling this needed: attribute-memory.mts reconciles
cgroup -> processes -> address-space mappings -> V8 spaces -> loaded code,
memory-probe.cjs is an injectable diagnostics preload (never shipped) that
tracks buffer and fetch allocation sites, analyze-heap-dominators.mjs computes
real retained sizes, and compare-stress-runs.mjs reports alternated runs with
their spread.
Also corrects an earlier claim in PHASE2-FINDINGS.md: the reported "7.86 MiB of
duplicate chunk sources" was an artefact of V8 truncating string contents to
~1 KiB in heap snapshots. There are zero byte-identical duplicates among all
3,055 server JS files. The real doubling is character width - one codepoint
above U+00FF forces a whole source string to two bytes per character, which
costs 9.45 MiB across 80 loaded files.
Adds 6 tests; this integration had none.
This commit is contained in:
9 files changed
+1808
-96
No files matched your search
@@ -46,6 +46,22 @@ const soakIntervalMs = Number(process.env.STRESS_SOAK_INTERVAL_MS ?? 20_000);
|
||||
const keepContainer = process.env.STRESS_KEEP_CONTAINER === "true";
|
||||
/** Dump a browser heap snapshot here (analyse with analyze-react-fibers.mjs). */
|
||||
const clientSnapshotPath = process.env.STRESS_CLIENT_SNAPSHOT;
|
||||
/**
|
||||
* Extra `docker run` wiring, so a diagnostics-only preload can be injected into a
|
||||
* shipped image without rebuilding it. Used to mount scripts/benchmarks at /probe and
|
||||
* point NODE_OPTIONS at memory-probe.cjs, which is what makes per-V8-space and
|
||||
* per-module attribution possible at every capture stage.
|
||||
* STRESS_MOUNT="hostPath:containerPath:ro" (comma-separated for several)
|
||||
* STRESS_ENV="KEY=value,KEY2=value2"
|
||||
*/
|
||||
const extraMounts = (process.env.STRESS_MOUNT ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const extraEnv = (process.env.STRESS_ENV ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const outputDirectory = path.resolve(process.env.STRESS_OUTPUT ?? `.bench/stress/${label}`);
|
||||
|
||||
const password = process.env.STRESS_PASSWORD ?? "demodemo";
|
||||
@@ -68,11 +84,45 @@ let startedAt = Date.now();
|
||||
|
||||
const log = (message: string) => console.log(`[stress:${label}] ${message}`);
|
||||
|
||||
/** Port the injected memory probe serves JSON on, if one was wired up via STRESS_ENV. */
|
||||
const probePort = /HOMARR_PROBE_PORT=(\d+)/.exec(extraEnv.join(","))?.[1];
|
||||
|
||||
/**
|
||||
* Pulls the probe's V8-space and loaded-module attribution alongside the coarse memory
|
||||
* numbers. Saved per stage so the *delta* between stages is attributable — "the 90 MiB
|
||||
* that appeared at sign-in was old-space objects, not code" is a different problem from
|
||||
* "the code got bigger", and only per-stage capture can tell them apart.
|
||||
*/
|
||||
const fetchProbeAsync = async (probePath: string) => {
|
||||
const script = `fetch("http://127.0.0.1:${probePort}${probePath}").then((r) => r.text()).then((t) => console.log(t)).catch((e) => console.log(JSON.stringify({ error: String(e && e.message) })));`;
|
||||
// NODE_OPTIONS is cleared so this helper does not also run the server's --require preload.
|
||||
const { stdout } = await execFileAsync("docker", ["exec", "-e", "NODE_OPTIONS=", containerName, "node", "-e", script], {
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
}).catch((error: Error) => ({ stdout: JSON.stringify({ error: error.message }) }));
|
||||
return stdout;
|
||||
};
|
||||
|
||||
const captureProbeAsync = async (name: string) => {
|
||||
if (!probePort) return;
|
||||
await writeFile(path.join(outputDirectory, `${name}.probe.json`), await fetchProbeAsync("/"));
|
||||
// Buffer allocation sites, when tracking is enabled. Cumulative, so the useful reading is
|
||||
// the difference between two stages: that isolates what one workload allocated.
|
||||
const buffers = await fetchProbeAsync("/buffers");
|
||||
if (!buffers.includes('"disabled"')) {
|
||||
await writeFile(path.join(outputDirectory, `${name}.buffers.json`), buffers);
|
||||
}
|
||||
const fetches = await fetchProbeAsync("/fetches");
|
||||
if (!fetches.includes('"disabled"')) {
|
||||
await writeFile(path.join(outputDirectory, `${name}.fetches.json`), fetches);
|
||||
}
|
||||
};
|
||||
|
||||
const captureAsync = async (name: string) => {
|
||||
const { stdout } = await execFileAsync("docker", ["exec", containerName, "sh", "-c", stressMemoryScript], {
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
await writeFile(path.join(outputDirectory, `${name}.txt`), stdout);
|
||||
await captureProbeAsync(name);
|
||||
const checkpoint = parseStressSnapshot(name, Date.now() - startedAt, stdout);
|
||||
checkpoints.push(checkpoint);
|
||||
log(`${name}: container=${toMiB(checkpoint.container.currentBytes)} MiB`);
|
||||
@@ -386,6 +436,8 @@ const main = async () => {
|
||||
// Escape hatch for A/B-ing V8 flags against an image whose run.sh does not read
|
||||
// HOMARR_MAX_OLD_SPACE_SIZE (e.g. measuring the heap cap against an older image).
|
||||
if (nodeOptions) runArgs.push("-e", `NODE_OPTIONS=${nodeOptions}`);
|
||||
for (const mount of extraMounts) runArgs.push("-v", mount);
|
||||
for (const entry of extraEnv) runArgs.push("-e", entry);
|
||||
runArgs.push(image);
|
||||
|
||||
await execFileAsync("docker", runArgs);
|
||||
|
||||
Reference in new issue
Block a user