From dc460575448bb674dbac9e7712d86d78a28c3b4a Mon Sep 17 00:00:00 2001 From: Thomas Camlong Date: Mon, 10 Aug 2026 18:25:00 +0200 Subject: [PATCH] docs(findings): definitive server-RAM A/B, and jemalloc rejected with evidence Two things this settles. First, the end-to-end number. Every Phase 2 result so far was an incremental A/B against the build immediately before it, which cannot be summed. Rebuilt the merge-base (06724cfeb) from a clean worktree and ran 2+2 alternated against HEAD, reporting cgroup anonymous memory - the figure that counts against a container limit and cannot be reclaimed: stage before after peak anon 376.6 -> 279.3 MiB (-26%) boot idle 130.5 -> 62.4 MiB (-52%) after sign-in 247.4 -> 142.9 MiB (-42%) dashboard 293.4 -> 170.4 MiB (-42%) 4 tabs 366.5 -> 258.2 MiB (-30%) after soak 323.7 -> 200.8 MiB (-38%) peak node RSS 430.7 -> 336.2 MiB (-22%) No before/after range overlaps at any stage. Costs: CPU 23.5 -> 27.7s (+18%, solid, non-overlapping). Median board load reads +30% but is NOT established - individual loads were [506,383,1233] and [848,431,574] before against [780,808,515] and [621,579,1004] after, heavily overlapping with no cold-then-warm pattern, so six samples per side cannot separate a real regression from the variance. compare-stress-runs.mjs now leads with anonMiB, which the harness recorded all along but the report was ignoring in favour of memory.current - a figure that includes page cache and swings 60+ MiB between runs of a byte-identical image. Second, jemalloc. The largest attributed-but-unfixed item is ~38 MiB of V8 zone memory that musl frees to its own pools and never returns to the kernel, so an allocator that decays spans back to the OS looked like the answer. Tested on the byte-identical image with only HOMARR_MALLOC differing, asserting the "Using jemalloc" log line: boot idle 62.4 -> 80.3 MiB (+29%) after soak 215.5 -> 279.6 MiB (+30%) 4 tabs 259.9 -> 319.6 MiB (+23%) CPU 28.0 -> 24.5 s (-13%) The hypothesis was backwards. jemalloc keeps more freed memory on purpose - thread caches plus time-based dirty-page decay - and spends it on speed. 13% cheaper CPU, 18-30% more resident memory. Reverted, and it closes off "swap the allocator" as a RAM direction: musl is already the frugal choice here. --- PHASE2-FINDINGS.md | 30 ++++++++++++++++++++++ scripts/benchmarks/compare-stress-runs.mjs | 17 ++++++++---- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/PHASE2-FINDINGS.md b/PHASE2-FINDINGS.md index 64f73ae26..8fc425a79 100644 --- a/PHASE2-FINDINGS.md +++ b/PHASE2-FINDINGS.md @@ -958,6 +958,36 @@ never observes the mid-response client disconnect that creates an orphan in the Recorded because the initial reading of the snapshot was wrong in a way that would have shipped a fix for a leak this deployment does not have. +## jemalloc: tested, and musl is the frugal one + +The largest thing that was attributed but not fixed is ~38 MiB of V8 parser/compiler zone memory +that is freed back to malloc and never returned to the kernel by musl. Swapping in jemalloc, which +decays unused spans back to the OS, looked like the obvious answer. + +Tested properly: `jemalloc` added to the image but activated only by `HOMARR_MALLOC=jemalloc` +through `LD_PRELOAD`, so both sides of the A/B ran the **byte-identical image** and differed by one +environment variable — no build-to-build variance at all. Presence of the "Using jemalloc" line in +the container log was asserted, so this is not inferred from the variable being set. + +| stage | musl | jemalloc | change | +| --- | --- | --- | --- | +| boot idle | 62.4 MiB | **80.3 MiB** | **+29%** | +| after sign-in | 144.9 MiB | **170.6 MiB** | +18% | +| dashboard loaded | 214.8 MiB | 184.4 MiB | −14% | +| 4 tabs | 259.9 MiB | **319.6 MiB** | +23% | +| after soak | 215.5 MiB | **279.6 MiB** | +30% | +| peak anon | 299.4 MiB | 322.5 MiB | +8% | +| **CPU** | 28.0 s | **24.5 s** | **−13%** | + +Every row non-overlapping. **The hypothesis was backwards.** jemalloc does not hand freed spans +back sooner; it deliberately keeps more of them — thread-local caches plus time-based dirty-page +decay — and spends that memory to be fast. It is 13% cheaper on CPU and 18–30% more expensive on +resident memory, which is the opposite of the trade wanted here. + +Reverted. The useful conclusion is a closed door: musl's allocator is already the memory-frugal +choice, so "swap the allocator" is not a direction for reducing RAM in this container. If CPU ever +becomes the binding constraint instead, this is a measured 13% for two lines. + ## Measured but not pursued diff --git a/scripts/benchmarks/compare-stress-runs.mjs b/scripts/benchmarks/compare-stress-runs.mjs index 510676429..f7f084284 100644 --- a/scripts/benchmarks/compare-stress-runs.mjs +++ b/scripts/benchmarks/compare-stress-runs.mjs @@ -61,13 +61,20 @@ const peakStage = (stages) => { return values.length ? Math.max(...values) : null; }; +/** + * Ordered with cgroup anonymous memory first, because that is the figure that counts against a + * container limit and cannot be reclaimed. `memory.current` also includes page cache, which has + * swung 60+ MiB between runs of an identical image, so it is reported but never led with. + */ const METRICS = [ - ["peak container MiB", (run) => peakStage(run.summary.containerMiB)], + ["PEAK anon MiB", (run) => peakStage(run.summary.anonMiB)], + ["anon: boot idle", (run) => run.summary.anonMiB?.["01-boot-idle"]], + ["anon: post-login", (run) => run.summary.anonMiB?.["03-post-login"]], + ["anon: board loaded", (run) => run.summary.anonMiB?.["04-board-loaded"]], + ["anon: multi-tab", (run) => run.summary.anonMiB?.["07-multi-tab"]], + ["anon: post-soak", (run) => run.summary.anonMiB?.["06-final"]], ["peak node RSS MiB", (run) => peakStage(run.summary.nodeRssMiB)], - ["peak node anon MiB", (run) => peakStage(run.summary.nodeAnonMiB)], - ["multi-tab container MiB", (run) => run.summary.containerMiB?.["07-multi-tab"]], - ["final (post-soak) MiB", (run) => run.summary.containerMiB?.["06-final"]], - ["boot idle MiB", (run) => run.summary.containerMiB?.["01-boot-idle"]], + ["peak container (w/ cache)", (run) => peakStage(run.summary.containerMiB)], ["peak arrayBuffers MiB", (run) => peakArrayBuffers(run.dir)], ["largest single alloc MiB", (run) => largestAllocation(run.dir)], ["client JS heap MiB", (run) => run.summary.client?.jsHeapUsedMiB],