From cb3a2a360e6c8b606bcdadaf56567a100625e1c9 Mon Sep 17 00:00:00 2001 From: Thomas Camlong Date: Sun, 9 Aug 2026 21:35:00 +0200 Subject: [PATCH] perf(umami): stop buffering 100k raw events per widget refresh Profiling the container end to end found one allocation site responsible for most of the peak: a single 46.3 MiB JSON body, assembled from 3,946 socket chunks, inside undici's fullyReadBody. That function holds the chunk list and the joined copy at the same instant, so a body of size N costs about 2N. The source was two Umami calls asking for pageSize=100000, with getMultiEventTimeSeriesAsync running them for every tracked event name concurrently via Promise.all. getEventNamesAsync was downloading every raw event of the last 30 days purely to collect a handful of distinct name strings. - getEventNamesAsync now reads the aggregated /metrics?type=event endpoint. - getWebsiteEventTimeSeriesAsync pages 5,000 records at a time, folding each page into the bucket map before requesting the next. - getMultiEventTimeSeriesAsync runs series sequentially. Instances that return aggregated {x,y} buckets still answer in one request, an instance that accepts pageSize but ignores page is detected rather than double-counted, and hitting the page ceiling logs instead of silently undercounting. Alternated runs, 2 per side, on a restored real backup: peak container 498.9 -> 398.5 MiB (-20%) peak node RSS 452.4 -> 353.3 MiB (-22%) peak during 4 tabs 498.5 -> 356.3 MiB (-29%) peak arrayBuffers 95.8 -> 6.6 MiB (-93%) largest allocation 46.4 -> 2.4 MiB (-95%) Costs +4% CPU from the extra round trips; board load latency unchanged. Post-soak steady state and boot idle are unchanged within run-to-run spread. Adds the attribution tooling this needed: attribute-memory.mts reconciles cgroup -> processes -> address-space mappings -> V8 spaces -> loaded code, memory-probe.cjs is an injectable diagnostics preload (never shipped) that tracks buffer and fetch allocation sites, analyze-heap-dominators.mjs computes real retained sizes, and compare-stress-runs.mjs reports alternated runs with their spread. Also corrects an earlier claim in PHASE2-FINDINGS.md: the reported "7.86 MiB of duplicate chunk sources" was an artefact of V8 truncating string contents to ~1 KiB in heap snapshots. There are zero byte-identical duplicates among all 3,055 server JS files. The real doubling is character width - one codepoint above U+00FF forces a whole source string to two bytes per character, which costs 9.45 MiB across 80 loaded files. Adds 6 tests; this integration had none. --- PHASE2-FINDINGS.md | 241 +++++++++- .../src/umami/test/umami-integration.spec.ts | 179 +++++++ .../src/umami/umami-integration.ts | 245 ++++++---- .../benchmarks/analyze-heap-dominators.mjs | 245 ++++++++++ scripts/benchmarks/analyze-probe-stages.mjs | 90 ++++ scripts/benchmarks/attribute-memory.mts | 451 ++++++++++++++++++ scripts/benchmarks/compare-stress-runs.mjs | 104 ++++ scripts/benchmarks/memory-probe.cjs | 297 ++++++++++++ scripts/benchmarks/stress-restore.mts | 52 ++ 9 files changed, 1808 insertions(+), 96 deletions(-) create mode 100644 packages/integrations/src/umami/test/umami-integration.spec.ts create mode 100644 scripts/benchmarks/analyze-heap-dominators.mjs create mode 100644 scripts/benchmarks/analyze-probe-stages.mjs create mode 100644 scripts/benchmarks/attribute-memory.mts create mode 100644 scripts/benchmarks/compare-stress-runs.mjs create mode 100644 scripts/benchmarks/memory-probe.cjs diff --git a/PHASE2-FINDINGS.md b/PHASE2-FINDINGS.md index 402643afb..4bf2a6626 100644 --- a/PHASE2-FINDINGS.md +++ b/PHASE2-FINDINGS.md @@ -328,10 +328,15 @@ What the server heap is actually made of (141 MiB used, 259 MiB RSS): fix catalog measured at zero, and why `serverExternalPackages` — which removes code from the boot-loaded bundle — was the only lever that moved idle memory. -Within that, **7.86 MiB is literal duplicate copies** of identical chunk sources: the same -module inlined into several Turbopack chunks and retained once per chunk. Worst offenders -were `@ioredis/commands`' command table (in 10 chunks, resident 6×, 2.07 MiB wasted) and -`got`/`@octokit/request-error` (2.1 MiB each). +> **Correction (later round).** An earlier version of this section claimed "7.86 MiB is +> literal duplicate copies" of identical chunk sources. That figure was wrong. It came from +> grouping heap-snapshot strings by content, but **V8 truncates string contents to ~1 KiB +> when serialising a snapshot**, so the comparison only ever saw each string's first +> kilobyte and reported near-identical chunks as identical. Hashing the real files found +> **zero byte-identical duplicates among all 3,055 `.next/server/**/*.js` files (77.6 MiB)**. +> The chunks that looked like copies are distinct files that happen to begin with the same +> vendored module. The real doubling has a different cause — see "Why the source costs +> twice what it should" below. `@ioredis/commands` is now external, which removes it from all 10 chunks. Recorded as **bundle hygiene, not a memory win**: idle went 89.7 → 89.4 MiB, inside noise, and @@ -355,6 +360,234 @@ left are both large: 2. **Stop the App Router re-rendering every second** (see above). That governs the browser side, which is where the 650 MB peak lives. +## Full attribution — every megabyte, top to bottom + +Everything above measured *totals*. This section measures *composition*: each layer is +reconciled against the one above it, so nothing hides in a rounding gap. Tooling: +`scripts/benchmarks/attribute-memory.mts` (cgroup → processes → mappings → V8 spaces → +loaded code) with `scripts/benchmarks/memory-probe.cjs` injected as a `--require` preload. + +The probe is diagnostics-only and is never in a shipped image: the harness mounts +`scripts/benchmarks` at `/probe` and sets `NODE_OPTIONS=--require /probe/memory-probe.cjs`. + +### Layer 1 — the container is one process + +| process | RSS at steady state | +| --- | --- | +| **next-server** | **188.1 MiB (95%)** | +| redis-server | 4.8 MiB | +| nginx (2 procs) | 3.7 MiB | +| shells | 1.3 MiB | + +Redis and nginx are rounding errors. Every question about Homarr's memory is a question +about one Node process. + +### Layer 2 — where that process's RSS lives + +Measured from `/proc//smaps`. Anonymous mappings cannot be told apart by size, so they +are classified by signature: V8 allocates its heap in **256 KiB pages** (`kRegularPageSize`), +and pthread reserves ~8 MiB per thread stack. + +At boot idle (139 MiB RSS): + +| region | RSS | what it is | +| --- | --- | --- | +| **node binary (code)** | **54.3 MiB** | the Node runtime's own text pages — **file-backed and shared**, so it counts in RSS but *not* in cgroup `anon` | +| V8 heap pages (256 KiB each) | 47.8 MiB | the JS heap | +| native malloc / small V8 spaces | 28.2 MiB | musl arenas, Buffers | +| V8 code range / trusted space | 4.8 MiB | | +| JIT code (anonymous executable) | 1.8 MiB | machine code V8 compiled | +| better_sqlite3.node | 2.0 MiB | the only native addon that matters | +| thread stacks | 0.1 MiB resident | 48 MiB *reserved*, essentially untouched | + +**This explains the single most confusing number in the whole exercise:** RSS 139 MiB but +cgroup `anon` only 89 MiB. The 54 MiB gap is the Node binary itself. It is reclaimable page +cache shared with every other Node process on the host, so it is real RSS but not a real +cost — which is exactly why `anon` is the metric this PR reports. + +Address space reserved but not committed: **16.8 GiB**. V8 claims its cage up front. It is +not memory. + +### Layer 3 — inside the JS heap + +| V8 space | boot idle | after real use | +| --- | --- | --- | +| old_space | 42.0 MiB | **97.2 MiB** | +| large_object_space | 22.7 MiB | 27.3 MiB | +| code_space + code_large | 1.9 MiB | 10.9 MiB | +| trusted_space | 3.9 MiB | 8.1 MiB | +| new_space | 0.0 MiB | 0.2 MiB | +| external (Buffers) | 4.1 MiB | 22.0 MiB | + +### Layer 4 — what those objects *are* + +From a post-GC heap snapshot at steady state (143.8 MiB heapUsed, 1.39M nodes): + +| bucket | size | share of heap | +| --- | --- | --- | +| **bundled JS source retained as strings** | **41.0 MiB** | **28%** | +| ArrayBuffer backing stores | 17.6 MiB | 12% | +| object property/element arrays | 16.4 MiB | 11% | +| other strings (translations, ids) | 10.3 MiB | 7% | +| closures | 5.7 MiB | 4% | +| plain objects | 4.5 MiB | 3% | +| shapes/maps/descriptors | 6.6 MiB | 5% | +| compiled code metadata | ~12 MiB | 8% | + +Confirmed again, with a dominator-tree analysis this time +(`scripts/benchmarks/analyze-heap-dominators.mjs`, real retained sizes rather than shallow +histograms): the largest single retainers are `code:` nodes named after chunk files, each +holding **0.5–1.4 MiB**, i.e. the Script objects keeping their own source alive. V8 retains +a script's source string for as long as any function in it might still be lazily compiled. + +**The memory is the code, not cached data.** + +### Why the source costs twice what it should + +`.next/server` chunks loaded at boot are 21.7 MiB on disk, but the heap holds 32.4 MiB of +source strings. The gap is not duplication — it is character width. + +A V8 string is either one byte per character (Latin-1) or two. **One character above U+00FF +anywhere in a file forces the entire source string to two bytes.** + +| loaded JS | files | on disk | heap cost | +| --- | --- | --- | --- | +| pure Latin-1 | 1,025 | 12.25 MiB | 12.25 MiB | +| contains ≥1 char > U+00FF | **80** | 9.45 MiB | **18.91 MiB** | + +Predicted total 31.16 MiB against 32.43 MiB measured — within 4%. + +Proven directly rather than inferred, by reading one chunk as shipped and again with its +non-Latin-1 characters replaced (identical length): + +``` +_049nb9n._.js 538,519 chars, 2 of them non-Latin-1 + as shipped : 1053 KiB per copy -> 2.00 bytes/char + ascii variant : 526 KiB per copy -> 1.00 bytes/char +``` + +**Two characters cost 527 KiB.** The offenders are almost all trivia in dependencies: + +| chunk | non-ASCII chars | the actual characters | +| --- | --- | --- | +| `_049nb9n._.js` (+2 siblings) | 2 | `process.versions.icu?"✅":"Y "` | +| `[root-of-the-server]__1ipwvlq._.js` | 13,141 | Next's log symbols `○ ⨯ ⚠` | +| `_1ew9xg8._.js` (+ssr twin) | 70,175 | a UTF-8 self-test containing `💩` | +| zod chunks ×4 | 13,137 | `✖ ${i.message}` | +| `app-page-turbo.runtime.prod.js` | 13 | an ellipsis `…` | +| `[root-of-the-server]__1xyzm2f._.js` | 13 | **an em-dash in Homarr's own MCP instructions** | + +Total recoverable: **~9.45 MiB of heap** (≈10% of idle `anon`), by emitting server bundles +with `\uXXXX` escapes instead of raw characters. Not implemented here — it means rewriting +minified vendor output at build time, and the safety argument needs to be made properly +before that ships. Recorded as a measured, mechanistically understood opportunity. + +## The peak: one integration was allocating 46 MiB per refresh + +The steady state above is ~237 MiB RSS. The *peak* during a 4-tab session was 429 MiB RSS / +539 MiB container, and that peak is a completely different phenomenon. + +Per-stage V8 accounting (`scripts/benchmarks/analyze-probe-stages.mjs`): + +| stage | rss | old_space | external | **arrayBuffers** | +| --- | --- | --- | --- | --- | +| boot idle | 138.6 | 42.0 | 4.1 | 0.3 | +| board loaded | 254.9 | 95.3 | 22.9 | 3.0 | +| nav ×3 | 361.9 | 122.5 | 24.7 | 4.8 | +| **4 tabs, 2 boards** | **429.3** | **152.6** | **114.3** | **94.3** | +| after 90 s idle | 237.0 | 97.2 | 22.0 | **2.0** | + +`arrayBuffers` goes **0.3 → 94.3 → 2.0**. It is entirely transient — which is precisely why +the app appears to "oscillate between 835 MB and 400 MB" and never settles at the high mark. +Soak growth was **−102 MiB/hour**: there is no leak. + +Tracking large `Buffer` allocations by stack (`HOMARR_PROBE_TRACK_BUFFERS`) put **48.3 MiB +in 16 allocations at one site, with a single allocation of 46.3 MiB**: + +``` +T (/app/apps/nextjs/.next/server/chunks/_1q3xovl._.js:92:32374) +process.processTicksAndRejections +``` + +That function is undici's `fullyReadBody` — the machinery behind `Response.json()`: + +```js +async function T(A,e,t){ let r=[],s=0; for(;;){ let{done:i,value:o}=await A.read(); + if(i) return void e(Buffer.concat(r,s)); r.push(o), s+=o.length } } +``` + +It holds the chunk list *and* the joined copy at the same moment, so a body of size N costs +~2N at the instant it completes. Capturing the bytes identified the response immediately: + +``` +46.3 MiB assembled from 3,946 socket chunks +{"data":[{"id":"1251f6db…","websiteId":"f133f10c…","sessionId":"9e02d227…","createdAt":"2026-08-09T… +``` + +**Umami.** And the cause was in plain source — two call sites requesting +`pageSize: "100000"`, i.e. up to 100,000 raw event records: + +1. `getEventNamesAsync` downloaded every raw event of the last 30 days **to collect a + handful of distinct event name strings**. +2. `getWebsiteEventTimeSeriesAsync` downloaded them to bucket them into a time series. +3. `getMultiEventTimeSeriesAsync` ran (2) for **every tracked event name concurrently** via + `Promise.all`, multiplying the resident pages by the number of events. + +A board with 3 Umami widgets and several tracked events therefore had several ~46 MiB bodies +in flight at once, each briefly doubled, plus ~100k parsed objects per response landing in +old_space. That is the 500–880 MB. + +### The fix + +| call site | before | after | +| --- | --- | --- | +| `getEventNamesAsync` | list 100k raw events, collect distinct names | aggregated `/metrics?type=event`, one row per name | +| `getWebsiteEventTimeSeriesAsync` | one page of up to 100k records | pages of 1,000, folded into buckets and released | +| `getMultiEventTimeSeriesAsync` | `Promise.all` over all event names | sequential | + +The paging path keeps the existing dual-shape handling: instances that return aggregated +`{x,y}` buckets still answer in one request, and an instance that accepts `pageSize` but +ignores `page` is detected (repeated first record id) and stopped rather than double-counted. +Hitting the page ceiling logs a warning instead of silently undercounting. + +Covered by `packages/integrations/src/umami/test/umami-integration.spec.ts` (6 tests, new — +there were none). + +### Result + +Alternated runs, 2 before and 2 after, same harness and workload, `before` = branch HEAD +`f707f6433`, `after` = HEAD + this change. Per-run values are given because the harness's +own spread is what decides whether a difference means anything. + +| metric | before | after | change | before spread / after spread | +| --- | --- | --- | --- | --- | +| **peak container** | 498.9 MiB | **398.5 MiB** | **−20%** | [461.6, 536.3] / [387.0, 410.0] | +| **peak node RSS** | 452.4 MiB | **353.3 MiB** | **−22%** | [432.7, 472.2] / [355.6, 350.9] | +| **peak during 4 tabs** | 498.5 MiB | **356.3 MiB** | **−29%** | [460.8, 536.3] / [341.8, 370.8] | +| **peak arrayBuffers** | 95.8 MiB | **6.6 MiB** | **−93%** | [95.7, 95.8] / [6.4, 6.7] | +| largest single allocation | 46.4 MiB | **2.4 MiB** | −95% | identical in both runs | +| post-soak steady state | 324.2 MiB | 322.5 MiB | −1% | [337.7, 310.7] / [280.7, 364.4] | +| boot idle | 119.4 MiB | 125.8 MiB | +5% | [130.0, 108.8] / [152.8, 98.8] | +| median board load | 526 ms | 523 ms | −1% | [389, 663] / [401, 645] | +| CPU over the run | 19.5 s | 20.4 s | +4% | [19.6, 19.4] / [20.3, 20.4] | + +Every peak metric separates cleanly — the two sides' ranges do not overlap. Two rows do +**not** support a claim and are listed so they are not read as one: + +- **post-soak steady state is inconclusive.** One "after" run finished at 364.4 MiB, above + a "before" run at 310.7. This change targets a transient spike, so a steady-state effect + was not expected. +- **boot idle is noise.** The spread is ±20 MiB on *identical* code (before: 130.0 vs + 108.8), which is wider than the apparent difference. Umami's code is not even loaded at + boot; it arrives when a board containing the widget renders. + +The cost is **+4% CPU**, from turning one large request into several smaller ones. Board +load latency did not regress. + +A prediction worth recording, because it confirms the mechanism rather than just the +outcome: at `pageSize=1000` the largest allocation measured 0.7 MiB, and at 5,000 it +measured 2.4 MiB — 5× the page size, 3.4× the bytes, exactly as a per-page bound implies. + ## Measured but not pursued diff --git a/packages/integrations/src/umami/test/umami-integration.spec.ts b/packages/integrations/src/umami/test/umami-integration.spec.ts new file mode 100644 index 000000000..7f228eb16 --- /dev/null +++ b/packages/integrations/src/umami/test/umami-integration.spec.ts @@ -0,0 +1,179 @@ +import { Request, Response } from "undici"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http"; + +import { UmamiIntegration } from "../umami-integration"; + +vi.mock("@homarr/core/infrastructure/http", () => ({ + fetchWithTrustedCertificatesAsync: vi.fn(), +})); + +const mockFetch = vi.mocked(fetchWithTrustedCertificatesAsync); + +/** Matches the page size the integration asks for; a full page is what triggers the next request. */ +const PAGE_SIZE = 5_000; +const WEBSITE_ID = "site-1"; + +interface RequestedUrl { + path: string; + params: URLSearchParams; +} + +const requestedUrls: RequestedUrl[] = []; + +/** + * Routes on the last path segment(s) and records every request, so tests can assert both the + * result and how many round trips it took — the point of these changes is the number and size + * of requests, not only the value returned. + */ +const setupMockFetch = (handler: (requested: RequestedUrl) => unknown) => { + mockFetch.mockImplementation((url) => { + const urlString = typeof url === "string" ? url : url instanceof Request ? url.url : url.toString(); + const parsed = new URL(urlString); + const requested = { path: parsed.pathname, params: parsed.searchParams }; + requestedUrls.push(requested); + const body = handler(requested); + return Promise.resolve( + new Response(JSON.stringify(body ?? []), { + status: 200, + headers: { "content-type": "application/json" }, + }) as unknown as Awaited>, + ); + }); +}; + +const createIntegration = () => + new UmamiIntegration({ + id: "test-umami", + name: "Test Umami", + url: "https://umami.example.com/api", + externalUrl: null, + decryptedSecrets: [{ kind: "apiKey", value: "test-key" }], + }); + +/** A raw Umami Cloud event record, the shape that used to be downloaded 100k at a time. */ +const rawEvent = (index: number, eventName: string, hour: number) => ({ + id: `event-${index}`, + websiteId: WEBSITE_ID, + sessionId: `session-${index}`, + eventType: 2, + eventName, + createdAt: `2026-08-09T${String(hour).padStart(2, "0")}:30:00.000Z`, +}); + +describe("UmamiIntegration", () => { + beforeEach(() => { + vi.clearAllMocks(); + requestedUrls.length = 0; + }); + + describe("getEventNamesAsync", () => { + test("reads names from the aggregated metrics endpoint, never the raw event list", async () => { + setupMockFetch(({ path }) => { + if (path.endsWith("/metrics")) { + return [ + { x: "signup", y: 12 }, + { x: "purchase", y: 3 }, + { x: "signup", y: 1 }, + ]; + } + throw new Error(`unexpected request to ${path}`); + }); + + const result = await createIntegration().getEventNamesAsync(WEBSITE_ID); + + expect(result).toEqual(["purchase", "signup"]); + expect(requestedUrls).toHaveLength(1); + expect(requestedUrls[0]?.path).toContain("/metrics"); + expect(requestedUrls[0]?.params.get("type")).toBe("event"); + // The regression this guards: listing /events pulled every raw record to derive names. + expect(requestedUrls.some((requested) => requested.path.endsWith("/events"))).toBe(false); + }); + + test("drops empty names and returns them sorted", async () => { + setupMockFetch(() => [ + { x: "beta", y: 1 }, + { x: null, y: 5 }, + { x: "alpha", y: 2 }, + ]); + + expect(await createIntegration().getEventNamesAsync(WEBSITE_ID)).toEqual(["alpha", "beta"]); + }); + }); + + describe("getMultiEventTimeSeriesAsync", () => { + test("pages through raw event records and aggregates across pages", async () => { + // Page 1 is full, so a second page is requested; page 2 is short, which ends it. + const firstPage = Array.from({ length: PAGE_SIZE }, (_, index) => rawEvent(index, "signup", 10)); + const secondPage = Array.from({ length: 5 }, (_, index) => rawEvent(PAGE_SIZE + index, "signup", 11)); + + setupMockFetch(({ path, params }) => { + if (!path.endsWith("/events")) return []; + return { data: params.get("page") === "1" ? firstPage : secondPage }; + }); + + const result = await createIntegration().getMultiEventTimeSeriesAsync("site-1", "24h", ["signup"]); + + expect(requestedUrls.map((requested) => requested.params.get("page"))).toEqual(["1", "2"]); + expect(requestedUrls[0]?.params.get("pageSize")).toBe(String(PAGE_SIZE)); + expect(result).toHaveLength(1); + // Both pages must be counted: a full page in the 10:00 bucket, 5 in the 11:00 bucket. + const total = result[0]?.dataPoints.reduce((sum, point) => sum + point.y, 0); + expect(total).toBe(PAGE_SIZE + 5); + expect(result[0]?.dataPoints).toHaveLength(2); + }); + + test("stops after one page when the instance returns aggregated buckets", async () => { + setupMockFetch(() => [ + { x: "2026-08-09 10:00:00", y: 4 }, + { x: "2026-08-09 11:00:00", y: 7 }, + ]); + + const result = await createIntegration().getMultiEventTimeSeriesAsync(WEBSITE_ID, "24h", ["signup"]); + + expect(requestedUrls).toHaveLength(1); + expect(result[0]?.dataPoints).toEqual([ + { x: "2026-08-09 10:00:00", y: 4 }, + { x: "2026-08-09 11:00:00", y: 7 }, + ]); + }); + + test("stops instead of double counting when the instance ignores the page parameter", async () => { + // Always returns the same full page, whatever page was asked for. + const page = Array.from({ length: PAGE_SIZE }, (_, index) => rawEvent(index, "signup", 10)); + setupMockFetch(() => ({ data: page })); + + const result = await createIntegration().getMultiEventTimeSeriesAsync(WEBSITE_ID, "24h", ["signup"]); + + // Page 1 is consumed, page 2 is recognised as a repeat and discarded. + expect(requestedUrls).toHaveLength(2); + expect(result[0]?.dataPoints.reduce((sum, point) => sum + point.y, 0)).toBe(PAGE_SIZE); + }); + + test("requests event series one at a time rather than all at once", async () => { + let inFlight = 0; + let maxInFlight = 0; + mockFetch.mockImplementation(async () => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((resolve) => setTimeout(resolve, 1)); + inFlight--; + return new Response(JSON.stringify([{ x: "2026-08-09 10:00:00", y: 1 }]), { + status: 200, + headers: { "content-type": "application/json" }, + }) as unknown as Awaited>; + }); + + const result = await createIntegration().getMultiEventTimeSeriesAsync(WEBSITE_ID, "24h", [ + "signup", + "purchase", + "refund", + ]); + + expect(result).toHaveLength(3); + // Concurrent fan-out multiplied resident event pages by the number of tracked events. + expect(maxInFlight).toBe(1); + }); + }); +}); diff --git a/packages/integrations/src/umami/umami-integration.ts b/packages/integrations/src/umami/umami-integration.ts index 730cd5477..2cd049e54 100644 --- a/packages/integrations/src/umami/umami-integration.ts +++ b/packages/integrations/src/umami/umami-integration.ts @@ -30,8 +30,39 @@ import { // Umami event type ID for custom events (type 1 = page view, type 2 = custom event) const UMAMI_CUSTOM_EVENT_TYPE = 2; +/** + * Records per page when /events returns raw event rows rather than aggregated buckets. + * + * Requesting the whole list at once makes the server hold all of it: one measured refresh + * pulled a 46.3 MiB JSON body assembled from 3,946 socket chunks, and `Response.json()` + * keeps the chunk list and the joined copy alive at the same time, so the true peak was + * roughly double. Pages are folded into the bucket map and released, so only one page is + * ever resident. + * + * 5,000 rather than something smaller because this trades against round trips, not just + * memory: a page is ~2.3 MiB, so the peak stays small, while any site with fewer than 5,000 + * events in the window — which is most of them — still completes in a single request. + */ +const UMAMI_EVENT_PAGE_SIZE = 5_000; +/** + * Ceiling on pages, so a very busy site degrades into an undercount that is logged rather + * than into unbounded memory. Preserves the previous 100k-record ceiling. + */ +const UMAMI_EVENT_PAGE_MAX = 20; + const logger = createLogger({ module: "umami-integration" }); +const pad = (number: number) => String(number).padStart(2, "0"); + +/** Umami returns either a bare array or `{ data: [...] }` depending on endpoint and version. */ +const extractDataArray = (json: unknown): unknown[] => { + if (Array.isArray(json)) return json; + if (json && typeof json === "object" && "data" in json && Array.isArray((json as { data: unknown }).data)) { + return (json as { data: unknown[] }).data; + } + return []; +}; + export class UmamiIntegration extends Integration { protected async testingAsync(input: IntegrationTestingInput): Promise { const authHeaders = await this.getAuthHeadersAsync(); @@ -168,32 +199,29 @@ export class UmamiIntegration extends Integration { }; } + /** + * Distinct custom event names over the last 30 days. + * + * Uses the aggregated `/metrics?type=event` endpoint, which returns one row per event + * name. The obvious alternative — listing `/events` and collecting the distinct + * `eventName` values — downloads every raw event record to compute a handful of strings: + * on a busy site that measured a 46.3 MiB JSON response, and `Response.json()` holds the + * socket chunks and the joined copy simultaneously, so the real cost was about twice that. + * + * The two are not exactly equivalent: /metrics returns the top names by count, so an + * extremely rare event could fall outside it. The list endpoint was bounded too — it + * capped at a record count, which on a busy site does not even span the 30-day window — + * so this trades one incomplete answer for a much cheaper one, in a picker where the + * names worth choosing are the ones with traffic. + */ public async getEventNamesAsync(websiteId: string): Promise { const authHeaders = await this.getAuthHeadersAsync(); - const now = Date.now(); + const endAt = Date.now(); const startAt = dayjs().subtract(30, "day").valueOf(); - const url = this.url(`/websites/${websiteId}/events`, { - startAt: startAt.toString(), - endAt: now.toString(), - pageSize: "100000", - }); - const response = await fetchWithTrustedCertificatesAsync(url, { headers: authHeaders }); - if (!response.ok) { - logger.warn("Failed to load event names", { status: response.status, websiteId }); - return []; - } - const json = await response.json(); - const rawArray: unknown[] = Array.isArray(json) - ? json - : json && typeof json === "object" && "data" in json && Array.isArray((json as { data: unknown }).data) - ? (json as { data: unknown[] }).data - : []; + const metrics = await this.getWebsiteMetricsAsync(websiteId, startAt, endAt, "event", authHeaders); const names = new Set(); - for (const item of rawArray) { - const record = item as Record; - if (typeof record.eventName === "string" && record.eventName) { - names.add(record.eventName); - } + for (const metric of metrics) { + if (metric.x) names.add(metric.x); } return [...names].sort(); } @@ -243,19 +271,23 @@ export class UmamiIntegration extends Integration { ): Promise { const authHeaders = await this.getAuthHeadersAsync(); const { startAt, endAt, unit } = this.computeTimeRange(timeFrame); - const results = await Promise.all( - eventNames.map(async (eventName) => { - const dataPoints = await this.getWebsiteEventTimeSeriesAsync( - websiteId, - startAt, - endAt, - unit, - eventName, - authHeaders, - ); - return { eventName, dataPoints: dataPoints ?? [] }; - }), - ); + + // Sequentially, not with Promise.all over every name. Each series can be several pages + // of raw events on a busy site, and running them all at once multiplies the resident + // pages by the number of tracked events — which is how one widget refresh was able to + // put well over a hundred megabytes in flight at the same time. + const results: UmamiEventSeries[] = []; + for (const eventName of eventNames) { + const dataPoints = await this.getWebsiteEventTimeSeriesAsync( + websiteId, + startAt, + endAt, + unit, + eventName, + authHeaders, + ); + results.push({ eventName, dataPoints: dataPoints ?? [] }); + } return results; } @@ -298,6 +330,16 @@ export class UmamiIntegration extends Integration { return []; } + /** + * Per-event time series. + * + * Two response shapes coexist: + * - Umami self-hosted returns aggregated buckets: { x: timestamp, y: count } + * - Umami Cloud returns raw event records: { createdAt, eventName, eventType } + * The raw records are bucketed here onto the same grid so the result joins cleanly with + * /pageviews timestamps. Raw pages are folded in one at a time — see UMAMI_EVENT_PAGE_SIZE + * for why fetching them in a single page was the largest allocation in the server. + */ private async getWebsiteEventTimeSeriesAsync( websiteId: string, startAt: number, @@ -306,79 +348,98 @@ export class UmamiIntegration extends Integration { eventName: string, authHeaders: Record, ): Promise { - const url = this.url(`/websites/${websiteId}/events`, { - startAt: startAt.toString(), - endAt: endAt.toString(), - unit, - timezone: "UTC", - eventName, - pageSize: "100000", - }); - const response = await fetchWithTrustedCertificatesAsync(url, { - headers: authHeaders, - }); + // Format must match Umami's pageview x-axis timestamps: "YYYY-MM-DD HH:MM:SS" + const truncate = (isoDate: string): string => { + const date = new Date(isoDate); + const dateString = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`; + if (unit === "hour") { + return `${dateString} ${pad(date.getUTCHours())}:00:00`; + } + if (unit === "month") { + return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-01 00:00:00`; + } + return `${dateString} 00:00:00`; + }; - if (!response.ok) { - logger.warn("Failed to load event time series", { status: response.status, websiteId, eventName }); - return undefined; - } + const buckets = new Map(); + let previousFirstId: string | undefined; + let truncated = false; - const json = await response.json(); - const rawArray: unknown[] = Array.isArray(json) - ? json - : json && typeof json === "object" && "data" in json && Array.isArray((json as { data: unknown }).data) - ? (json as { data: unknown[] }).data - : []; + for (let page = 1; ; page++) { + const url = this.url(`/websites/${websiteId}/events`, { + startAt: startAt.toString(), + endAt: endAt.toString(), + unit, + timezone: "UTC", + eventName, + page: page.toString(), + pageSize: UMAMI_EVENT_PAGE_SIZE.toString(), + }); + const response = await fetchWithTrustedCertificatesAsync(url, { + headers: authHeaders, + }); - if (rawArray.length === 0) return undefined; + if (!response.ok) { + logger.warn("Failed to load event time series", { status: response.status, websiteId, eventName, page }); + break; + } - const firstItem = rawArray[0] as Record; + const rawArray = extractDataArray(await response.json()); + if (rawArray.length === 0) break; - // Two response shapes coexist: - // - Umami self-hosted returns aggregated buckets: { x: timestamp, y: count } - // - Umami Cloud returns raw event records: { createdAt, eventName, eventType } - // We aggregate the raw Cloud records ourselves into the same bucket grid so - // the result joins cleanly with /pageviews timestamps. + const firstItem = rawArray[0] as Record; - // Aggregated { x, y } format — parse directly (self-hosted instances) - if ("x" in firstItem && "y" in firstItem) { - const points = umamiPageviewDataPointSchema.array().parse(rawArray); - return points.length === 0 ? undefined : points; - } + // Aggregated { x, y }: this instance ignored paging and returned the whole series, so + // there is nothing to accumulate across pages. + if ("x" in firstItem && "y" in firstItem) { + const points = umamiPageviewDataPointSchema.array().parse(rawArray); + return points.length === 0 ? undefined : points; + } + if (!("createdAt" in firstItem)) return undefined; - // Raw event records — aggregate by time bucket (Umami Cloud) - if ("createdAt" in firstItem) { - const pad = (number: number) => String(number).padStart(2, "0"); - // Format must match Umami's pageview x-axis timestamps: "YYYY-MM-DD HH:MM:SS" - const truncate = (isoDate: string): string => { - const date = new Date(isoDate); - const dateString = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`; - if (unit === "hour") { - return `${dateString} ${pad(date.getUTCHours())}:00:00`; - } - if (unit === "month") { - return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-01 00:00:00`; - } - return `${dateString} 00:00:00`; - }; + // An instance that accepts pageSize but ignores page would hand back the same rows + // forever, and each pass would count them into the buckets again. Falls back to + // session+timestamp where records carry no id, so the guard still holds. + const firstId = + typeof firstItem.id === "string" + ? firstItem.id + : typeof firstItem.createdAt === "string" + ? `${String(firstItem.sessionId ?? "")}@${firstItem.createdAt}` + : undefined; + if (page > 1 && firstId !== undefined && firstId === previousFirstId) { + logger.warn("Umami ignored event paging, stopping after the first page", { websiteId, eventName }); + break; + } + previousFirstId = firstId; - const buckets = new Map(); for (const item of rawArray) { const event = item as { eventType?: number; eventName?: string; createdAt: string }; if (event.eventType !== UMAMI_CUSTOM_EVENT_TYPE || event.eventName !== eventName) continue; const bucket = truncate(event.createdAt); buckets.set(bucket, (buckets.get(bucket) ?? 0) + 1); } - if (buckets.size === 0) return undefined; - return ( - Array.from(buckets.entries()) - .sort(([itemA], [itemB]) => itemA.localeCompare(itemB)) - // eslint-disable-next-line id-length - .map(([x, y]) => ({ x, y })) - ); + + if (rawArray.length < UMAMI_EVENT_PAGE_SIZE) break; + if (page >= UMAMI_EVENT_PAGE_MAX) { + truncated = true; + break; + } } - return undefined; + if (truncated) { + logger.warn("Event time series hit the page ceiling, counts are a lower bound", { + websiteId, + eventName, + maxRecords: UMAMI_EVENT_PAGE_SIZE * UMAMI_EVENT_PAGE_MAX, + }); + } + if (buckets.size === 0) return undefined; + return ( + Array.from(buckets.entries()) + .sort(([itemA], [itemB]) => itemA.localeCompare(itemB)) + // eslint-disable-next-line id-length + .map(([x, y]) => ({ x, y })) + ); } private async getWebsiteStatsAsync( diff --git a/scripts/benchmarks/analyze-heap-dominators.mjs b/scripts/benchmarks/analyze-heap-dominators.mjs new file mode 100644 index 000000000..d977087f4 --- /dev/null +++ b/scripts/benchmarks/analyze-heap-dominators.mjs @@ -0,0 +1,245 @@ +// Computes real retained sizes from a .heapsnapshot and reports the biggest retainers. +// +// node --max-old-space-size=8192 scripts/benchmarks/analyze-heap-dominators.mjs +// +// Shallow-size histograms answer "what kind of objects exist" and are misleading for +// "what is costing memory": 4 MiB of `Object` spread over 95k instances says nothing about +// which subsystem is holding them. Retained size — everything that would be freed if a node +// were released — is the number that maps onto a cause, and it needs the dominator tree. +// +// Same algorithm Chrome DevTools uses: postorder the reachable graph, solve dominators by +// iterated predecessor intersection, then accumulate self sizes up the dominator tree. +import { readFileSync } from "node:fs"; + +const file = process.argv[2]; +if (!file) throw new Error("usage: analyze-heap-dominators.mjs [topN]"); +const topN = Number(process.argv[3] ?? 40); + +const snap = JSON.parse(readFileSync(file, "utf8")); +const meta = snap.snapshot.meta; + +const nf = meta.node_fields; +const nStride = nf.length; +const nType = nf.indexOf("type"); +const nName = nf.indexOf("name"); +const nSelf = nf.indexOf("self_size"); +const nEdgeCount = nf.indexOf("edge_count"); +const nodeTypes = meta.node_types[0]; + +const ef = meta.edge_fields; +const eStride = ef.length; +const eType = ef.indexOf("type"); +const eName = ef.indexOf("name_or_index"); +const eTo = ef.indexOf("to_node"); +const edgeTypes = meta.edge_types[0]; + +const nodes = snap.nodes; +const edges = snap.edges; +const strings = snap.strings; +const N = snap.snapshot.node_count; +const E = snap.snapshot.edge_count; + +const nodeName = (o) => strings[nodes[o * nStride + nName]] ?? ""; +const nodeType = (o) => nodeTypes[nodes[o * nStride + nType]] ?? "?"; +const selfSize = (o) => nodes[o * nStride + nSelf]; + +console.log(`file: ${file}`); +console.log(`nodes: ${N.toLocaleString()} edges: ${E.toLocaleString()}`); + +// firstEdge[o] = index of node o's first edge. +const firstEdge = new Uint32Array(N + 1); +for (let o = 0, acc = 0; o < N; o++) { + firstEdge[o] = acc; + acc += nodes[o * nStride + nEdgeCount]; +} +firstEdge[N] = E; + +/** + * Weak edges do not keep their target alive, so counting them would let a WeakMap or a + * weak handle claim retention it does not actually have. + */ +const isWeak = (e) => edgeTypes[edges[e * eStride + eType]] === "weak"; + +// ---------------------------------------------------------------- postorder from the root +const postOrder = new Int32Array(N).fill(-1); // node ordinal -> postorder index +const postOrderNodes = new Int32Array(N); +{ + const visited = new Uint8Array(N); + // Explicit stack: the graph is far too deep for recursion. + const stackNode = new Int32Array(N + 1); + const stackEdge = new Uint32Array(N + 1); + let post = 0; + let top = 0; + stackNode[0] = 0; + stackEdge[0] = firstEdge[0]; + visited[0] = 1; + while (top >= 0) { + const node = stackNode[top]; + let e = stackEdge[top]; + let descended = false; + while (e < firstEdge[node + 1]) { + if (isWeak(e)) { + e++; + continue; + } + const child = edges[e * eStride + eTo] / nStride; + e++; + if (!visited[child]) { + visited[child] = 1; + stackEdge[top] = e; + top++; + stackNode[top] = child; + stackEdge[top] = firstEdge[child]; + descended = true; + break; + } + } + if (descended) continue; + stackEdge[top] = e; + if (e >= firstEdge[node + 1]) { + postOrder[node] = post; + postOrderNodes[post] = node; + post++; + top--; + } + } + console.log(`reachable from root: ${post.toLocaleString()} nodes`); + // Unreachable nodes keep postOrder -1 and are excluded from the dominator solve. + var reachableCount = post; +} + +// ------------------------------------------------- reverse edges, restricted to reachable +const inDeg = new Uint32Array(N); +for (let src = 0; src < N; src++) { + if (postOrder[src] === -1) continue; + for (let e = firstEdge[src]; e < firstEdge[src + 1]; e++) { + if (isWeak(e)) continue; + const dst = edges[e * eStride + eTo] / nStride; + if (postOrder[dst] !== -1) inDeg[dst]++; + } +} +const revStart = new Uint32Array(N + 1); +{ + let acc = 0; + for (let o = 0; o < N; o++) { + revStart[o] = acc; + acc += inDeg[o]; + } + revStart[N] = acc; +} +const revTo = new Uint32Array(revStart[N]); +{ + const cursor = Uint32Array.from(revStart.subarray(0, N)); + for (let src = 0; src < N; src++) { + if (postOrder[src] === -1) continue; + for (let e = firstEdge[src]; e < firstEdge[src + 1]; e++) { + if (isWeak(e)) continue; + const dst = edges[e * eStride + eTo] / nStride; + if (postOrder[dst] !== -1) revTo[cursor[dst]++] = src; + } + } +} + +// -------------------------------------------------------------------- dominators +const rootPost = postOrder[0]; +const dom = new Int32Array(reachableCount).fill(-1); +dom[rootPost] = rootPost; +{ + const intersect = (a, b) => { + // Higher postorder index is closer to the root, so walking the smaller one upward + // converges on the nearest common dominator. + while (a !== b) { + if (a < b) a = dom[a]; + else b = dom[b]; + if (a === -1 || b === -1) return -1; + } + return a; + }; + let changed = true; + let sweeps = 0; + while (changed) { + changed = false; + sweeps++; + for (let i = reachableCount - 1; i >= 0; i--) { + if (i === rootPost) continue; + const node = postOrderNodes[i]; + let newDom = -1; + for (let r = revStart[node]; r < revStart[node + 1]; r++) { + const predPost = postOrder[revTo[r]]; + if (predPost === -1 || dom[predPost] === -1) continue; + newDom = newDom === -1 ? predPost : intersect(newDom, predPost); + if (newDom === -1) break; + } + if (newDom !== -1 && dom[i] !== newDom) { + dom[i] = newDom; + changed = true; + } + } + } + console.log(`dominator solve converged in ${sweeps} sweeps`); +} + +// ---------------------------------------------------------------- retained sizes +const retained = new Float64Array(reachableCount); +for (let i = 0; i < reachableCount; i++) retained[i] = selfSize(postOrderNodes[i]); +for (let i = 0; i < reachableCount; i++) { + if (i === rootPost) continue; + const d = dom[i]; + if (d !== -1 && d !== i) retained[d] += retained[i]; +} + +const MiB = (bytes) => (bytes / 1048576).toFixed(2); + +// A node whose dominator is the root is a top-level owner: its retained size is memory that +// only it holds. Those are the entries worth naming. +const label = (o) => { + const type = nodeType(o); + const name = nodeName(o) || "(anonymous)"; + return `${type}:${name}`; +}; + +const candidates = []; +for (let i = 0; i < reachableCount; i++) { + if (i === rootPost) continue; + if (retained[i] < 262144) continue; + candidates.push(i); +} +candidates.sort((a, b) => retained[b] - retained[a]); + +console.log(`\n=== largest retainers (retained size = freed if this went away) ===`); +console.log(`${"retained".padStart(10)} ${"self".padStart(9)} what`); +const shown = []; +for (const i of candidates) { + const node = postOrderNodes[i]; + // Skip a node whose dominator is already listed: its bytes are counted in that ancestor, + // so printing both would double-report the same memory. + let ancestor = dom[i]; + let covered = false; + for (let guard = 0; guard < 200 && ancestor !== -1 && ancestor !== rootPost; guard++) { + if (shown.includes(ancestor)) { + covered = true; + break; + } + ancestor = dom[ancestor]; + } + if (covered) continue; + shown.push(i); + console.log(`${MiB(retained[i]).padStart(10)} ${MiB(selfSize(node)).padStart(9)} ${label(node).slice(0, 96)}`); + if (shown.length >= topN) break; +} + +// Grouping by constructor over retained size shows which *kind* of thing dominates, which +// is what a shallow histogram cannot express. +const byName = new Map(); +for (let i = 0; i < reachableCount; i++) { + const node = postOrderNodes[i]; + const key = label(node); + const acc = byName.get(key) ?? { retained: 0, self: 0, count: 0 }; + acc.self += selfSize(node); + acc.count++; + byName.set(key, acc); +} +console.log(`\n=== self size by constructor, top 15 (for cross-checking the above) ===`); +for (const [name, acc] of [...byName.entries()].sort((a, b) => b[1].self - a[1].self).slice(0, 15)) { + console.log(`${MiB(acc.self).padStart(10)} MiB ${String(acc.count).padStart(8)}x ${name.slice(0, 80)}`); +} diff --git a/scripts/benchmarks/analyze-probe-stages.mjs b/scripts/benchmarks/analyze-probe-stages.mjs new file mode 100644 index 000000000..66daedb1a --- /dev/null +++ b/scripts/benchmarks/analyze-probe-stages.mjs @@ -0,0 +1,90 @@ +// Turns the per-stage probe captures of a stress run into one table, so growth can be +// attributed to a specific part of the process rather than to "memory". +// +// node scripts/benchmarks/analyze-probe-stages.mjs .bench/stress/