perf(umami): stop buffering 100k raw events per widget refresh

Profiling the container end to end found one allocation site responsible for
most of the peak: a single 46.3 MiB JSON body, assembled from 3,946 socket
chunks, inside undici's fullyReadBody. That function holds the chunk list and
the joined copy at the same instant, so a body of size N costs about 2N.

The source was two Umami calls asking for pageSize=100000, with
getMultiEventTimeSeriesAsync running them for every tracked event name
concurrently via Promise.all. getEventNamesAsync was downloading every raw
event of the last 30 days purely to collect a handful of distinct name strings.

  - getEventNamesAsync now reads the aggregated /metrics?type=event endpoint.
  - getWebsiteEventTimeSeriesAsync pages 5,000 records at a time, folding each
    page into the bucket map before requesting the next.
  - getMultiEventTimeSeriesAsync runs series sequentially.

Instances that return aggregated {x,y} buckets still answer in one request, an
instance that accepts pageSize but ignores page is detected rather than
double-counted, and hitting the page ceiling logs instead of silently
undercounting.

Alternated runs, 2 per side, on a restored real backup:

  peak container       498.9 -> 398.5 MiB  (-20%)
  peak node RSS        452.4 -> 353.3 MiB  (-22%)
  peak during 4 tabs   498.5 -> 356.3 MiB  (-29%)
  peak arrayBuffers     95.8 ->   6.6 MiB  (-93%)
  largest allocation    46.4 ->   2.4 MiB  (-95%)

Costs +4% CPU from the extra round trips; board load latency unchanged.
Post-soak steady state and boot idle are unchanged within run-to-run spread.

Adds the attribution tooling this needed: attribute-memory.mts reconciles
cgroup -> processes -> address-space mappings -> V8 spaces -> loaded code,
memory-probe.cjs is an injectable diagnostics preload (never shipped) that
tracks buffer and fetch allocation sites, analyze-heap-dominators.mjs computes
real retained sizes, and compare-stress-runs.mjs reports alternated runs with
their spread.

Also corrects an earlier claim in PHASE2-FINDINGS.md: the reported "7.86 MiB of
duplicate chunk sources" was an artefact of V8 truncating string contents to
~1 KiB in heap snapshots. There are zero byte-identical duplicates among all
3,055 server JS files. The real doubling is character width - one codepoint
above U+00FF forces a whole source string to two bytes per character, which
costs 9.45 MiB across 80 loaded files.

Adds 6 tests; this integration had none.

packages/integrations/src/umami/test/umami-integration.spec.ts

@@ -0,0 +1,179 @@
+import { Request, Response } from "undici";
+import { beforeEach, describe, expect, test, vi } from "vitest";
+
+import { fetchWithTrustedCertificatesAsync } from "@homarr/core/infrastructure/http";
+
+import { UmamiIntegration } from "../umami-integration";
+
+vi.mock("@homarr/core/infrastructure/http", () => ({
+  fetchWithTrustedCertificatesAsync: vi.fn(),
+}));
+
+const mockFetch = vi.mocked(fetchWithTrustedCertificatesAsync);
+
+/** Matches the page size the integration asks for; a full page is what triggers the next request. */
+const PAGE_SIZE = 5_000;
+const WEBSITE_ID = "site-1";
+
+interface RequestedUrl {
+  path: string;
+  params: URLSearchParams;
+}
+
+const requestedUrls: RequestedUrl[] = [];
+
+/**
+ * Routes on the last path segment(s) and records every request, so tests can assert both the
+ * result and how many round trips it took — the point of these changes is the number and size
+ * of requests, not only the value returned.
+ */
+const setupMockFetch = (handler: (requested: RequestedUrl) => unknown) => {
+  mockFetch.mockImplementation((url) => {
+    const urlString = typeof url === "string" ? url : url instanceof Request ? url.url : url.toString();
+    const parsed = new URL(urlString);
+    const requested = { path: parsed.pathname, params: parsed.searchParams };
+    requestedUrls.push(requested);
+    const body = handler(requested);
+    return Promise.resolve(
+      new Response(JSON.stringify(body ?? []), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      }) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>,
+    );
+  });
+};
+
+const createIntegration = () =>
+  new UmamiIntegration({
+    id: "test-umami",
+    name: "Test Umami",
+    url: "https://umami.example.com/api",
+    externalUrl: null,
+    decryptedSecrets: [{ kind: "apiKey", value: "test-key" }],
+  });
+
+/** A raw Umami Cloud event record, the shape that used to be downloaded 100k at a time. */
+const rawEvent = (index: number, eventName: string, hour: number) => ({
+  id: `event-${index}`,
+  websiteId: WEBSITE_ID,
+  sessionId: `session-${index}`,
+  eventType: 2,
+  eventName,
+  createdAt: `2026-08-09T${String(hour).padStart(2, "0")}:30:00.000Z`,
+});
+
+describe("UmamiIntegration", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    requestedUrls.length = 0;
+  });
+
+  describe("getEventNamesAsync", () => {
+    test("reads names from the aggregated metrics endpoint, never the raw event list", async () => {
+      setupMockFetch(({ path }) => {
+        if (path.endsWith("/metrics")) {
+          return [
+            { x: "signup", y: 12 },
+            { x: "purchase", y: 3 },
+            { x: "signup", y: 1 },
+          ];
+        }
+        throw new Error(`unexpected request to ${path}`);
+      });
+
+      const result = await createIntegration().getEventNamesAsync(WEBSITE_ID);
+
+      expect(result).toEqual(["purchase", "signup"]);
+      expect(requestedUrls).toHaveLength(1);
+      expect(requestedUrls[0]?.path).toContain("/metrics");
+      expect(requestedUrls[0]?.params.get("type")).toBe("event");
+      // The regression this guards: listing /events pulled every raw record to derive names.
+      expect(requestedUrls.some((requested) => requested.path.endsWith("/events"))).toBe(false);
+    });
+
+    test("drops empty names and returns them sorted", async () => {
+      setupMockFetch(() => [
+        { x: "beta", y: 1 },
+        { x: null, y: 5 },
+        { x: "alpha", y: 2 },
+      ]);
+
+      expect(await createIntegration().getEventNamesAsync(WEBSITE_ID)).toEqual(["alpha", "beta"]);
+    });
+  });
+
+  describe("getMultiEventTimeSeriesAsync", () => {
+    test("pages through raw event records and aggregates across pages", async () => {
+      // Page 1 is full, so a second page is requested; page 2 is short, which ends it.
+      const firstPage = Array.from({ length: PAGE_SIZE }, (_, index) => rawEvent(index, "signup", 10));
+      const secondPage = Array.from({ length: 5 }, (_, index) => rawEvent(PAGE_SIZE + index, "signup", 11));
+
+      setupMockFetch(({ path, params }) => {
+        if (!path.endsWith("/events")) return [];
+        return { data: params.get("page") === "1" ? firstPage : secondPage };
+      });
+
+      const result = await createIntegration().getMultiEventTimeSeriesAsync("site-1", "24h", ["signup"]);
+
+      expect(requestedUrls.map((requested) => requested.params.get("page"))).toEqual(["1", "2"]);
+      expect(requestedUrls[0]?.params.get("pageSize")).toBe(String(PAGE_SIZE));
+      expect(result).toHaveLength(1);
+      // Both pages must be counted: a full page in the 10:00 bucket, 5 in the 11:00 bucket.
+      const total = result[0]?.dataPoints.reduce((sum, point) => sum + point.y, 0);
+      expect(total).toBe(PAGE_SIZE + 5);
+      expect(result[0]?.dataPoints).toHaveLength(2);
+    });
+
+    test("stops after one page when the instance returns aggregated buckets", async () => {
+      setupMockFetch(() => [
+        { x: "2026-08-09 10:00:00", y: 4 },
+        { x: "2026-08-09 11:00:00", y: 7 },
+      ]);
+
+      const result = await createIntegration().getMultiEventTimeSeriesAsync(WEBSITE_ID, "24h", ["signup"]);
+
+      expect(requestedUrls).toHaveLength(1);
+      expect(result[0]?.dataPoints).toEqual([
+        { x: "2026-08-09 10:00:00", y: 4 },
+        { x: "2026-08-09 11:00:00", y: 7 },
+      ]);
+    });
+
+    test("stops instead of double counting when the instance ignores the page parameter", async () => {
+      // Always returns the same full page, whatever page was asked for.
+      const page = Array.from({ length: PAGE_SIZE }, (_, index) => rawEvent(index, "signup", 10));
+      setupMockFetch(() => ({ data: page }));
+
+      const result = await createIntegration().getMultiEventTimeSeriesAsync(WEBSITE_ID, "24h", ["signup"]);
+
+      // Page 1 is consumed, page 2 is recognised as a repeat and discarded.
+      expect(requestedUrls).toHaveLength(2);
+      expect(result[0]?.dataPoints.reduce((sum, point) => sum + point.y, 0)).toBe(PAGE_SIZE);
+    });
+
+    test("requests event series one at a time rather than all at once", async () => {
+      let inFlight = 0;
+      let maxInFlight = 0;
+      mockFetch.mockImplementation(async () => {
+        inFlight++;
+        maxInFlight = Math.max(maxInFlight, inFlight);
+        await new Promise((resolve) => setTimeout(resolve, 1));
+        inFlight--;
+        return new Response(JSON.stringify([{ x: "2026-08-09 10:00:00", y: 1 }]), {
+          status: 200,
+          headers: { "content-type": "application/json" },
+        }) as unknown as Awaited<ReturnType<typeof fetchWithTrustedCertificatesAsync>>;
+      });
+
+      const result = await createIntegration().getMultiEventTimeSeriesAsync(WEBSITE_ID, "24h", [
+        "signup",
+        "purchase",
+        "refund",
+      ]);
+
+      expect(result).toHaveLength(3);
+      // Concurrent fan-out multiplied resident event pages by the number of tracked events.
+      expect(maxInFlight).toBe(1);
+    });
+  });
+});

packages/integrations/src/umami/umami-integration.ts

@@ -30,8 +30,39 @@ import {
 // Umami event type ID for custom events (type 1 = page view, type 2 = custom event)
 const UMAMI_CUSTOM_EVENT_TYPE = 2;
 
+/**
+ * Records per page when /events returns raw event rows rather than aggregated buckets.
+ *
+ * Requesting the whole list at once makes the server hold all of it: one measured refresh
+ * pulled a 46.3 MiB JSON body assembled from 3,946 socket chunks, and `Response.json()`
+ * keeps the chunk list and the joined copy alive at the same time, so the true peak was
+ * roughly double. Pages are folded into the bucket map and released, so only one page is
+ * ever resident.
+ *
+ * 5,000 rather than something smaller because this trades against round trips, not just
+ * memory: a page is ~2.3 MiB, so the peak stays small, while any site with fewer than 5,000
+ * events in the window — which is most of them — still completes in a single request.
+ */
+const UMAMI_EVENT_PAGE_SIZE = 5_000;
+/**
+ * Ceiling on pages, so a very busy site degrades into an undercount that is logged rather
+ * than into unbounded memory. Preserves the previous 100k-record ceiling.
+ */
+const UMAMI_EVENT_PAGE_MAX = 20;
+
 const logger = createLogger({ module: "umami-integration" });
 
+const pad = (number: number) => String(number).padStart(2, "0");
+
+/** Umami returns either a bare array or `{ data: [...] }` depending on endpoint and version. */
+const extractDataArray = (json: unknown): unknown[] => {
+  if (Array.isArray(json)) return json;
+  if (json && typeof json === "object" && "data" in json && Array.isArray((json as { data: unknown }).data)) {
+    return (json as { data: unknown[] }).data;
+  }
+  return [];
+};
+
 export class UmamiIntegration extends Integration {
   protected async testingAsync(input: IntegrationTestingInput): Promise<TestingResult> {
     const authHeaders = await this.getAuthHeadersAsync();
@@ -168,32 +199,29 @@ export class UmamiIntegration extends Integration {
     };
   }
 
+  /**
+   * Distinct custom event names over the last 30 days.
+   *
+   * Uses the aggregated `/metrics?type=event` endpoint, which returns one row per event
+   * name. The obvious alternative — listing `/events` and collecting the distinct
+   * `eventName` values — downloads every raw event record to compute a handful of strings:
+   * on a busy site that measured a 46.3 MiB JSON response, and `Response.json()` holds the
+   * socket chunks and the joined copy simultaneously, so the real cost was about twice that.
+   *
+   * The two are not exactly equivalent: /metrics returns the top names by count, so an
+   * extremely rare event could fall outside it. The list endpoint was bounded too — it
+   * capped at a record count, which on a busy site does not even span the 30-day window —
+   * so this trades one incomplete answer for a much cheaper one, in a picker where the
+   * names worth choosing are the ones with traffic.
+   */
   public async getEventNamesAsync(websiteId: string): Promise<string[]> {
     const authHeaders = await this.getAuthHeadersAsync();
-    const now = Date.now();
+    const endAt = Date.now();
     const startAt = dayjs().subtract(30, "day").valueOf();
-    const url = this.url(`/websites/${websiteId}/events`, {
-      startAt: startAt.toString(),
-      endAt: now.toString(),
-      pageSize: "100000",
-    });
-    const response = await fetchWithTrustedCertificatesAsync(url, { headers: authHeaders });
-    if (!response.ok) {
-      logger.warn("Failed to load event names", { status: response.status, websiteId });
-      return [];
-    }
-    const json = await response.json();
-    const rawArray: unknown[] = Array.isArray(json)
-      ? json
-      : json && typeof json === "object" && "data" in json && Array.isArray((json as { data: unknown }).data)
-        ? (json as { data: unknown[] }).data
-        : [];
+    const metrics = await this.getWebsiteMetricsAsync(websiteId, startAt, endAt, "event", authHeaders);
     const names = new Set<string>();
-    for (const item of rawArray) {
-      const record = item as Record<string, unknown>;
-      if (typeof record.eventName === "string" && record.eventName) {
-        names.add(record.eventName);
-      }
+    for (const metric of metrics) {
+      if (metric.x) names.add(metric.x);
     }
     return [...names].sort();
   }
@@ -243,19 +271,23 @@ export class UmamiIntegration extends Integration {
   ): Promise<UmamiEventSeries[]> {
     const authHeaders = await this.getAuthHeadersAsync();
     const { startAt, endAt, unit } = this.computeTimeRange(timeFrame);
-    const results = await Promise.all(
-      eventNames.map(async (eventName) => {
-        const dataPoints = await this.getWebsiteEventTimeSeriesAsync(
-          websiteId,
-          startAt,
-          endAt,
-          unit,
-          eventName,
-          authHeaders,
-        );
-        return { eventName, dataPoints: dataPoints ?? [] };
-      }),
-    );
+
+    // Sequentially, not with Promise.all over every name. Each series can be several pages
+    // of raw events on a busy site, and running them all at once multiplies the resident
+    // pages by the number of tracked events — which is how one widget refresh was able to
+    // put well over a hundred megabytes in flight at the same time.
+    const results: UmamiEventSeries[] = [];
+    for (const eventName of eventNames) {
+      const dataPoints = await this.getWebsiteEventTimeSeriesAsync(
+        websiteId,
+        startAt,
+        endAt,
+        unit,
+        eventName,
+        authHeaders,
+      );
+      results.push({ eventName, dataPoints: dataPoints ?? [] });
+    }
     return results;
   }
 
@@ -298,6 +330,16 @@ export class UmamiIntegration extends Integration {
     return [];
   }
 
+  /**
+   * Per-event time series.
+   *
+   * Two response shapes coexist:
+   *   - Umami self-hosted returns aggregated buckets: { x: timestamp, y: count }
+   *   - Umami Cloud returns raw event records: { createdAt, eventName, eventType }
+   * The raw records are bucketed here onto the same grid so the result joins cleanly with
+   * /pageviews timestamps. Raw pages are folded in one at a time — see UMAMI_EVENT_PAGE_SIZE
+   * for why fetching them in a single page was the largest allocation in the server.
+   */
   private async getWebsiteEventTimeSeriesAsync(
     websiteId: string,
     startAt: number,
@@ -306,79 +348,98 @@ export class UmamiIntegration extends Integration {
     eventName: string,
     authHeaders: Record<string, string>,
   ): Promise<UmamiPageviewDataPoint[] | undefined> {
-    const url = this.url(`/websites/${websiteId}/events`, {
-      startAt: startAt.toString(),
-      endAt: endAt.toString(),
-      unit,
-      timezone: "UTC",
-      eventName,
-      pageSize: "100000",
-    });
-    const response = await fetchWithTrustedCertificatesAsync(url, {
-      headers: authHeaders,
-    });
+    // Format must match Umami's pageview x-axis timestamps: "YYYY-MM-DD HH:MM:SS"
+    const truncate = (isoDate: string): string => {
+      const date = new Date(isoDate);
+      const dateString = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`;
+      if (unit === "hour") {
+        return `${dateString} ${pad(date.getUTCHours())}:00:00`;
+      }
+      if (unit === "month") {
+        return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-01 00:00:00`;
+      }
+      return `${dateString} 00:00:00`;
+    };
 
-    if (!response.ok) {
-      logger.warn("Failed to load event time series", { status: response.status, websiteId, eventName });
-      return undefined;
-    }
+    const buckets = new Map<string, number>();
+    let previousFirstId: string | undefined;
+    let truncated = false;
 
-    const json = await response.json();
-    const rawArray: unknown[] = Array.isArray(json)
-      ? json
-      : json && typeof json === "object" && "data" in json && Array.isArray((json as { data: unknown }).data)
-        ? (json as { data: unknown[] }).data
-        : [];
+    for (let page = 1; ; page++) {
+      const url = this.url(`/websites/${websiteId}/events`, {
+        startAt: startAt.toString(),
+        endAt: endAt.toString(),
+        unit,
+        timezone: "UTC",
+        eventName,
+        page: page.toString(),
+        pageSize: UMAMI_EVENT_PAGE_SIZE.toString(),
+      });
+      const response = await fetchWithTrustedCertificatesAsync(url, {
+        headers: authHeaders,
+      });
 
-    if (rawArray.length === 0) return undefined;
+      if (!response.ok) {
+        logger.warn("Failed to load event time series", { status: response.status, websiteId, eventName, page });
+        break;
+      }
 
-    const firstItem = rawArray[0] as Record<string, unknown>;
+      const rawArray = extractDataArray(await response.json());
+      if (rawArray.length === 0) break;
 
-    // Two response shapes coexist:
-    //   - Umami self-hosted returns aggregated buckets: { x: timestamp, y: count }
-    //   - Umami Cloud returns raw event records: { createdAt, eventName, eventType }
-    // We aggregate the raw Cloud records ourselves into the same bucket grid so
-    // the result joins cleanly with /pageviews timestamps.
+      const firstItem = rawArray[0] as Record<string, unknown>;
 
-    // Aggregated { x, y } format — parse directly (self-hosted instances)
-    if ("x" in firstItem && "y" in firstItem) {
-      const points = umamiPageviewDataPointSchema.array().parse(rawArray);
-      return points.length === 0 ? undefined : points;
-    }
+      // Aggregated { x, y }: this instance ignored paging and returned the whole series, so
+      // there is nothing to accumulate across pages.
+      if ("x" in firstItem && "y" in firstItem) {
+        const points = umamiPageviewDataPointSchema.array().parse(rawArray);
+        return points.length === 0 ? undefined : points;
+      }
+      if (!("createdAt" in firstItem)) return undefined;
 
-    // Raw event records — aggregate by time bucket (Umami Cloud)
-    if ("createdAt" in firstItem) {
-      const pad = (number: number) => String(number).padStart(2, "0");
-      // Format must match Umami's pageview x-axis timestamps: "YYYY-MM-DD HH:MM:SS"
-      const truncate = (isoDate: string): string => {
-        const date = new Date(isoDate);
-        const dateString = `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`;
-        if (unit === "hour") {
-          return `${dateString} ${pad(date.getUTCHours())}:00:00`;
-        }
-        if (unit === "month") {
-          return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-01 00:00:00`;
-        }
-        return `${dateString} 00:00:00`;
-      };
+      // An instance that accepts pageSize but ignores page would hand back the same rows
+      // forever, and each pass would count them into the buckets again. Falls back to
+      // session+timestamp where records carry no id, so the guard still holds.
+      const firstId =
+        typeof firstItem.id === "string"
+          ? firstItem.id
+          : typeof firstItem.createdAt === "string"
+            ? `${String(firstItem.sessionId ?? "")}@${firstItem.createdAt}`
+            : undefined;
+      if (page > 1 && firstId !== undefined && firstId === previousFirstId) {
+        logger.warn("Umami ignored event paging, stopping after the first page", { websiteId, eventName });
+        break;
+      }
+      previousFirstId = firstId;
 
-      const buckets = new Map<string, number>();
       for (const item of rawArray) {
         const event = item as { eventType?: number; eventName?: string; createdAt: string };
         if (event.eventType !== UMAMI_CUSTOM_EVENT_TYPE || event.eventName !== eventName) continue;
         const bucket = truncate(event.createdAt);
         buckets.set(bucket, (buckets.get(bucket) ?? 0) + 1);
       }
-      if (buckets.size === 0) return undefined;
-      return (
-        Array.from(buckets.entries())
-          .sort(([itemA], [itemB]) => itemA.localeCompare(itemB))
-          // eslint-disable-next-line id-length
-          .map(([x, y]) => ({ x, y }))
-      );
+
+      if (rawArray.length < UMAMI_EVENT_PAGE_SIZE) break;
+      if (page >= UMAMI_EVENT_PAGE_MAX) {
+        truncated = true;
+        break;
+      }
     }
 
-    return undefined;
+    if (truncated) {
+      logger.warn("Event time series hit the page ceiling, counts are a lower bound", {
+        websiteId,
+        eventName,
+        maxRecords: UMAMI_EVENT_PAGE_SIZE * UMAMI_EVENT_PAGE_MAX,
+      });
+    }
+    if (buckets.size === 0) return undefined;
+    return (
+      Array.from(buckets.entries())
+        .sort(([itemA], [itemB]) => itemA.localeCompare(itemB))
+        // eslint-disable-next-line id-length
+        .map(([x, y]) => ({ x, y }))
+    );
   }
 
   private async getWebsiteStatsAsync(