homarr/scripts/benchmarks/memory-probe.cjs

/**
 * Preloaded into the server with `NODE_OPTIONS=--require /probe/memory-probe.cjs` so
 * memory attribution can reach things the inspector cannot.
 *
 * Node's inspector evaluates in the main context, where `require` is a module-scoped
 * variable rather than a global — and the Next standalone entry does not set
 * `process.mainModule` — so `v8.getHeapSpaceStatistics()` and the CommonJS module cache
 * are both unreachable from a plain `Runtime.evaluate`. A preload runs as a real module,
 * so it can park what is needed on `globalThis`, which the inspector *can* see.
 *
 * Diagnostics only. Never part of a shipped image: it is mounted in by the benchmark
 * harness and costs a few hundred KiB of retained bookkeeping.
 */
const v8 = require("v8");
const fs = require("fs");
const path = require("path");
const Module = require("module");

/** Cache stat() results: the same chunk is asked about at every capture stage. */
const sizeCache = new Map();
const fileSize = (file) => {
  if (sizeCache.has(file)) return sizeCache.get(file);
  let size = 0;
  try {
    size = fs.statSync(file).size;
  } catch {
    size = 0;
  }
  sizeCache.set(file, size);
  return size;
};

/**
 * Which package a loaded file belongs to. Server bundle chunks live under
 * `.next/server/chunks`, so they are grouped separately from real node_modules
 * packages — a chunk's cost belongs to "what Next inlined", not to one dependency.
 */
const attributeFile = (file) => {
  const nodeModules = file.lastIndexOf("/node_modules/");
  if (nodeModules !== -1) {
    const rest = file.slice(nodeModules + "/node_modules/".length);
    const parts = rest.split("/");
    const pkg = parts[0].startsWith("@") ? `${parts[0]}/${parts[1]}` : parts[0];
    if (pkg === "next") {
      if (rest.includes("/dist/compiled/")) return "next/dist/compiled (vendored deps)";
      return "next/dist (framework runtime)";
    }
    return `node_modules: ${pkg}`;
  }
  if (file.includes("/.next/server/chunks/")) return ".next/server/chunks (bundled app code)";
  if (file.includes("/.next/server/app/")) return ".next/server/app (route bundles)";
  if (file.includes("/.next/server/pages/")) return ".next/server/pages";
  if (file.includes("/.next/server/")) return ".next/server (other)";
  if (file.includes("/.next/")) return ".next (other)";
  return `app: ${path.dirname(file).split("/").slice(-2).join("/")}`;
};

globalThis.__homarrProbe = {
  /** Per-space heap statistics: separates live JS objects from compiled code. */
  spaces: () => v8.getHeapSpaceStatistics(),
  heap: () => v8.getHeapStatistics(),
  usage: () => process.memoryUsage(),

  /**
   * Every file the CommonJS loader has resolved, with its on-disk byte size grouped by
   * owner. V8 keeps a script's source string alive for as long as any function in it
   * might still need lazy compilation, so these sizes are a direct lower bound on how
   * much heap the loaded code costs before a single object is allocated.
   */
  modules: () => {
    const cache = Module._cache ?? {};
    const groups = new Map();
    let total = 0;
    let count = 0;
    for (const file of Object.keys(cache)) {
      const size = fileSize(file);
      total += size;
      count++;
      const key = attributeFile(file);
      const acc = groups.get(key) ?? { bytes: 0, files: 0 };
      acc.bytes += size;
      acc.files++;
      groups.set(key, acc);
    }
    return {
      totalBytes: total,
      fileCount: count,
      groups: [...groups.entries()]
        .map(([name, acc]) => ({ name, ...acc }))
        .sort((a, b) => b.bytes - a.bytes),
    };
  },

  /** The individual loaded files, largest first — for naming specific offenders. */
  largestModules: (limit = 40) =>
    Object.keys(Module._cache ?? {})
      .map((file) => ({ file, bytes: fileSize(file) }))
      .sort((a, b) => b.bytes - a.bytes)
      .slice(0, limit),

  /** Raw list of resolved files, so two capture stages can be diffed. */
  moduleFiles: () => Object.keys(Module._cache ?? {}),

  /** Native bindings pulled in by the process, in load order. */
  moduleLoadList: () => process.moduleLoadList.slice(),
};

/**
 * Opt-in allocation sampling, and on-demand heap snapshots, over an in-process inspector
 * session.
 *
 * A heap snapshot cannot explain a peak. V8 runs a full GC before serialising one, so by
 * construction it shows what survived, never the transient churn that *is* the peak — the
 * 94 MiB of ArrayBuffers that drove this container to 500 MB were invisible in every
 * snapshot taken. The sampling heap profiler is the tool that sees them: it records an
 * allocation stack every N bytes allocated, whether or not the object survives.
 *
 * An in-process `inspector.Session` is used rather than SIGUSR1 plus a debug port, so this
 * needs no signal, opens no socket, and works while the process is under load.
 *
 * Enable with HOMARR_PROBE_SAMPLE_ALLOCATIONS=<sampling interval in bytes>.
 */
const inspector = require("inspector");
let session = null;
const ensureSession = () => {
  if (session) return session;
  session = new inspector.Session();
  session.connect();
  return session;
};
const postAsync = (method, params) =>
  new Promise((resolve, reject) =>
    ensureSession().post(method, params, (error, result) => (error ? reject(error) : resolve(result))),
  );

const samplingInterval = Number(process.env.HOMARR_PROBE_SAMPLE_ALLOCATIONS ?? 0);
if (samplingInterval > 0) {
  postAsync("HeapProfiler.enable")
    .then(() => postAsync("HeapProfiler.startSampling", { samplingInterval }))
    .then(() => console.log(`[memory-probe] allocation sampling every ${samplingInterval} bytes`))
    .catch((error) => console.error(`[memory-probe] could not start sampling: ${error.message}`));

  /**
   * Flattens the profile's call tree to self-bytes per allocation site.
   *
   * Sites are keyed by frame and aggregated over the *whole* tree, not just the largest
   * nodes, so the reported totals actually sum to the profile total — a truncated list
   * silently drops most of the bytes and makes every percentage wrong.
   *
   * `getSamplingProfile` reports cumulative bytes since sampling began, which mixes boot with
   * load. Capturing at two stages and subtracting is what isolates a single stage, so the key
   * is kept stable and machine-comparable for exactly that purpose.
   */
  globalThis.__homarrProbe.allocations = async () => {
    const { profile } = await postAsync("HeapProfiler.getSamplingProfile");
    const sites = new Map();
    let total = 0;
    const walk = (node, ancestry) => {
      const frame = node.callFrame;
      const key = `${frame.functionName || "(anonymous)"}|${frame.url}:${frame.lineNumber + 1}`;
      const stack = [key, ...ancestry];
      const selfSize = node.selfSize ?? 0;
      if (selfSize > 0) {
        total += selfSize;
        const acc = sites.get(key) ?? { bytes: 0, samples: 0, stack: stack.slice(0, 6) };
        acc.bytes += selfSize;
        acc.samples++;
        sites.set(key, acc);
      }
      for (const child of node.children ?? []) walk(child, stack);
    };
    walk(profile.head, []);
    return {
      total,
      sites: [...sites.entries()]
        .map(([key, acc]) => ({ key, ...acc }))
        .sort((a, b) => b.bytes - a.bytes)
        .slice(0, 400),
    };
  };
}

/**
 * Writes a heap snapshot to a path inside the container. Exposed so the harness can grab one
 * at a chosen moment — in particular while tabs are still open, which is a different heap
 * from the post-soak one every earlier snapshot captured.
 */
globalThis.__homarrProbe.snapshot = (destination) => {
  // v8.writeHeapSnapshot streams to the file from C++. The CDP route
  // (HeapProfiler.takeHeapSnapshot) delivers the snapshot as JS string chunks instead, and
  // for a ~135 MB snapshot that briefly added ~134 MiB of ArrayBuffers to the very process
  // being measured — an observer effect large enough to be mistaken for an application
  // problem, which is exactly what happened on the first attempt.
  const before = process.memoryUsage();
  v8.writeHeapSnapshot(destination);
  const after = process.memoryUsage();
  return {
    written: destination,
    bytes: fs.statSync(destination).size,
    // Reported so the cost of observing is visible rather than assumed to be zero.
    observerCostMiB: Math.round(((after.rss - before.rss) / 1048576) * 10) / 10,
  };
};

/** Address-space breakdown at the moment of capture; only /proc shows native vs JS vs code. */
globalThis.__homarrProbe.maps = () => {
  try {
    return { smaps: fs.readFileSync(`/proc/${process.pid}/smaps`, "utf8") };
  } catch (error) {
    return { error: String(error && error.message) };
  }
};

/**
 * Opt-in recording of the CommonJS require graph.
 *
 * Knowing that 24 MiB of source is resident at boot does not say *why* any of it is there.
 * The module cache is a flat list with no parentage, so the only way to answer "what pulled
 * this chunk in" is to record parent → child as the loader resolves it. The preload runs
 * before Next, so the graph is complete from the entry point down.
 *
 * Enable with HOMARR_PROBE_TRACK_REQUIRES=1.
 */
if (process.env.HOMARR_PROBE_TRACK_REQUIRES === "1") {
  const edges = new Set();
  const originalLoad = Module._load;
  Module._load = function (request, parent, isMain) {
    const result = originalLoad.call(this, request, parent, isMain);
    try {
      const resolved = Module._resolveFilename(request, parent, isMain);
      // Builtins resolve to their bare name and carry no file cost, so they only add noise.
      if (resolved.includes("/")) edges.add(`${parent ? parent.filename : "(entry)"}\t${resolved}`);
    } catch {
      // An unresolvable request is the caller's problem, not the probe's.
    }
    return result;
  };
  globalThis.__homarrProbe.requireGraph = () => [...edges].map((edge) => edge.split("\t"));
}

/**
 * Opt-in tracking of large Buffer allocations, by allocation site.
 *
 * `process.memoryUsage().arrayBuffers` can show ~100 MiB without any hint of who asked for
 * it, and a heap snapshot does not help: an ArrayBuffer's backing store lives outside the
 * JS heap, so the snapshot shows a small wrapper object and none of the bytes. Wrapping the
 * allocators and keeping a stack per call is the only way to get from "94 MiB of buffers"
 * to a file and line.
 *
 * Only allocations at or above the threshold are recorded, so the common small-buffer path
 * keeps its normal cost. Enable with HOMARR_PROBE_TRACK_BUFFERS=<bytes>.
 */
const trackThreshold = Number(process.env.HOMARR_PROBE_TRACK_BUFFERS ?? 0);
const bufferSites = new Map();
let trackedBytes = 0;
let trackedCount = 0;
if (trackThreshold > 0) {
  const record = (size) => {
    if (size < trackThreshold) return;
    trackedBytes += size;
    trackedCount++;
    // Frames 0-2 are Error/this wrapper, so skip them to land on the real caller.
    const stack = (new Error().stack ?? "").split("\n").slice(3, 9).join("\n");
    const acc = bufferSites.get(stack) ?? { bytes: 0, count: 0, max: 0 };
    acc.bytes += size;
    acc.count++;
    if (size > acc.max) acc.max = size;
    bufferSites.set(stack, acc);
  };

  for (const name of ["alloc", "allocUnsafe", "allocUnsafeSlow"]) {
    const original = Buffer[name];
    if (typeof original !== "function") continue;
    Buffer[name] = function (size, ...rest) {
      record(typeof size === "number" ? size : 0);
      return original.call(this, size, ...rest);
    };
  }
  /**
   * Buffers big enough to matter get their first bytes recorded. A stack says undici
   * buffered a body; only the bytes say whether that body was JSON, an image or a video
   * stream, and that is what decides whether the fix is a cap, a stream or a different URL.
   */
  const PREVIEW_THRESHOLD = 4 * 1048576;
  const previews = [];
  const originalConcat = Buffer.concat;
  Buffer.concat = function (list, totalLength) {
    const size = typeof totalLength === "number" ? totalLength : list.reduce((sum, item) => sum + item.length, 0);
    record(size);
    const result = originalConcat.call(this, list, totalLength);
    if (size >= PREVIEW_THRESHOLD && previews.length < 20) {
      previews.push({
        size,
        chunks: list.length,
        hex: result.subarray(0, 24).toString("hex"),
        text: result.subarray(0, 220).toString("latin1").replace(/[^\x20-\x7e]/g, "."),
        stack: (new Error().stack ?? "").split("\n").slice(2, 12).join("\n"),
      });
    }
    return result;
  };
  globalThis.__homarrProbe.previews = () => previews;

  // Default of 10 frames loses the caller behind undici's microtask hops.
  Error.stackTraceLimit = 30;

  globalThis.__homarrProbe.buffers = () => ({
    threshold: trackThreshold,
    previews: globalThis.__homarrProbe.previews ? globalThis.__homarrProbe.previews() : [],
    totalBytes: trackedBytes,
    count: trackedCount,
    sites: [...bufferSites.entries()]
      .map(([stack, acc]) => ({ stack, ...acc }))
      .sort((a, b) => b.bytes - a.bytes)
      .slice(0, 25),
  });
}

/**
 * Opt-in tracking of outbound fetch responses, by URL.
 *
 * Buffer-allocation stacks say *how* memory was allocated (undici buffering a whole body)
 * but not *which request* did it, because the URL is not on the stack. Wrapping fetch before
 * Next loads means every caller gets the instrumented version, so a single oversized
 * integration response can be named. Enable with HOMARR_PROBE_TRACK_FETCH=1.
 */
if (process.env.HOMARR_PROBE_TRACK_FETCH === "1" && typeof globalThis.fetch === "function") {
  const originalFetch = globalThis.fetch;
  const fetchStats = new Map();
  /** Group by origin+path with the query string dropped: tokens and ids would fragment the table. */
  const keyFor = (url) => {
    try {
      const parsed = new URL(url);
      return `${parsed.origin}${parsed.pathname}`;
    } catch {
      return String(url).slice(0, 120);
    }
  };

  globalThis.fetch = async function (resource, options) {
    const url = typeof resource === "string" ? resource : (resource?.url ?? String(resource));
    const response = await originalFetch.call(this, resource, options);
    const key = keyFor(url);
    const acc = fetchStats.get(key) ?? { calls: 0, declaredBytes: 0, readBytes: 0, maxRead: 0, unknownLength: 0 };
    acc.calls++;
    const declared = Number(response.headers.get("content-length") ?? 0);
    if (declared > 0) acc.declaredBytes += declared;
    else acc.unknownLength++;
    fetchStats.set(key, acc);

    // Body readers are wrapped rather than trusting content-length, because chunked and
    // gzipped responses declare nothing and those are exactly the large ones.
    for (const method of ["json", "text", "arrayBuffer"]) {
      const originalMethod = response[method];
      if (typeof originalMethod !== "function") continue;
      response[method] = async function (...rest) {
        const before = process.memoryUsage().arrayBuffers;
        const value = await originalMethod.apply(this, rest);
        const size =
          method === "text" && typeof value === "string"
            ? value.length
            : method === "arrayBuffer" && value?.byteLength
              ? value.byteLength
              : Math.max(0, process.memoryUsage().arrayBuffers - before);
        acc.readBytes += size;
        if (size > acc.maxRead) acc.maxRead = size;
        return value;
      };
    }
    return response;
  };

  globalThis.__homarrProbe.fetches = () => ({
    total: [...fetchStats.values()].reduce((sum, acc) => sum + acc.readBytes, 0),
    sites: [...fetchStats.entries()]
      .map(([url, acc]) => ({ url, ...acc }))
      .sort((a, b) => Math.max(b.readBytes, b.declaredBytes) - Math.max(a.readBytes, a.declaredBytes))
      .slice(0, 30),
  });
}

/**
 * A localhost-only JSON endpoint for the same data. Sampling over the inspector needs
 * SIGUSR1 and a CDP handshake per capture, which is too heavy to run at every stage of a
 * stress run; this is a plain fetch. Only started when HOMARR_PROBE_PORT is set, so the
 * preload does nothing observable unless the harness asks for it.
 */
const probePort = Number(process.env.HOMARR_PROBE_PORT ?? 0);
if (probePort > 0) {
  const probe = globalThis.__homarrProbe;
  require("http")
    .createServer((request, response) => {
      // Several endpoints are async (the inspector ones), so every route resolves through a
      // promise rather than some returning a value and some a pending Promise object.
      const [route, query] = (request.url ?? "/").split("?");
      const resolve = async () => {
        switch (route) {
          // The raw file list is served separately: it is long, and every stage of a stress
          // run fetches the summary, so keeping it out of the default payload keeps those
          // captures small while still allowing a boot-vs-loaded diff on demand.
          case "/files":
            return probe.moduleFiles();
          case "/buffers":
            return probe.buffers ? probe.buffers() : { disabled: true };
          case "/fetches":
            return probe.fetches ? probe.fetches() : { disabled: true };
          case "/requires":
            return probe.requireGraph ? probe.requireGraph() : { disabled: true };
          case "/allocations":
            return probe.allocations ? await probe.allocations() : { disabled: true };
          case "/maps":
            return probe.maps();
          case "/snapshot":
            return await probe.snapshot(new URLSearchParams(query).get("path") ?? "/tmp/peak.heapsnapshot");
          default:
            return {
              usage: probe.usage(),
              spaces: probe.spaces(),
              heap: probe.heap(),
              modules: probe.modules(),
              largestModules: probe.largestModules(25),
              uptime: process.uptime(),
              execArgv: process.execArgv,
            };
        }
      };
      resolve()
        .then((value) => {
          response.writeHead(200, { "content-type": "application/json" });
          response.end(JSON.stringify(value));
        })
        .catch((error) => {
          response.writeHead(500, { "content-type": "application/json" });
          response.end(JSON.stringify({ error: String(error && error.message) }));
        });
    })
    // NODE_OPTIONS applies to every node process in the container, including the DB
    // migration step and any `docker exec node` the harness runs, so all of them load this
    // preload and race for the port. Only the long-lived server needs to serve; the rest
    // must carry on silently rather than dying on EADDRINUSE.
    .on("error", (error) => {
      if (error.code !== "EADDRINUSE") throw error;
    })
    .listen(probePort, "127.0.0.1", () => console.log(`[memory-probe] listening on 127.0.0.1:${probePort}`));
}