homarr/scripts/benchmarks/stress-restore-lib.mts

/**
 * Pure helpers for the dashboard stress benchmark (`stress-restore.mts`).
 *
 * Kept free of Docker/Playwright so the measurement maths and the container
 * snapshot script stay unit-testable.
 */

export type ProcessSample = {
  command: string;
  parentPid: number;
  pid: number;
  pssBytes: number;
  rssBytes: number;
};

export type StressCheckpoint = {
  name: string;
  capturedAt: string;
  elapsedMs: number;
  container: {
    currentBytes: number;
    peakBytes: number | null;
    anonymousBytes: number;
    fileBytes: number;
  };
  /**
   * Cumulative cgroup CPU time. Needed because most ways of lowering memory
   * (smaller GC spaces, tighter heap caps) buy it with CPU, and a win that just
   * moves the cost somewhere unmeasured is not a win.
   */
  cpu: { usageUsec: number; userUsec: number; systemUsec: number } | null;
  processes: ProcessSample[];
  redis: { usedMemoryBytes: number; peakMemoryBytes: number } | null;
};

/**
 * Emitted inside the container and parsed by `parseStressSnapshot`.
 *
 * cgroup v2 `memory.current` is the number that matters for a container memory
 * limit (it is what the OOM killer compares against), so it is the headline
 * metric rather than the sum of process RSS, which double-counts shared pages.
 */
export const stressMemoryScript = String.raw`
echo "memory_current=$(cat /sys/fs/cgroup/memory.current 2>/dev/null || echo 0)"
memory_peak="$(cat /sys/fs/cgroup/memory.peak 2>/dev/null || true)"
case "$memory_peak" in ''|'max') ;; *) echo "memory_peak=$memory_peak" ;; esac
awk '/^anon / { print "cgroup_anon=" $2 } /^file / { print "cgroup_file=" $2 }' /sys/fs/cgroup/memory.stat 2>/dev/null
awk '/^usage_usec / { print "cpu_usage_usec=" $2 } /^user_usec / { print "cpu_user_usec=" $2 } /^system_usec / { print "cpu_system_usec=" $2 }' /sys/fs/cgroup/cpu.stat 2>/dev/null
for status_file in /proc/[0-9]*/status; do
  [ -r "$status_file" ] || continue
  process_dir="$(dirname "$status_file")"
  process_values="$(awk '/^Name:/ { name=$2 } /^Pid:/ { pid=$2 } /^PPid:/ { ppid=$2 } /^VmRSS:/ { rss=$2 } END { if (rss == "") rss=0; print pid "|" ppid "|" rss "|" name }' "$status_file" 2>/dev/null)"
  process_pss="$(awk '/^Pss:/ { print $2 }' "$process_dir/smaps_rollup" 2>/dev/null || echo 0)"
  [ -n "$process_pss" ] || process_pss=0
  [ -n "$process_values" ] && echo "process=$process_values|$process_pss"
done
redis_info="$(redis-cli INFO memory 2>/dev/null | tr -d '\r' || true)"
redis_used="$(printf '%s\n' "$redis_info" | awk -F: '/^used_memory:/ { print $2 }')"
redis_peak="$(printf '%s\n' "$redis_info" | awk -F: '/^used_memory_peak:/ { print $2 }')"
if [ -n "$redis_used" ] && [ -n "$redis_peak" ]; then
  echo "redis_used_memory=$redis_used"
  echo "redis_used_memory_peak=$redis_peak"
fi
`;

const readNumericLine = (input: string, key: string) => {
  const match = new RegExp(`^${key}[=:]\\s*(\\d+)`, "m").exec(input);
  return match?.[1] === undefined ? null : Number(match[1]);
};

export const parseStressSnapshot = (name: string, elapsedMs: number, output: string): StressCheckpoint => {
  const currentBytes = readNumericLine(output, "memory_current");
  if (currentBytes === null) throw new Error(`Snapshot "${name}" is missing memory_current`);

  const processes = output
    .split("\n")
    .filter((line) => line.startsWith("process="))
    .flatMap<ProcessSample>((line) => {
      const [pid, parentPid, rssKiB, command, pssKiB] = line.slice("process=".length).split("|");
      if (!pid || !parentPid || rssKiB === undefined || !command) return [];
      return [
        {
          pid: Number(pid),
          parentPid: Number(parentPid),
          rssBytes: Number(rssKiB) * 1024,
          pssBytes: Number(pssKiB ?? 0) * 1024,
          command,
        },
      ];
    });

  const redisUsed = readNumericLine(output, "redis_used_memory");
  const redisPeak = readNumericLine(output, "redis_used_memory_peak");
  const cpuUsage = readNumericLine(output, "cpu_usage_usec");

  return {
    name,
    capturedAt: new Date().toISOString(),
    elapsedMs,
    container: {
      currentBytes,
      peakBytes: readNumericLine(output, "memory_peak"),
      anonymousBytes: readNumericLine(output, "cgroup_anon") ?? 0,
      fileBytes: readNumericLine(output, "cgroup_file") ?? 0,
    },
    cpu:
      cpuUsage === null
        ? null
        : {
            usageUsec: cpuUsage,
            userUsec: readNumericLine(output, "cpu_user_usec") ?? 0,
            systemUsec: readNumericLine(output, "cpu_system_usec") ?? 0,
          },
    processes,
    redis: redisUsed === null || redisPeak === null ? null : { usedMemoryBytes: redisUsed, peakMemoryBytes: redisPeak },
  };
};

/** Node processes only — `next-server` plus any child it forks. */
export const getNodeProcesses = (checkpoint: StressCheckpoint) =>
  checkpoint.processes.filter((process) => /node|next/i.test(process.command));

export const sumNodeRssBytes = (checkpoint: StressCheckpoint) =>
  getNodeProcesses(checkpoint).reduce((total, process) => total + process.rssBytes, 0);

/** Largest single node process — the Next.js server in a normal container. */
export const getPrimaryNodeRssBytes = (checkpoint: StressCheckpoint) =>
  getNodeProcesses(checkpoint).reduce((max, process) => Math.max(max, process.rssBytes), 0);

export const toMiB = (bytes: number) => Math.round((bytes / 1024 / 1024) * 10) / 10;

/**
 * Least-squares slope over (elapsedMs, bytes), reported per hour. A large
 * positive slope across the idle soak is the signature of a retention leak
 * rather than of ordinary warm-up.
 */
export const getGrowthBytesPerHour = (samples: { elapsedMs: number; bytes: number }[]) => {
  if (samples.length < 2) return null;
  const n = samples.length;
  const meanX = samples.reduce((sum, s) => sum + s.elapsedMs, 0) / n;
  const meanY = samples.reduce((sum, s) => sum + s.bytes, 0) / n;
  let numerator = 0;
  let denominator = 0;
  for (const sample of samples) {
    const dx = sample.elapsedMs - meanX;
    numerator += dx * (sample.bytes - meanY);
    denominator += dx * dx;
  }
  if (denominator === 0) return null;
  return Math.round((numerator / denominator) * 3_600_000);
};

export const summarizeStress = (checkpoints: StressCheckpoint[], soakPrefix = "soak") => {
  const containerMiBByName = (name: string) => {
    const checkpoint = checkpoints.find((candidate) => candidate.name === name);
    return checkpoint ? toMiB(checkpoint.container.currentBytes) : null;
  };
  // Anonymous memory is the number to compare across runs: memory.current also
  // counts reclaimable page cache, which swings tens of MiB with unrelated disk I/O.
  const anonMiBByName = (name: string) => {
    const checkpoint = checkpoints.find((candidate) => candidate.name === name);
    return checkpoint ? toMiB(checkpoint.container.anonymousBytes) : null;
  };
  const soakSamples = checkpoints
    .filter((checkpoint) => checkpoint.name.startsWith(soakPrefix))
    .map((checkpoint) => ({ elapsedMs: checkpoint.elapsedMs, bytes: checkpoint.container.anonymousBytes }));

  const peak = checkpoints.reduce(
    (max, checkpoint) => Math.max(max, checkpoint.container.peakBytes ?? checkpoint.container.currentBytes),
    0,
  );

  return {
    checkpointCount: checkpoints.length,
    containerMiB: Object.fromEntries(
      checkpoints.map((checkpoint) => [checkpoint.name, toMiB(checkpoint.container.currentBytes)]),
    ),
    nodeRssMiB: Object.fromEntries(
      checkpoints.map((checkpoint) => [checkpoint.name, toMiB(sumNodeRssBytes(checkpoint))]),
    ),
    anonMiB: Object.fromEntries(
      checkpoints.map((checkpoint) => [checkpoint.name, toMiB(checkpoint.container.anonymousBytes)]),
    ),
    // Primary comparison metric: anonymous (non-reclaimable) memory.
    headline: {
      bootIdleAnonMiB: anonMiBByName("01-boot-idle"),
      boardLoadedAnonMiB: anonMiBByName("04-board-loaded"),
      afterStressAnonMiB: anonMiBByName("05-after-stress"),
      bootIdleMiB: containerMiBByName("01-boot-idle"),
      boardLoadedMiB: containerMiBByName("04-board-loaded"),
      afterStressMiB: containerMiBByName("05-after-stress"),
      peakMiB: toMiB(peak),
      soakAnonGrowthMiBPerHour: (() => {
        const slope = getGrowthBytesPerHour(soakSamples);
        return slope === null ? null : toMiB(slope);
      })(),
    },
    redisPeakMiB: (() => {
      const peaks = checkpoints.flatMap((c) => (c.redis ? [c.redis.peakMemoryBytes] : []));
      return peaks.length === 0 ? null : toMiB(Math.max(...peaks));
    })(),
    // Total CPU the container burned over the run, so a memory reduction bought
    // with extra GC work shows up instead of hiding.
    cpuSeconds: (() => {
      const samples = checkpoints.flatMap((c) => (c.cpu ? [c.cpu.usageUsec] : []));
      if (samples.length < 2) return null;
      return Math.round(((Math.max(...samples) - Math.min(...samples)) / 1_000_000) * 10) / 10;
    })(),
  };
};