diff --git a/scripts/benchmarks/stress-restore-lib.mts b/scripts/benchmarks/stress-restore-lib.mts index 75fa91441..1c01e9fa7 100644 --- a/scripts/benchmarks/stress-restore-lib.mts +++ b/scripts/benchmarks/stress-restore-lib.mts @@ -139,9 +139,15 @@ export const summarizeStress = (checkpoints: StressCheckpoint[], soakPrefix = "s const checkpoint = checkpoints.find((candidate) => candidate.name === name); return checkpoint ? toMiB(checkpoint.container.currentBytes) : null; }; + // Anonymous memory is the number to compare across runs: memory.current also + // counts reclaimable page cache, which swings tens of MiB with unrelated disk I/O. + const anonMiBByName = (name: string) => { + const checkpoint = checkpoints.find((candidate) => candidate.name === name); + return checkpoint ? toMiB(checkpoint.container.anonymousBytes) : null; + }; const soakSamples = checkpoints .filter((checkpoint) => checkpoint.name.startsWith(soakPrefix)) - .map((checkpoint) => ({ elapsedMs: checkpoint.elapsedMs, bytes: checkpoint.container.currentBytes })); + .map((checkpoint) => ({ elapsedMs: checkpoint.elapsedMs, bytes: checkpoint.container.anonymousBytes })); const peak = checkpoints.reduce( (max, checkpoint) => Math.max(max, checkpoint.container.peakBytes ?? checkpoint.container.currentBytes), @@ -156,12 +162,19 @@ export const summarizeStress = (checkpoints: StressCheckpoint[], soakPrefix = "s nodeRssMiB: Object.fromEntries( checkpoints.map((checkpoint) => [checkpoint.name, toMiB(sumNodeRssBytes(checkpoint))]), ), + anonMiB: Object.fromEntries( + checkpoints.map((checkpoint) => [checkpoint.name, toMiB(checkpoint.container.anonymousBytes)]), + ), + // Primary comparison metric: anonymous (non-reclaimable) memory. headline: { + bootIdleAnonMiB: anonMiBByName("01-boot-idle"), + boardLoadedAnonMiB: anonMiBByName("04-board-loaded"), + afterStressAnonMiB: anonMiBByName("05-after-stress"), bootIdleMiB: containerMiBByName("01-boot-idle"), boardLoadedMiB: containerMiBByName("04-board-loaded"), afterStressMiB: containerMiBByName("05-after-stress"), peakMiB: toMiB(peak), - soakGrowthMiBPerHour: (() => { + soakAnonGrowthMiBPerHour: (() => { const slope = getGrowthBytesPerHour(soakSamples); return slope === null ? null : toMiB(slope); })(), diff --git a/scripts/benchmarks/stress-restore-lib.spec.ts b/scripts/benchmarks/stress-restore-lib.spec.ts index 6eb07fc3c..d13083179 100644 --- a/scripts/benchmarks/stress-restore-lib.spec.ts +++ b/scripts/benchmarks/stress-restore-lib.spec.ts @@ -14,18 +14,20 @@ import { const snapshot = ({ current = 300 * 1024 * 1024, peak = 400 * 1024 * 1024, + anon = 250_000_000, processes = ["1|0|4096|nginx|2048", "20|1|204800|node|180000", "30|1|8192|redis-server|4096"], redis = true, }: { current?: number; peak?: number; + anon?: number; processes?: string[]; redis?: boolean; } = {}) => [ `memory_current=${current}`, `memory_peak=${peak}`, - "cgroup_anon=250000000", + `cgroup_anon=${anon}`, "cgroup_file=50000000", ...processes.map((process) => `process=${process}`), ...(redis ? ["redis_used_memory=1048576", "redis_used_memory_peak=2097152"] : []), @@ -136,8 +138,13 @@ describe("getGrowthBytesPerHour", () => { }); describe("summarizeStress", () => { + /** anon defaults to current so the fixture's two metrics move together. */ const build = (name: string, currentMiB: number, elapsedMs: number) => - parseStressSnapshot(name, elapsedMs, snapshot({ current: currentMiB * 1024 * 1024, peak: 900 * 1024 * 1024 })); + parseStressSnapshot( + name, + elapsedMs, + snapshot({ current: currentMiB * 1024 * 1024, anon: currentMiB * 1024 * 1024, peak: 900 * 1024 * 1024 }), + ); it("surfaces the headline lifecycle numbers", () => { const summary = summarizeStress([ @@ -146,6 +153,8 @@ describe("summarizeStress", () => { build("05-after-stress", 600, 120_000), ]); + expect(summary.headline.bootIdleAnonMiB).toBe(200); + expect(summary.headline.boardLoadedAnonMiB).toBe(450); expect(summary.headline.bootIdleMiB).toBe(200); expect(summary.headline.boardLoadedMiB).toBe(450); expect(summary.headline.afterStressMiB).toBe(600); @@ -153,6 +162,15 @@ describe("summarizeStress", () => { expect(summary.checkpointCount).toBe(3); }); + it("compares on anon, not memory.current, so page cache cannot skew a run", () => { + const summary = summarizeStress([ + parseStressSnapshot("01-boot-idle", 0, snapshot({ current: 500 * 1024 * 1024, anon: 100 * 1024 * 1024 })), + ]); + + expect(summary.headline.bootIdleAnonMiB).toBe(100); + expect(summary.headline.bootIdleMiB).toBe(500); + }); + it("derives soak growth only from soak checkpoints", () => { const summary = summarizeStress([ build("01-boot-idle", 1_000, 0), @@ -160,7 +178,7 @@ describe("summarizeStress", () => { build("soak-02", 200, 3_600_000), ]); - expect(summary.headline.soakGrowthMiBPerHour).toBe(100); + expect(summary.headline.soakAnonGrowthMiBPerHour).toBe(100); }); it("leaves headline fields null when their checkpoints were not captured", () => { @@ -168,7 +186,7 @@ describe("summarizeStress", () => { expect(summary.headline.bootIdleMiB).toBeNull(); expect(summary.headline.boardLoadedMiB).toBeNull(); - expect(summary.headline.soakGrowthMiBPerHour).toBeNull(); + expect(summary.headline.soakAnonGrowthMiBPerHour).toBeNull(); }); it("reports the peak redis footprint across checkpoints", () => { diff --git a/scripts/benchmarks/stress-restore.mts b/scripts/benchmarks/stress-restore.mts index e25e34b91..1c43440b4 100644 --- a/scripts/benchmarks/stress-restore.mts +++ b/scripts/benchmarks/stress-restore.mts @@ -146,18 +146,36 @@ const restoreBackupAsync = async (page: Page, baseUrl: string) => { }; const loginAsync = async (page: Page, baseUrl: string) => { - await page.goto(`${baseUrl}/auth/login`, { waitUntil: "domcontentloaded" }); - // Selectors mirror e2e/lazy-widgets.spec.ts: getByLabel(/password/i) is ambiguous - // because Mantine's PasswordInput also renders a visibility-toggle button. - await page.getByLabel("Username").fill(username); - await page.locator("#password").fill(password); - await page.locator("css=button[type='submit']").click(); - // Poll location instead of page.waitForURL: sign-in redirects client-side, so - // waitForURL's implicit "load" state may never be reached for that navigation. - await page.waitForFunction(() => !window.location.pathname.includes("/auth/login"), undefined, { - timeout: 60_000, - }); - log(`signed in as ${username}`); + // networkidle: submitting before the form hydrates is a no-op click, which then + // looks exactly like an authentication failure. + await page.goto(`${baseUrl}/auth/login`, { waitUntil: "networkidle" }); + + for (let attempt = 1; attempt <= 3; attempt++) { + // Selectors mirror e2e/lazy-widgets.spec.ts: getByLabel(/password/i) is ambiguous + // because Mantine's PasswordInput also renders a visibility-toggle button. + await page.getByLabel("Username").fill(username); + await page.locator("#password").fill(password); + await page.locator("css=button[type='submit']").first().click(); + try { + // Poll location rather than page.waitForURL: sign-in redirects client-side, so + // waitForURL's implicit "load" state is never reached for that navigation. + await page.waitForFunction(() => !window.location.pathname.includes("/auth/login"), undefined, { + timeout: 20_000, + }); + log(`signed in as ${username}`); + return; + } catch { + if (attempt === 3) { + const error = await page + .locator('[role="alert"], .mantine-Alert-message') + .first() + .innerText() + .catch(() => "(no alert on page)"); + throw new Error(`Login did not navigate away from /auth/login after 3 attempts. Page alert: ${error}`); + } + log(`login attempt ${attempt} did not navigate (likely a pre-hydration click), retrying`); + } + } }; const waitForBoardAsync = async (page: Page) => {