diff --git a/.github/scripts/backend-memory-report.mts b/.github/scripts/backend-memory-report.mts index a758beb979..77ea7883e6 100644 --- a/.github/scripts/backend-memory-report.mts +++ b/.github/scripts/backend-memory-report.mts @@ -144,7 +144,7 @@ function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) { ]; for (const graph of [ - heapSnapshotUtil.renderHeapSnapshotSankey(baseHeapSnapshotReport, 'Base'), + //heapSnapshotUtil.renderHeapSnapshotSankey(baseHeapSnapshotReport, 'Base'), heapSnapshotUtil.renderHeapSnapshotSankey(headHeapSnapshotReport, 'Head'), ]) { if (graph == null) continue; @@ -374,6 +374,10 @@ if (heapSnapshotSection != null) { lines.push(''); } +const artifactUrl = process.env.MK_MEMORY_HEAP_SNAPSHOT_ARTIFACT_URL_HEAD!.trim(); +lines.push(`[Download representative V8 heap snapshot (head)](${artifactUrl})`); +lines.push(''); + const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint); if (jsFootprintSection != null) { lines.push(jsFootprintSection); @@ -420,6 +424,6 @@ if (warningMetric != null && warningDiffPercent != null && warningDiffPercent > lines.push(''); } -lines.push(`[See workflow logs for details](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`); +//lines.push(`[See workflow logs for details](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`); await writeFile(outputFile, `${lines.join('\n')}\n`); diff --git a/.github/scripts/measure-backend-memory-comparison.mts b/.github/scripts/measure-backend-memory-comparison.mts index 99ba4f64ae..3dce74dc0a 100644 --- a/.github/scripts/measure-backend-memory-comparison.mts +++ b/.github/scripts/measure-backend-memory-comparison.mts @@ -4,7 +4,7 @@ */ import { createRequire } from 'node:module'; -import { writeFile } from 'node:fs/promises'; +import { copyFile, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import * as util from './utility.mts'; import * as heapSnapshotUtil from './heap-snapshot-util.mts'; @@ -39,6 +39,8 @@ export type MemoryReport = { const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2); const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = util.readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1); +const HEAD_HEAP_SNAPSHOT_WORK_DIR = resolve('head-heap-snapshots'); +const HEAD_HEAP_SNAPSHOT_OUTPUT_PATH = resolve('head-heap-snapshot.heapsnapshot'); async function resetState(repoDir: string) { const require = createRequire(join(repoDir, 'packages/backend/package.json')); @@ -165,7 +167,7 @@ function summarizeSamples(samples: MemoryReport['samples']) { return summary; } -async function measureRepo(label: string, repoDir: string, round: number) { +async function measureRepo(label: string, repoDir: string, round: number, options: { heapSnapshotSavePath?: string } = {}) { process.stderr.write(`[${label}] Resetting database and Redis\n`); await resetState(repoDir); @@ -182,6 +184,7 @@ async function measureRepo(label: string, repoDir: string, round: number) { MK_MEMORY_SAMPLE_COUNT: '1', } as NodeJS.ProcessEnv; if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0'; + if (options.heapSnapshotSavePath != null) measureEnv.MK_MEMORY_HEAP_SNAPSHOT_SAVE_PATH = options.heapSnapshotSavePath; const stdout = await util.run('node', ['packages/backend/scripts/measure-memory.mts'], { cwd: repoDir, @@ -194,6 +197,40 @@ async function measureRepo(label: string, repoDir: string, round: number) { return sample; } +function headHeapSnapshotPath(round: number) { + return join(HEAD_HEAP_SNAPSHOT_WORK_DIR, `round-${round}.heapsnapshot`); +} + +function selectRepresentativeHeadHeapSnapshotRound(samples: MemoryReport['samples'], summary: MemoryReport['summary']) { + const medianTotal = summary.afterGc.heapSnapshot?.categories?.total; + if (medianTotal == null || !Number.isFinite(medianTotal)) return null; + + let selected: { round: number; distance: number } | null = null; + for (const sample of samples) { + const total = sample.phases.afterGc.heapSnapshot?.categories?.total; + if (total == null || !Number.isFinite(total)) continue; + + const distance = Math.abs(total - medianTotal); + if (selected == null || distance < selected.distance || (distance === selected.distance && sample.round < selected.round)) { + selected = { + round: sample.round, + distance, + }; + } + } + + return selected?.round ?? null; +} + +async function saveRepresentativeHeadHeapSnapshot(samples: MemoryReport['samples'], summary: MemoryReport['summary']) { + const round = selectRepresentativeHeadHeapSnapshotRound(samples, summary); + if (round == null) return; + + await copyFile(headHeapSnapshotPath(round), HEAD_HEAP_SNAPSHOT_OUTPUT_PATH); + process.stderr.write(`Selected head heap snapshot round ${round} for artifact\n`); + await rm(HEAD_HEAP_SNAPSHOT_WORK_DIR, { recursive: true, force: true }); +} + async function main() { const baseDir = resolve(baseDirArg); const headDir = resolve(headDirArg); @@ -203,6 +240,9 @@ async function main() { const warmupRounds = util.readIntegerEnv('MK_MEMORY_COMPARE_WARMUP_ROUNDS', 1, 0); const startedAt = new Date().toISOString(); + await rm(HEAD_HEAP_SNAPSHOT_WORK_DIR, { recursive: true, force: true }); + await rm(HEAD_HEAP_SNAPSHOT_OUTPUT_PATH, { force: true }); + const reports = { base: { dir: baseDir, @@ -225,8 +265,12 @@ async function main() { const order = round % 2 === 1 ? ['base', 'head'] as const : ['head', 'base'] as const; process.stderr.write(`Starting measurement round ${round}/${rounds}: ${order.join(' -> ')}\n`); - for (const [orderIndex, label] of order.entries()) { - const sample = await measureRepo(label, reports[label].dir, round); + for (const label of order) { + const shouldSaveHeadHeapSnapshot = label === 'head'; + const options = shouldSaveHeadHeapSnapshot + ? { heapSnapshotSavePath: headHeapSnapshotPath(round) } + : {}; + const sample = await measureRepo(label, reports[label].dir, round, options); reports[label].samples.push({ ...sample, round, @@ -234,6 +278,12 @@ async function main() { } } + const summaries = { + base: summarizeSamples(reports.base.samples), + head: summarizeSamples(reports.head.samples), + }; + await saveRepresentativeHeadHeapSnapshot(reports.head.samples, summaries.head); + for (const label of ['base', 'head'] as const) { const report = { timestamp: new Date().toISOString(), @@ -245,7 +295,7 @@ async function main() { warmupRounds, startedAt, }, - summary: summarizeSamples(reports[label].samples), + summary: summaries[label], samples: reports[label].samples, }; diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index d25e3db1d9..c845cc81fd 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -96,6 +96,13 @@ jobs: MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1 MK_MEMORY_HEAP_SNAPSHOT: 1 run: node head/.github/scripts/measure-backend-memory-comparison.mts base head memory-base.json memory-head.json + - name: Upload head heap snapshot + uses: actions/upload-artifact@v7 + with: + name: backend-memory-head-heap-snapshot + path: head-heap-snapshot.heapsnapshot + if-no-files-found: error + retention-days: 7 - name: Measure backend loaded JS footprint run: | node head/.github/scripts/backend-js-footprint.mjs base js-footprint-base.json diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index 147e20d4d6..b3a98e046c 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -33,9 +33,26 @@ jobs: - name: Load PR Number id: load-pr-num run: echo "pr-number=$(cat artifacts/pr_number)" >> "$GITHUB_OUTPUT" + - name: Find head heap snapshot artifact + id: find-heap-snapshot-artifact + uses: actions/github-script@v8 + with: + script: | + const { owner, repo } = context.repo; + const run_id = context.payload.workflow_run.id; + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner, + repo, + run_id, + }); + const artifact = artifacts.find(artifact => artifact.name === 'backend-memory-head-heap-snapshot'); + if (artifact == null) return; + core.setOutput('url', `https://github.com/${owner}/${repo}/actions/runs/${run_id}/artifacts/${artifact.id}`); - id: build-comment name: Build memory comment + env: + MK_MEMORY_HEAP_SNAPSHOT_ARTIFACT_URL_HEAD: ${{ steps.find-heap-snapshot-artifact.outputs.url }} run: node .github/scripts/backend-memory-report.mts ./artifacts/memory-base.json ./artifacts/memory-head.json ./output.md ./artifacts/js-footprint-base.json ./artifacts/js-footprint-head.json - uses: thollander/actions-comment-pull-request@v3 with: diff --git a/packages/backend/scripts/measure-memory.mts b/packages/backend/scripts/measure-memory.mts index c14f4854b2..3032f1ae15 100644 --- a/packages/backend/scripts/measure-memory.mts +++ b/packages/backend/scripts/measure-memory.mts @@ -41,6 +41,7 @@ const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0); const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false); const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1); const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1); +const HEAP_SNAPSHOT_SAVE_PATH = process.env.MK_MEMORY_HEAP_SNAPSHOT_SAVE_PATH; const procStatusKeys = ['VmPeak', 'VmSize', 'VmHWM', 'VmRSS', 'VmData', 'VmStk', 'VmExe', 'VmLib', 'VmPTE', 'VmSwap'] as const; const smapsRollupKeys = ['Pss', 'Shared_Clean', 'Shared_Dirty', 'Private_Clean', 'Private_Dirty', 'Swap', 'SwapPss'] as const; @@ -320,6 +321,11 @@ async function getHeapSnapshotStatistics(serverProcess: ChildProcess): Promise