diff --git a/.github/scripts/backend-memory-report.mjs b/.github/scripts/backend-memory-report.mjs index a04ee187a5..2aeaf24733 100644 --- a/.github/scripts/backend-memory-report.mjs +++ b/.github/scripts/backend-memory-report.mjs @@ -8,7 +8,7 @@ if (baseFile == null || headFile == null || outputFile == null) { } const numberFormatter = new Intl.NumberFormat('en-US', { - maximumFractionDigits: 2, + maximumFractionDigits: 1, }); const phases = [ @@ -26,6 +26,39 @@ const metrics = [ 'External', ]; +const heapSnapshotCategories = [ + 'Total', + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', +]; + +const heapSnapshotCategoriesColors = { + 'Total': 'gray', + 'Code': 'orange', + 'Strings': 'red', + 'JS arrays': 'cyan', + 'Typed arrays': 'green', + 'System objects': 'yellow', + 'Other JS objects': 'violet', + 'Other non-JS objects': 'pink', +}; + +const heapSnapshotCategoriesColorsHex = { + 'Total': '#888888', + 'Code': '#f28e2c', + 'Strings': '#e15759', + 'JS arrays': '#76b7b2', + 'Typed arrays': '#59a14f', + 'System objects': '#edc949', + 'Other JS objects': '#af7aa1', + 'Other non-JS objects': '#ff9da7', +}; + function formatNumber(value) { return numberFormatter.format(value); } @@ -45,6 +78,14 @@ function formatPercent(value) { return `${formatNumber(value)}%`; } +function formatDeltaPercent(diff, baseValue) { + if (diff === 0) return '0%'; + if (baseValue <= 0) return '-'; + + const sign = diff > 0 ? '+' : '-'; + return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`, diff); +} + function formatMathText(text) { return text .replaceAll('\\', '\\\\') @@ -58,23 +99,6 @@ function formatColoredDiff(text, diff) { return `$\\color{${color}}{\\text{${formatMathText(text).replaceAll('\\%', '\\\\%')}}}$`; } -function formatDiff(baseKiB, headKiB) { - const diff = headKiB - baseKiB; - if (diff === 0) return formatMemory(0); - - const sign = diff > 0 ? '+' : '-'; - return formatColoredDiff(`${sign}${formatMemory(Math.abs(diff))}`, diff); -} - -function formatDiffPercent(baseKiB, headKiB) { - const diff = headKiB - baseKiB; - if (diff === 0) return '0%'; - if (baseKiB <= 0) return '-'; - - const sign = diff > 0 ? '+' : '-'; - return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseKiB))}`, diff); -} - function getMemoryValue(report, phase, metric) { const value = report?.[phase]?.[metric]; return Number.isFinite(value) ? value : null; @@ -163,8 +187,8 @@ function pairedDeltaSummary(base, head, phase, metric) { function renderTable(base, head, phase) { const lines = [ - '| Metric | Base | Head | Δ | Δ (%) |', - '| --- | ---: | ---: | ---: | ---: |', + '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', ]; for (const metric of metrics) { @@ -174,27 +198,12 @@ function renderTable(base, head, phase) { const baseSpread = getSampleSpread(base, phase, metric); const headSpread = getSampleSpread(head, phase, metric); - - lines.push(`| ${metric} | ${formatMemory(baseValue)}
± ${formatMemory(baseSpread)} | ${formatMemory(headValue)}
± ${formatMemory(headSpread)} | ${formatDiff(baseValue, headValue)} | ${formatDiffPercent(baseValue, headValue)} |`); - } - - return lines.join('\n'); -} - -function renderPairedDeltaTable(base, head, phase) { - const lines = [ - '| Metric | Δ median | Δ MAD | Δ min | Δ max |', - '| --- | ---: | ---: | ---: | ---: |', - ]; - - for (const metric of metrics) { const summary = pairedDeltaSummary(base, head, phase, metric); - if (summary == null) continue; + const deltaMedian = summary == null ? '-' : `${formatDeltaMemory(summary.median)}
${formatDeltaPercent(summary.median, baseValue)}`; - lines.push(`| ${metric} | ${formatDeltaMemory(summary.median)} | ${summary.mad == null ? '-' : formatMemory(summary.mad)} | ${formatDeltaMemory(summary.min)} | ${formatDeltaMemory(summary.max)} |`); + lines.push(`| **${metric}** | ${formatMemory(baseValue)}
± ${formatMemory(baseSpread)} | ${formatMemory(headValue)}
± ${formatMemory(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : formatMemory(summary.mad)} | ${summary == null ? '-' : formatDeltaMemory(summary.min)} | ${summary == null ? '-' : formatDeltaMemory(summary.max)} |`); } - if (lines.length === 2) return null; return lines.join('\n'); } @@ -273,6 +282,241 @@ function formatPlainDiffPercent(baseValue, headValue) { return `${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`; } +function getHeapSnapshotCategoryValue(report, phase, category) { + const value = report?.[phase]?.heapSnapshot?.categories?.[category]; + return Number.isFinite(value) ? value : null; +} + +function getHeapSnapshotBreakdownEntries(report, phase, category) { + const breakdown = report?.[phase]?.heapSnapshot?.breakdowns?.[category]; + if (breakdown == null || typeof breakdown !== 'object') return []; + + return Object.entries(breakdown) + .filter(([, value]) => Number.isFinite(value) && value > 0) + .toSorted((a, b) => b[1] - a[1]); +} + +const heapSnapshotSankeyChildMinRatio = 0.3; +const heapSnapshotSankeyParentMinPercent = 10; + +function escapeCsvValue(value) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +function formatSankeyPercentValue(value) { + const rounded = Math.round(value * 100) / 100; + if (rounded === 0 && value > 0) return '0.01'; + if (Number.isInteger(rounded)) return String(rounded); + return rounded.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); +} + +function formatHeapSnapshotSankeyChildLabel(label) { + return String(label).replace(/^[^:]+:\s*/, ''); +} + +function renderHeapSnapshotSankey(report, phase, title) { + const total = getHeapSnapshotCategoryValue(report, phase, 'Total'); + if (total == null || total <= 0) return null; + + const categories = heapSnapshotCategories + .filter(category => category !== 'Total') + .map(category => { + const value = getHeapSnapshotCategoryValue(report, phase, category); + if (value == null || value <= 0) return null; + const breakdownEntries = getHeapSnapshotBreakdownEntries(report, phase, category); + const breakdownTotal = breakdownEntries.reduce((sum, [, childValue]) => sum + childValue, 0); + const percent = (value * 100) / total; + const childEntries = []; + let otherPercent = 0; + + if (breakdownTotal > 0 && percent > heapSnapshotSankeyParentMinPercent) { + for (const [childName, childValue] of breakdownEntries) { + const childRatio = childValue / breakdownTotal; + const childPercent = percent * childRatio; + if (childRatio >= heapSnapshotSankeyChildMinRatio) { + childEntries.push([formatHeapSnapshotSankeyChildLabel(childName), childPercent]); + } else { + otherPercent += childPercent; + } + } + + if (childEntries.length > 0 && otherPercent > 0) { + childEntries.push(['Other', otherPercent]); + } + } + + return { + category, + percent, + childEntries, + }; + }) + .filter(value => value != null); + + if (categories.length === 0) return null; + + const nodeColors = { + [title]: heapSnapshotCategoriesColorsHex.Total, + }; + for (const { category, childEntries } of categories) { + const categoryColor = heapSnapshotCategoriesColorsHex[category] ?? heapSnapshotCategoriesColorsHex.Total; + nodeColors[category] = categoryColor; + + for (const [childName] of childEntries) { + nodeColors[childName] = categoryColor; + } + } + + const lines = [ + `
${title} heap snapshot composition`, + '', + '```mermaid', + `%%{init: ${JSON.stringify({ + sankey: { + showValues: false, + linkColor: 'target', + labelStyle: 'outlined', + nodeAlignment: 'center', + nodePadding: 10, + nodeColors: { + ...nodeColors, + 'Other': '#888888', + }, + }, + })}}%%`, + 'sankey-beta', + ]; + + for (const { category, percent, childEntries } of categories) { + lines.push(`${escapeCsvValue(title)},${escapeCsvValue(category)},${formatSankeyPercentValue(percent)}`); + + for (const [childName, childPercent] of childEntries) { + lines.push(`${escapeCsvValue(category)},${escapeCsvValue(childName)},${formatSankeyPercentValue(childPercent)}`); + } + } + + lines.push('```'); + lines.push(''); + lines.push('
'); + + return lines.join('\n'); +} + +function formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal) { + if (category === 'Total' || baseTotal == null || headTotal == null || baseTotal <= 0 || headTotal <= 0) return `**${category}**`; + + const basePercent = formatPercent((baseValue * 100) / baseTotal); + const headPercent = formatPercent((headValue * 100) / headTotal); + return `**${category}**
${basePercent} → ${headPercent}`; +} + +function getHeapSnapshotSampleValues(report, phase, category) { + if (!Array.isArray(report?.samples)) return []; + + return report.samples + .map(sample => getHeapSnapshotCategoryValue(sample, phase, category)) + .filter(value => Number.isFinite(value)); +} + +function getHeapSnapshotSampleSpread(report, phase, category) { + const values = getHeapSnapshotSampleValues(report, phase, category); + if (values.length < 2) return null; + + const center = median(values); + return median(values.map(value => Math.abs(value - center))); +} + +function getPairedHeapSnapshotDeltaValues(base, head, phase, category) { + const baseSamplesByRound = getSamplesByRound(base); + const headSamplesByRound = getSamplesByRound(head); + const values = []; + + for (const [round, baseSample] of baseSamplesByRound) { + const headSample = headSamplesByRound.get(round); + if (headSample == null) continue; + + const baseValue = getHeapSnapshotCategoryValue(baseSample, phase, category); + const headValue = getHeapSnapshotCategoryValue(headSample, phase, category); + if (baseValue == null || headValue == null) continue; + + values.push(headValue - baseValue); + } + + return values; +} + +function formatDeltaBytes(diffBytes) { + if (diffBytes === 0) return formatBytes(0); + + const sign = diffBytes > 0 ? '+' : '-'; + return formatColoredDiff(`${sign}${formatBytes(Math.abs(diffBytes))}`, diffBytes); +} + +function pairedHeapSnapshotDeltaSummary(base, head, phase, category) { + const values = getPairedHeapSnapshotDeltaValues(base, head, phase, category); + if (values.length === 0) return null; + + return { + median: median(values), + mad: mad(values), + min: Math.min(...values), + max: Math.max(...values), + samples: values.length, + }; +} + +function renderHeapSnapshotTable(base, head, phase) { + const lines = [ + '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + const baseTotal = getHeapSnapshotCategoryValue(base, phase, 'Total'); + const headTotal = getHeapSnapshotCategoryValue(head, phase, 'Total'); + + for (const category of heapSnapshotCategories) { + const baseValue = getHeapSnapshotCategoryValue(base, phase, category); + const headValue = getHeapSnapshotCategoryValue(head, phase, category); + if (baseValue == null || headValue == null) continue; + + const baseSpread = getHeapSnapshotSampleSpread(base, phase, category); + const headSpread = getHeapSnapshotSampleSpread(head, phase, category); + const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category); + const deltaMedian = summary == null ? '-' : `${formatDeltaBytes(summary.median)}
${formatDeltaPercent(summary.median, baseValue)}`; + const categoryLabel = formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal); + + lines.push(`| $\\color{${heapSnapshotCategoriesColors[category]}}{\\rule{8pt}{8pt}}$ ${categoryLabel} | ${formatBytes(baseValue)}
± ${baseSpread == null ? '-' : formatBytes(baseSpread)} | ${formatBytes(headValue)}
± ${headSpread == null ? '-' : formatBytes(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : formatBytes(summary.mad)} | ${summary == null ? '-' : formatDeltaBytes(summary.min)} | ${summary == null ? '-' : formatDeltaBytes(summary.max)} |`); + if (category === 'Total') { + lines.push('| | | | | | | |'); + } + } + + if (lines.length === 2) return null; + return lines.join('\n'); +} + +function renderHeapSnapshotSection(base, head) { + const table = renderHeapSnapshotTable(base, head, 'afterRequest'); + if (table == null) return null; + + const lines = [ + '### V8 Heap Snapshot Statistics', + '', + table, + '', + ]; + + for (const graph of [ + renderHeapSnapshotSankey(base, 'afterRequest', 'Base'), + renderHeapSnapshotSankey(head, 'afterRequest', 'Head'), + ]) { + if (graph == null) continue; + lines.push(graph); + lines.push(''); + } + + return lines.join('\n'); +} + function getJsFootprintValue(report, phase, key) { const value = report?.[phase]?.totals?.[key]; return Number.isFinite(value) ? value : null; @@ -301,7 +545,7 @@ function renderJsFootprintMetricTable(base, head) { const headValue = getJsFootprintValue(head, 'afterRequest', key); if (baseValue == null || headValue == null) continue; - lines.push(`| ${title} | ${formatter(baseValue)} | ${formatter(headValue)} | ${formatPlainDiff(baseValue, headValue, formatter)} | ${formatPlainDiffPercent(baseValue, headValue)} |`); + lines.push(`| **${title}** | ${formatter(baseValue)} | ${formatter(headValue)} | ${formatPlainDiff(baseValue, headValue, formatter)} | ${formatPlainDiffPercent(baseValue, headValue)} |`); } return lines.join('\n'); @@ -443,12 +687,10 @@ function renderJsFootprintSection(base, head) { '', renderJsFootprintMetricTable(base, head), '', - '#### Load Phase Breakdown', - '', - renderJsFootprintPhaseTable(base, head), - '', - '', - '', + //'#### Load Phase Breakdown', + //'', + //renderJsFootprintPhaseTable(base, head), + //'', ]; for (const block of [ @@ -461,6 +703,9 @@ function renderJsFootprintSection(base, head) { lines.push(''); } + lines.push(''); + lines.push(''); + return lines.join('\n'); } @@ -469,7 +714,7 @@ const head = JSON.parse(await readFile(headFile, 'utf8')); const baseJsFootprint = baseJsFootprintFile == null ? null : JSON.parse(await readFile(baseJsFootprintFile, 'utf8')); const headJsFootprint = headJsFootprintFile == null ? null : JSON.parse(await readFile(headJsFootprintFile, 'utf8')); const lines = [ - '## Backend Memory Usage Report', + '## ⚙️ Backend Memory Usage Report', '', ]; @@ -483,14 +728,12 @@ for (const phase of phases) { lines.push(`### ${phase.title}`); lines.push(renderTable(base, head, phase.key)); lines.push(''); +} - const pairedDeltaTable = renderPairedDeltaTable(base, head, phase.key); - if (pairedDeltaTable != null) { - lines.push('#### Paired Delta Summary'); - lines.push(''); - lines.push(pairedDeltaTable); - lines.push(''); - } +const heapSnapshotSection = renderHeapSnapshotSection(base, head); +if (heapSnapshotSection != null) { + lines.push(heapSnapshotSection); + lines.push(''); } const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint); diff --git a/.github/scripts/frontend-js-size.mjs b/.github/scripts/frontend-js-size.mjs index ddfc66e51f..ffd01d9c56 100644 --- a/.github/scripts/frontend-js-size.mjs +++ b/.github/scripts/frontend-js-size.mjs @@ -530,12 +530,68 @@ function renderFrontendBundleReport(before, after) { return lines.join('\n'); } +const visualizerTreemapLimit = 30; + +function mermaidTreemapLabel(value) { + const label = String(value) + .replaceAll('\\', '/') + .replaceAll('"', "'") + .replaceAll('`', "'") + .replaceAll('\r', ' ') + .replaceAll('\n', ' ') + .trim(); + return label === '' ? '(unknown)' : label; +} + +function mermaidTreemapModuleLabel(id) { + const normalizedId = String(id).replaceAll('\\', '/'); + const filePath = normalizedId.split(/[?#]/, 1)[0]; + const fileName = path.posix.basename(filePath); + return mermaidTreemapLabel(fileName || normalizedId); +} + +function renderVisualizerTreemap(label, report) { + const rows = report.hotModules + .filter((row) => row.renderedLength > 0) + .slice(0, visualizerTreemapLimit); + const topRendered = rows.reduce((sum, row) => sum + row.renderedLength, 0); + const otherRendered = Math.max(0, report.metrics.renderedLength - topRendered); + const lines = [ + '```mermaid', + 'treemap-beta', + `"${mermaidTreemapLabel(label)}"`, + ]; + + for (const row of rows) { + lines.push(` "${mermaidTreemapModuleLabel(row.id)}": ${Math.round(row.renderedLength)}`); + } + if (otherRendered > 0) { + lines.push(` "Other": ${Math.round(otherRendered)}`); + } + + lines.push('```'); + return lines.join('\n'); +} + +function renderVisualizerTreemapDetails(label, report, open = false) { + return [ + ``, + `${label} rendered size treemap (top ${visualizerTreemapLimit} + Other)`, + '', + renderVisualizerTreemap(label, report), + '', + '', + ].join('\n'); +} + const args = process.argv.slice(2); const [beforeDir, afterDir, beforeStatsFile, afterStatsFile, outFile] = args; const before = await collectReport(beforeDir); const after = await collectReport(afterDir); const beforeStats = JSON.parse(await fs.readFile(beforeStatsFile, 'utf8')); const afterStats = JSON.parse(await fs.readFile(afterStatsFile, 'utf8')); +const beforeVisualizerReport = collectVisualizerReport(beforeStats); +const afterVisualizerReport = collectVisualizerReport(afterStats); const visualizerArtifactLink = `[Download detailed HTML](${process.env.FRONTEND_BUNDLE_REPORT_ARTIFACT_URL})`; const body = [ @@ -547,7 +603,11 @@ const body = [ '', '## Bundle Stats', '', - renderFrontendBundleReport(collectVisualizerReport(beforeStats), collectVisualizerReport(afterStats)), + renderFrontendBundleReport(beforeVisualizerReport, afterVisualizerReport), + '', + renderVisualizerTreemapDetails('Before', beforeVisualizerReport), + '', + renderVisualizerTreemapDetails('After', afterVisualizerReport, true), '', visualizerArtifactLink, ].join('\n'); diff --git a/.github/scripts/measure-backend-memory-comparison.mjs b/.github/scripts/measure-backend-memory-comparison.mjs index da543570ac..3857499bdd 100644 --- a/.github/scripts/measure-backend-memory-comparison.mjs +++ b/.github/scripts/measure-backend-memory-comparison.mjs @@ -9,6 +9,16 @@ import { writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; const phases = ['beforeGc', 'afterGc', 'afterRequest']; +const heapSnapshotCategories = [ + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', + 'Total', +]; const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2); @@ -27,6 +37,8 @@ function readIntegerEnv(name, defaultValue, min) { return value; } +const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1); + function commandName(command) { if (process.platform !== 'win32') return command; if (command === 'pnpm') return 'pnpm.cmd'; @@ -101,6 +113,51 @@ function median(values) { return Math.round((sorted[center - 1] + sorted[center]) / 2); } +function summarizeHeapSnapshotBreakdowns(samples, phase) { + const breakdowns = {}; + + for (const category of heapSnapshotCategories) { + if (category === 'Total') continue; + + const childKeys = new Set(); + for (const sample of samples) { + for (const childKey of Object.keys(sample[phase]?.heapSnapshot?.breakdowns?.[category] ?? {})) { + childKeys.add(childKey); + } + } + + const categoryBreakdown = {}; + for (const childKey of childKeys) { + const values = samples + .map(sample => sample[phase]?.heapSnapshot?.breakdowns?.[category]?.[childKey]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) categoryBreakdown[childKey] = median(values); + } + + if (Object.keys(categoryBreakdown).length > 0) { + breakdowns[category] = collapseHeapSnapshotBreakdown(categoryBreakdown); + } + } + + return breakdowns; +} + +function collapseHeapSnapshotBreakdown(breakdown) { + const entries = Object.entries(breakdown) + .filter(([, value]) => value > 0) + .toSorted((a, b) => b[1] - a[1]); + + const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N); + const otherValue = entries + .slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N) + .reduce((sum, [, value]) => sum + value, 0); + + const collapsed = Object.fromEntries(topEntries); + if (otherValue > 0) collapsed.Other = otherValue; + return collapsed; +} + function summarizeSamples(samples) { const summary = {}; @@ -121,6 +178,34 @@ function summarizeSamples(samples) { if (values.length > 0) summary[phase][key] = median(values); } + + const heapSnapshotCategoryValues = {}; + for (const category of heapSnapshotCategories) { + const values = samples + .map(sample => sample[phase]?.heapSnapshot?.categories?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotCategoryValues[category] = median(values); + } + + const heapSnapshotNodeCountValues = {}; + for (const category of heapSnapshotCategories) { + const values = samples + .map(sample => sample[phase]?.heapSnapshot?.nodeCounts?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values); + } + + if (Object.keys(heapSnapshotCategoryValues).length > 0) { + const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(samples, phase); + + summary[phase].heapSnapshot = { + categories: heapSnapshotCategoryValues, + nodeCounts: heapSnapshotNodeCountValues, + ...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}), + }; + } } return summary; @@ -138,12 +223,15 @@ async function measureRepo(label, repoDir, round, orderIndex) { }); process.stderr.write(`[${label}] Measuring memory\n`); + const measureEnv = { + ...process.env, + MK_MEMORY_SAMPLE_COUNT: '1', + }; + if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0'; + const stdout = await run('node', ['packages/backend/scripts/measure-memory.mjs'], { cwd: repoDir, - env: { - ...process.env, - MK_MEMORY_SAMPLE_COUNT: '1', - }, + env: measureEnv, }); const report = JSON.parse(stdout); diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index 9f5af707c7..c21c516721 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -93,6 +93,7 @@ jobs: env: MK_MEMORY_COMPARE_ROUNDS: 5 MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1 + MK_MEMORY_HEAP_SNAPSHOT: 1 run: node head/.github/scripts/measure-backend-memory-comparison.mjs base head memory-base.json memory-head.json - name: Measure backend loaded JS footprint run: | diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index beefcd7ed0..b3a7056e1a 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,6 +14,7 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; import * as http from 'node:http'; import * as fs from 'node:fs/promises'; @@ -30,11 +31,22 @@ function readIntegerEnv(name, defaultValue, min) { return value; } +function readBooleanEnv(name, defaultValue) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (rawValue === '1' || rawValue === 'true') return true; + if (rawValue === '0' || rawValue === 'false') return false; + throw new Error(`${name} must be one of: 1, 0, true, false`); +} + const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0); +const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false); +const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1); +const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1); const procStatusKeys = { VmPeak: 0, @@ -74,6 +86,45 @@ const memoryKeys = { const phases = ['beforeGc', 'afterGc', 'afterRequest']; +const heapSnapshotCategories = [ + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', + 'Total', +]; + +const typedArrayNames = new Set([ + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'system / JSArrayBufferData', +]); + +const otherJsNodeTypes = new Set([ + 'object', + 'closure', + 'regexp', + 'number', + 'symbol', + 'bigint', +]); + function parseMemoryFile(content, keys, path, required) { const result = {}; for (const key of Object.keys(keys)) { @@ -91,6 +142,161 @@ function bytesToKiB(value) { return Math.round(value / 1024); } +function createEmptyHeapSnapshotCategoryMap() { + return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0])); +} + +function isTypedArrayNode(type, name) { + return typedArrayNames.has(name) || + (type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray'))); +} + +function isSystemNode(type, name) { + return type === 'hidden' || + type === 'synthetic' || + type === 'object shape' || + name.startsWith('system /') || + name.startsWith('(system '); +} + +function classifyHeapSnapshotNode(type, name) { + if (type === 'code') return 'Code'; + if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'Strings'; + if (isTypedArrayNode(type, name)) return 'Typed arrays'; + if (type === 'array' || (type === 'object' && name === 'Array')) return 'JS arrays'; + if (isSystemNode(type, name)) return 'System objects'; + if (otherJsNodeTypes.has(type)) return 'Other JS objects'; + return 'Other non-JS objects'; +} + +function addValue(map, key, value) { + map[key] = (map[key] ?? 0) + value; +} + +function sanitizeHeapSnapshotBreakdownLabel(value, fallback = 'unknown') { + const label = String(value ?? '').replace(/\s+/g, ' ').trim(); + if (label === '') return fallback; + if (label.length <= 80) return label; + return `${label.slice(0, 77)}...`; +} + +function classifyHeapSnapshotBreakdown(category, type, name) { + if (category === 'Strings') return type; + + if (category === 'JS arrays') { + if (type === 'array') return 'array nodes'; + if (type === 'object' && name === 'Array') return 'Array objects'; + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`); + } + + if (category === 'Typed arrays') { + if (name === 'system / JSArrayBufferData') return 'ArrayBuffer data'; + if (name === 'Uint8Array') return 'Uint8Array / Buffer'; + if (typedArrayNames.has(name)) return name; + if (type === 'native' && name.includes('ArrayBuffer')) return 'native ArrayBuffer'; + if (type === 'native' && name.includes('TypedArray')) return 'native TypedArray'; + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`); + } + + if (category === 'System objects') { + if (name.startsWith('system /')) return sanitizeHeapSnapshotBreakdownLabel(name); + if (name.startsWith('(system ')) return sanitizeHeapSnapshotBreakdownLabel(name); + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); + } + + if (category === 'Other JS objects') { + if (type === 'object') return sanitizeHeapSnapshotBreakdownLabel(`object: ${name}`, 'object: unknown'); + return type; + } + + if (category === 'Other non-JS objects') { + if (type === 'native') return sanitizeHeapSnapshotBreakdownLabel(`native: ${name}`, 'native: unknown'); + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); + } + + if (category === 'Code') { + const lowerName = name.toLowerCase(); + if (lowerName.includes('bytecode')) return 'bytecode'; + if (lowerName.includes('builtin')) return 'builtins'; + if (lowerName.includes('regexp')) return 'regexp code'; + if (lowerName.includes('stub')) return 'stubs'; + return sanitizeHeapSnapshotBreakdownLabel(`code: ${name}`, 'code: unknown'); + } + + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); +} + +function collapseHeapSnapshotBreakdown(breakdowns) { + const collapsed = {}; + + for (const [category, children] of Object.entries(breakdowns)) { + const entries = Object.entries(children) + .filter(([, value]) => value > 0) + .toSorted((a, b) => b[1] - a[1]); + + const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N); + const otherValue = entries + .slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N) + .reduce((sum, [, value]) => sum + value, 0); + + const categoryBreakdown = Object.fromEntries(topEntries); + if (otherValue > 0) categoryBreakdown.Other = otherValue; + if (Object.keys(categoryBreakdown).length > 0) collapsed[category] = categoryBreakdown; + } + + return collapsed; +} + +function analyzeHeapSnapshot(snapshot) { + const meta = snapshot?.snapshot?.meta; + const nodes = snapshot?.nodes; + const strings = snapshot?.strings; + if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) { + throw new Error('Invalid heap snapshot format'); + } + + const nodeFields = meta.node_fields; + if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields'); + + const typeOffset = nodeFields.indexOf('type'); + const nameOffset = nodeFields.indexOf('name'); + const selfSizeOffset = nodeFields.indexOf('self_size'); + if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) { + throw new Error('Heap snapshot is missing required node fields'); + } + + const nodeTypeNames = meta.node_types?.[typeOffset]; + if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types'); + + const fieldCount = nodeFields.length; + const categories = createEmptyHeapSnapshotCategoryMap(); + const nodeCounts = createEmptyHeapSnapshotCategoryMap(); + const breakdowns = Object.fromEntries( + heapSnapshotCategories + .filter(category => category !== 'Total') + .map(category => [category, {}]), + ); + + for (let offset = 0; offset < nodes.length; offset += fieldCount) { + const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown'; + const name = strings[nodes[offset + nameOffset]] ?? ''; + const selfSize = nodes[offset + selfSizeOffset] ?? 0; + const category = classifyHeapSnapshotNode(type, name); + + categories[category] += selfSize; + categories.Total += selfSize; + nodeCounts[category]++; + nodeCounts.Total++; + addValue(breakdowns[category], classifyHeapSnapshotBreakdown(category, type, name), selfSize); + } + + return { + categories, + nodeCounts, + breakdowns: collapseHeapSnapshotBreakdown(breakdowns), + }; +} + async function getMemoryUsage(pid) { const path = `/proc/${pid}/status`; const status = await fs.readFile(path, 'utf-8'); @@ -150,6 +356,39 @@ async function getRuntimeMemoryUsage(serverProcess) { }; } +async function getHeapSnapshotStatistics(serverProcess) { + if (!HEAP_SNAPSHOT) return null; + + const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`); + const response = waitForMessage( + serverProcess, + message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'), + 'heap snapshot', + HEAP_SNAPSHOT_TIMEOUT, + ); + + serverProcess.send({ + type: 'heap snapshot', + path: snapshotPath, + }); + + const message = await response; + if (message.type === 'heap snapshot error') { + throw new Error(`Failed to write heap snapshot: ${message.message}`); + } + + const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath; + + try { + const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8')); + return analyzeHeapSnapshot(snapshot); + } finally { + await fs.unlink(writtenPath).catch(err => { + process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`); + }); + } +} + async function getAllMemoryUsage(serverProcess) { const pid = serverProcess.pid; return { @@ -166,6 +405,36 @@ function median(values) { return Math.round((sorted[center - 1] + sorted[center]) / 2); } +function summarizeHeapSnapshotBreakdowns(results, phase) { + const breakdowns = {}; + + for (const category of heapSnapshotCategories) { + if (category === 'Total') continue; + + const childKeys = new Set(); + for (const result of results) { + for (const childKey of Object.keys(result[phase]?.heapSnapshot?.breakdowns?.[category] ?? {})) { + childKeys.add(childKey); + } + } + + const categoryBreakdown = {}; + for (const childKey of childKeys) { + const values = results + .map(result => result[phase]?.heapSnapshot?.breakdowns?.[category]?.[childKey]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) categoryBreakdown[childKey] = median(values); + } + + if (Object.keys(categoryBreakdown).length > 0) { + breakdowns[category] = collapseHeapSnapshotBreakdown({ [category]: categoryBreakdown })[category] ?? categoryBreakdown; + } + } + + return breakdowns; +} + function summarizeResults(results) { const summary = {}; @@ -180,6 +449,34 @@ function summarizeResults(results) { summary[phase][key] = median(values); } } + + const heapSnapshotCategoryValues = {}; + for (const category of heapSnapshotCategories) { + const values = results + .map(result => result[phase]?.heapSnapshot?.categories?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotCategoryValues[category] = median(values); + } + + const heapSnapshotNodeCountValues = {}; + for (const category of heapSnapshotCategories) { + const values = results + .map(result => result[phase]?.heapSnapshot?.nodeCounts?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values); + } + + if (Object.keys(heapSnapshotCategoryValues).length > 0) { + const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(results, phase); + + summary[phase].heapSnapshot = { + categories: heapSnapshotCategoryValues, + nodeCounts: heapSnapshotNodeCountValues, + ...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}), + }; + } } return summary; @@ -290,6 +587,8 @@ async function measureMemory() { await triggerGc(); const afterRequest = await getAllMemoryUsage(serverProcess); + const heapSnapshot = await getHeapSnapshotStatistics(serverProcess); + if (heapSnapshot != null) afterRequest.heapSnapshot = heapSnapshot; // Stop the server serverProcess.kill('SIGTERM'); @@ -339,6 +638,11 @@ async function main() { memorySettleTimeMs: MEMORY_SETTLE_TIME, ipcTimeoutMs: IPC_TIMEOUT, requestCount: REQUEST_COUNT, + heapSnapshot: { + enabled: HEAP_SNAPSHOT, + timeoutMs: HEAP_SNAPSHOT_TIMEOUT, + breakdownTopN: HEAP_SNAPSHOT_BREAKDOWN_TOP_N, + }, }, ...summary, samples: results, diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index b5f258e9ee..9fcb4d73ca 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -9,6 +9,7 @@ import cluster from 'node:cluster'; import { EventEmitter } from 'node:events'; +import { writeHeapSnapshot } from 'node:v8'; import chalk from 'chalk'; import Xev from 'xev'; import Logger from '@/logger.js'; @@ -106,6 +107,21 @@ process.on('message', msg => { value: process.memoryUsage(), }); } + } else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') { + if (process.send != null) { + try { + const path = writeHeapSnapshot(msg.path); + process.send({ + type: 'heap snapshot', + path, + }); + } catch (err) { + process.send({ + type: 'heap snapshot error', + message: err instanceof Error ? err.message : String(err), + }); + } + } } });