diff --git a/.github/scripts/backend-memory-report.mjs b/.github/scripts/backend-memory-report.mjs index 6ea8a5b5af..ab26e92233 100644 --- a/.github/scripts/backend-memory-report.mjs +++ b/.github/scripts/backend-memory-report.mjs @@ -26,6 +26,17 @@ const metrics = [ 'External', ]; +const heapSnapshotCategories = [ + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', + 'Total', +]; + function formatNumber(value) { return numberFormatter.format(value); } @@ -273,6 +284,143 @@ function formatPlainDiffPercent(baseValue, headValue) { return `${sign}${formatPercent(Math.abs((diff * 100) / baseValue))}`; } +function getHeapSnapshotCategoryValue(report, phase, category) { + const value = report?.[phase]?.heapSnapshot?.categories?.[category]; + return Number.isFinite(value) ? value : null; +} + +function getHeapSnapshotSampleValues(report, phase, category) { + if (!Array.isArray(report?.samples)) return []; + + return report.samples + .map(sample => getHeapSnapshotCategoryValue(sample, phase, category)) + .filter(value => Number.isFinite(value)); +} + +function getHeapSnapshotSampleSpread(report, phase, category) { + const values = getHeapSnapshotSampleValues(report, phase, category); + if (values.length < 2) return null; + + const center = median(values); + return median(values.map(value => Math.abs(value - center))); +} + +function formatDiffBytes(baseBytes, headBytes) { + const diff = headBytes - baseBytes; + if (diff === 0) return formatBytes(0); + + const sign = diff > 0 ? '+' : '-'; + return formatColoredDiff(`${sign}${formatBytes(Math.abs(diff))}`, diff); +} + +function formatDiffBytesPercent(baseBytes, headBytes) { + const diff = headBytes - baseBytes; + if (diff === 0) return '0%'; + if (baseBytes <= 0) return '-'; + + const sign = diff > 0 ? '+' : '-'; + return formatColoredDiff(`${sign}${formatPercent(Math.abs((diff * 100) / baseBytes))}`, diff); +} + +function getPairedHeapSnapshotDeltaValues(base, head, phase, category) { + const baseSamplesByRound = getSamplesByRound(base); + const headSamplesByRound = getSamplesByRound(head); + const values = []; + + for (const [round, baseSample] of baseSamplesByRound) { + const headSample = headSamplesByRound.get(round); + if (headSample == null) continue; + + const baseValue = getHeapSnapshotCategoryValue(baseSample, phase, category); + const headValue = getHeapSnapshotCategoryValue(headSample, phase, category); + if (baseValue == null || headValue == null) continue; + + values.push(headValue - baseValue); + } + + return values; +} + +function formatDeltaBytes(diffBytes) { + if (diffBytes === 0) return formatBytes(0); + + const sign = diffBytes > 0 ? '+' : '-'; + return formatColoredDiff(`${sign}${formatBytes(Math.abs(diffBytes))}`, diffBytes); +} + +function pairedHeapSnapshotDeltaSummary(base, head, phase, category) { + const values = getPairedHeapSnapshotDeltaValues(base, head, phase, category); + if (values.length === 0) return null; + + return { + median: median(values), + mad: mad(values), + min: Math.min(...values), + max: Math.max(...values), + samples: values.length, + }; +} + +function renderHeapSnapshotTable(base, head, phase) { + const lines = [ + '| Category | Base | Head | Δ | Δ (%) |', + '| --- | ---: | ---: | ---: | ---: |', + ]; + + for (const category of heapSnapshotCategories) { + const baseValue = getHeapSnapshotCategoryValue(base, phase, category); + const headValue = getHeapSnapshotCategoryValue(head, phase, category); + if (baseValue == null || headValue == null) continue; + + const baseSpread = getHeapSnapshotSampleSpread(base, phase, category); + const headSpread = getHeapSnapshotSampleSpread(head, phase, category); + + lines.push(`| ${category} | ${formatBytes(baseValue)}
± ${baseSpread == null ? '-' : formatBytes(baseSpread)} | ${formatBytes(headValue)}
± ${headSpread == null ? '-' : formatBytes(headSpread)} | ${formatDiffBytes(baseValue, headValue)} | ${formatDiffBytesPercent(baseValue, headValue)} |`); + } + + if (lines.length === 2) return null; + return lines.join('\n'); +} + +function renderHeapSnapshotPairedDeltaTable(base, head, phase) { + const lines = [ + '| Category | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: |', + ]; + + for (const category of heapSnapshotCategories) { + const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category); + if (summary == null) continue; + + lines.push(`| ${category} | ${formatDeltaBytes(summary.median)} | ${summary.mad == null ? '-' : formatBytes(summary.mad)} | ${formatDeltaBytes(summary.min)} | ${formatDeltaBytes(summary.max)} |`); + } + + if (lines.length === 2) return null; + return lines.join('\n'); +} + +function renderHeapSnapshotSection(base, head) { + const table = renderHeapSnapshotTable(base, head, 'afterRequest'); + if (table == null) return null; + + const lines = [ + '### V8 Heap Snapshot Statistics', + '', + table, + '', + ]; + + const pairedDeltaTable = renderHeapSnapshotPairedDeltaTable(base, head, 'afterRequest'); + if (pairedDeltaTable != null) { + lines.push('#### Paired Delta Summary'); + lines.push(''); + lines.push(pairedDeltaTable); + lines.push(''); + } + + return lines.join('\n'); +} + function getJsFootprintValue(report, phase, key) { const value = report?.[phase]?.totals?.[key]; return Number.isFinite(value) ? value : null; @@ -494,6 +642,12 @@ for (const phase of phases) { } } +const heapSnapshotSection = renderHeapSnapshotSection(base, head); +if (heapSnapshotSection != null) { + lines.push(heapSnapshotSection); + lines.push(''); +} + const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint); if (jsFootprintSection != null) { lines.push(jsFootprintSection); diff --git a/.github/scripts/measure-backend-memory-comparison.mjs b/.github/scripts/measure-backend-memory-comparison.mjs index da543570ac..25b8942572 100644 --- a/.github/scripts/measure-backend-memory-comparison.mjs +++ b/.github/scripts/measure-backend-memory-comparison.mjs @@ -9,6 +9,16 @@ import { writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; const phases = ['beforeGc', 'afterGc', 'afterRequest']; +const heapSnapshotCategories = [ + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', + 'Total', +]; const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2); @@ -121,6 +131,31 @@ function summarizeSamples(samples) { if (values.length > 0) summary[phase][key] = median(values); } + + const heapSnapshotCategoryValues = {}; + for (const category of heapSnapshotCategories) { + const values = samples + .map(sample => sample[phase]?.heapSnapshot?.categories?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotCategoryValues[category] = median(values); + } + + const heapSnapshotNodeCountValues = {}; + for (const category of heapSnapshotCategories) { + const values = samples + .map(sample => sample[phase]?.heapSnapshot?.nodeCounts?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values); + } + + if (Object.keys(heapSnapshotCategoryValues).length > 0) { + summary[phase].heapSnapshot = { + categories: heapSnapshotCategoryValues, + nodeCounts: heapSnapshotNodeCountValues, + }; + } } return summary; @@ -138,12 +173,15 @@ async function measureRepo(label, repoDir, round, orderIndex) { }); process.stderr.write(`[${label}] Measuring memory\n`); + const measureEnv = { + ...process.env, + MK_MEMORY_SAMPLE_COUNT: '1', + }; + if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0'; + const stdout = await run('node', ['packages/backend/scripts/measure-memory.mjs'], { cwd: repoDir, - env: { - ...process.env, - MK_MEMORY_SAMPLE_COUNT: '1', - }, + env: measureEnv, }); const report = JSON.parse(stdout); diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index 9f5af707c7..c21c516721 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -93,6 +93,7 @@ jobs: env: MK_MEMORY_COMPARE_ROUNDS: 5 MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1 + MK_MEMORY_HEAP_SNAPSHOT: 1 run: node head/.github/scripts/measure-backend-memory-comparison.mjs base head memory-base.json memory-head.json - name: Measure backend loaded JS footprint run: | diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index beefcd7ed0..8d2c1e1e77 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -14,6 +14,7 @@ import { fork } from 'node:child_process'; import { setTimeout } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; import * as http from 'node:http'; import * as fs from 'node:fs/promises'; @@ -30,11 +31,21 @@ function readIntegerEnv(name, defaultValue, min) { return value; } +function readBooleanEnv(name, defaultValue) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (rawValue === '1' || rawValue === 'true') return true; + if (rawValue === '0' || rawValue === 'false') return false; + throw new Error(`${name} must be one of: 1, 0, true, false`); +} + const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0); +const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false); +const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1); const procStatusKeys = { VmPeak: 0, @@ -74,6 +85,45 @@ const memoryKeys = { const phases = ['beforeGc', 'afterGc', 'afterRequest']; +const heapSnapshotCategories = [ + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', + 'Total', +]; + +const typedArrayNames = new Set([ + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'system / JSArrayBufferData', +]); + +const otherJsNodeTypes = new Set([ + 'object', + 'closure', + 'regexp', + 'number', + 'symbol', + 'bigint', +]); + function parseMemoryFile(content, keys, path, required) { const result = {}; for (const key of Object.keys(keys)) { @@ -91,6 +141,76 @@ function bytesToKiB(value) { return Math.round(value / 1024); } +function createEmptyHeapSnapshotCategoryMap() { + return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0])); +} + +function isTypedArrayNode(type, name) { + return typedArrayNames.has(name) || + (type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray'))); +} + +function isSystemNode(type, name) { + return type === 'hidden' || + type === 'synthetic' || + type === 'object shape' || + name.startsWith('system /') || + name.startsWith('(system '); +} + +function classifyHeapSnapshotNode(type, name) { + if (type === 'code') return 'Code'; + if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'Strings'; + if (isTypedArrayNode(type, name)) return 'Typed arrays'; + if (type === 'array' || (type === 'object' && name === 'Array')) return 'JS arrays'; + if (isSystemNode(type, name)) return 'System objects'; + if (otherJsNodeTypes.has(type)) return 'Other JS objects'; + return 'Other non-JS objects'; +} + +function analyzeHeapSnapshot(snapshot) { + const meta = snapshot?.snapshot?.meta; + const nodes = snapshot?.nodes; + const strings = snapshot?.strings; + if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) { + throw new Error('Invalid heap snapshot format'); + } + + const nodeFields = meta.node_fields; + if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields'); + + const typeOffset = nodeFields.indexOf('type'); + const nameOffset = nodeFields.indexOf('name'); + const selfSizeOffset = nodeFields.indexOf('self_size'); + if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) { + throw new Error('Heap snapshot is missing required node fields'); + } + + const nodeTypeNames = meta.node_types?.[typeOffset]; + if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types'); + + const fieldCount = nodeFields.length; + const categories = createEmptyHeapSnapshotCategoryMap(); + const nodeCounts = createEmptyHeapSnapshotCategoryMap(); + + for (let offset = 0; offset < nodes.length; offset += fieldCount) { + const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown'; + const name = strings[nodes[offset + nameOffset]] ?? ''; + const selfSize = nodes[offset + selfSizeOffset] ?? 0; + const category = classifyHeapSnapshotNode(type, name); + + categories[category] += selfSize; + categories.Total += selfSize; + nodeCounts[category]++; + nodeCounts.Total++; + } + + return { + categories, + nodeCounts, + }; +} + async function getMemoryUsage(pid) { const path = `/proc/${pid}/status`; const status = await fs.readFile(path, 'utf-8'); @@ -150,6 +270,39 @@ async function getRuntimeMemoryUsage(serverProcess) { }; } +async function getHeapSnapshotStatistics(serverProcess) { + if (!HEAP_SNAPSHOT) return null; + + const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`); + const response = waitForMessage( + serverProcess, + message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'), + 'heap snapshot', + HEAP_SNAPSHOT_TIMEOUT, + ); + + serverProcess.send({ + type: 'heap snapshot', + path: snapshotPath, + }); + + const message = await response; + if (message.type === 'heap snapshot error') { + throw new Error(`Failed to write heap snapshot: ${message.message}`); + } + + const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath; + + try { + const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8')); + return analyzeHeapSnapshot(snapshot); + } finally { + await fs.unlink(writtenPath).catch(err => { + process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`); + }); + } +} + async function getAllMemoryUsage(serverProcess) { const pid = serverProcess.pid; return { @@ -180,6 +333,31 @@ function summarizeResults(results) { summary[phase][key] = median(values); } } + + const heapSnapshotCategoryValues = {}; + for (const category of heapSnapshotCategories) { + const values = results + .map(result => result[phase]?.heapSnapshot?.categories?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotCategoryValues[category] = median(values); + } + + const heapSnapshotNodeCountValues = {}; + for (const category of heapSnapshotCategories) { + const values = results + .map(result => result[phase]?.heapSnapshot?.nodeCounts?.[category]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) heapSnapshotNodeCountValues[category] = median(values); + } + + if (Object.keys(heapSnapshotCategoryValues).length > 0) { + summary[phase].heapSnapshot = { + categories: heapSnapshotCategoryValues, + nodeCounts: heapSnapshotNodeCountValues, + }; + } } return summary; @@ -290,6 +468,8 @@ async function measureMemory() { await triggerGc(); const afterRequest = await getAllMemoryUsage(serverProcess); + const heapSnapshot = await getHeapSnapshotStatistics(serverProcess); + if (heapSnapshot != null) afterRequest.heapSnapshot = heapSnapshot; // Stop the server serverProcess.kill('SIGTERM'); @@ -339,6 +519,10 @@ async function main() { memorySettleTimeMs: MEMORY_SETTLE_TIME, ipcTimeoutMs: IPC_TIMEOUT, requestCount: REQUEST_COUNT, + heapSnapshot: { + enabled: HEAP_SNAPSHOT, + timeoutMs: HEAP_SNAPSHOT_TIMEOUT, + }, }, ...summary, samples: results, diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index b5f258e9ee..9fcb4d73ca 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -9,6 +9,7 @@ import cluster from 'node:cluster'; import { EventEmitter } from 'node:events'; +import { writeHeapSnapshot } from 'node:v8'; import chalk from 'chalk'; import Xev from 'xev'; import Logger from '@/logger.js'; @@ -106,6 +107,21 @@ process.on('message', msg => { value: process.memoryUsage(), }); } + } else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') { + if (process.send != null) { + try { + const path = writeHeapSnapshot(msg.path); + process.send({ + type: 'heap snapshot', + path, + }); + } catch (err) { + process.send({ + type: 'heap snapshot error', + message: err instanceof Error ? err.message : String(err), + }); + } + } } });