diff --git a/.github/scripts/measure-backend-memory-comparison.mts b/.github/scripts/measure-backend-memory-comparison.mts index 2e7e67fe5a..ee95c68165 100644 --- a/.github/scripts/measure-backend-memory-comparison.mts +++ b/.github/scripts/measure-backend-memory-comparison.mts @@ -8,30 +8,19 @@ import { copyFile, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import * as util from './utility.mts'; import * as heapSnapshotUtil from './heap-snapshot-util.mts'; -import type { MemoryReportRaw } from '../../packages/backend/scripts/measure-memory.mts'; +import type { MemorySample } from '../../packages/backend/scripts/measure-memory.mts'; const phases = ['afterGc'] as const; export type MemoryReport = { timestamp: string; - sampleCount: any; + sampleCount: number; aggregation: string; - measurement: { - startupTimeoutMs: any; - memorySettleTimeMs: any; - ipcTimeoutMs: any; - requestCount: any; - heapSnapshot: { - enabled: any; - timeoutMs: any; - breakdownTopN: any; - }; - }; summary: Record; heapSnapshot?: heapSnapshotUtil.HeapSnapshotData; }>; - samples: (MemoryReportRaw['samples'][number] & { + samples: (MemorySample & { round: number; })[]; }; @@ -41,6 +30,8 @@ const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slic const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = util.readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', heapSnapshotUtil.defaultHeapSnapshotBreakdownTopN, 1); const HEAD_HEAP_SNAPSHOT_WORK_DIR = resolve('head-heap-snapshots'); const HEAD_HEAP_SNAPSHOT_OUTPUT_PATH = resolve('head-heap-snapshot.heapsnapshot'); +// Use the head checkout's measurement harness for both targets so only the built backend differs. +const MEASURE_MEMORY_SCRIPT = resolve(import.meta.dirname, '../../packages/backend/scripts/measure-memory.mts'); async function resetState(repoDir: string) { const require = createRequire(join(repoDir, 'packages/backend/package.json')); @@ -115,20 +106,17 @@ async function measureRepo(label: string, repoDir: string, round: number, option process.stderr.write(`[${label}] Measuring memory\n`); const measureEnv = { ...process.env, - MK_MEMORY_SAMPLE_COUNT: '1', + MK_MEMORY_BACKEND_DIR: resolve(repoDir, 'packages/backend'), } as NodeJS.ProcessEnv; if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0'; if (options.heapSnapshotSavePath != null) measureEnv.MK_MEMORY_HEAP_SNAPSHOT_SAVE_PATH = options.heapSnapshotSavePath; - const stdout = await util.run('node', ['packages/backend/scripts/measure-memory.mts'], { + const stdout = await util.run('node', [MEASURE_MEMORY_SCRIPT], { cwd: repoDir, env: measureEnv, }); - const report = JSON.parse(stdout) as MemoryReportRaw; - const sample = report.samples[0]; - - return sample; + return JSON.parse(stdout) as MemorySample; } function headHeapSnapshotPath(round: number) { diff --git a/.github/scripts/memory-stability-util.mts b/.github/scripts/memory-stability-util.mts new file mode 100644 index 0000000000..0a9a8d4f52 --- /dev/null +++ b/.github/scripts/memory-stability-util.mts @@ -0,0 +1,75 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { setTimeout } from 'node:timers/promises'; + +type MemoryStabilityTimer = { + now: () => number; + wait: (durationMs: number) => Promise; +}; + +const intervalMs = 2000; +const maxWaitMs = 10000; +const windowSize = 3; +const slopeThresholdKiBPerSecond = 256; +const stabilityMetrics = ['Pss', 'Private_Dirty'] as const; + +const defaultTimer: MemoryStabilityTimer = { + now: () => performance.now(), + wait: durationMs => setTimeout(durationMs), +}; + +function getMaxAbsoluteSlopes>(readings: { elapsedMs: number; memoryUsage: T }[]) { + const result = {} as Record; + + for (const metric of stabilityMetrics) { + let maxAbsoluteSlope = 0; + for (let i = 1; i < readings.length; i++) { + const previous = readings[i - 1]; + const current = readings[i]; + const durationSeconds = (current.elapsedMs - previous.elapsedMs) / 1000; + maxAbsoluteSlope = Math.max(maxAbsoluteSlope, Math.abs(current.memoryUsage[metric] - previous.memoryUsage[metric]) / durationSeconds); + } + result[metric] = maxAbsoluteSlope; + } + + return result; +} + +export async function measureMemoryUntilStable>( + readMemoryUsage: () => Promise, + timer: MemoryStabilityTimer = defaultTimer, +) { + const startedAt = timer.now(); + const readings: { elapsedMs: number; memoryUsage: T }[] = []; + let maxAbsoluteSlopesKiBPerSecond: Record | null = null; + + while (true) { + const memoryUsage = await readMemoryUsage(); + const elapsedMs = timer.now() - startedAt; + readings.push({ elapsedMs, memoryUsage }); + + let converged = false; + if (readings.length >= windowSize) { + const latestSlopes = getMaxAbsoluteSlopes(readings.slice(-windowSize)); + maxAbsoluteSlopesKiBPerSecond = latestSlopes; + converged = stabilityMetrics.every(metric => latestSlopes[metric] <= slopeThresholdKiBPerSecond); + } + + if (converged || elapsedMs >= maxWaitMs) { + return { + memoryUsage, + stability: { + converged, + readingCount: readings.length, + elapsedMs, + maxAbsoluteSlopesKiBPerSecond, + }, + }; + } + + await timer.wait(Math.min(intervalMs, maxWaitMs - elapsedMs)); + } +} diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index 400a81a7d9..0d0bec3307 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -12,6 +12,7 @@ on: - .github/scripts/utility.mts - .github/scripts/backend-memory-report.mts - .github/scripts/measure-backend-memory-comparison.mts + - .github/scripts/memory-stability-util*.mts - .github/scripts/backend-js-footprint.mjs - .github/scripts/backend-js-footprint-loader.mjs - .github/scripts/backend-js-footprint-require.cjs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d3c745aea6..c8e24fb19b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,6 +19,7 @@ on: - packages/shared/eslint.config.js - scripts/check-dts*.mjs - .github/scripts/frontend-js-size*.mts + - .github/scripts/memory-stability-util*.mts - .github/scripts/utility.mts - .github/workflows/lint.yml - package.json @@ -37,6 +38,7 @@ on: - packages/shared/eslint.config.js - scripts/check-dts*.mjs - .github/scripts/frontend-js-size*.mts + - .github/scripts/memory-stability-util*.mts - .github/scripts/utility.mts - .github/workflows/lint.yml - package.json diff --git a/packages/backend/scripts/measure-memory.mts b/packages/backend/scripts/measure-memory.mts index eb7a9c037a..55aab61a84 100644 --- a/packages/backend/scripts/measure-memory.mts +++ b/packages/backend/scripts/measure-memory.mts @@ -4,16 +4,15 @@ */ import { ChildProcess, fork } from 'node:child_process'; -import { setTimeout } from 'node:timers/promises'; -import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { tmpdir } from 'node:os'; -//import * as http from 'node:http'; import * as fs from 'node:fs/promises'; import { analyzeHeapSnapshot, defaultHeapSnapshotBreakdownTopN, type HeapSnapshotData } from '../../../.github/scripts/heap-snapshot-util.mts'; +import { measureMemoryUntilStable } from '../../../.github/scripts/memory-stability-util.mts'; -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); +const backendDir = process.env.MK_MEMORY_BACKEND_DIR == null || process.env.MK_MEMORY_BACKEND_DIR === '' + ? join(import.meta.dirname, '..') + : resolve(process.env.MK_MEMORY_BACKEND_DIR); function readIntegerEnv(name, defaultValue, min) { const rawValue = process.env[name]; @@ -33,11 +32,8 @@ function readBooleanEnv(name, defaultValue) { throw new Error(`${name} must be one of: 1, 0, true, false`); } -const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup -const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses -const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0); const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false); const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1); const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', defaultHeapSnapshotBreakdownTopN, 1); @@ -187,17 +183,23 @@ async function getHeapSnapshotStatistics(serverProcess: ChildProcess): Promise getSmapsRollupMemoryUsage(pid), + ); + return { - ...await getMemoryUsage(pid), - ...await getSmapsRollupMemoryUsage(pid), - ...await getRuntimeMemoryUsage(serverProcess), + memoryUsage: { + ...await getMemoryUsage(pid), + ...stableSmapsRollup.memoryUsage, + ...await getRuntimeMemoryUsage(serverProcess), + }, + stability: stableSmapsRollup.stability, }; } async function measureMemory() { - // Start the Misskey backend server using fork to enable IPC - const serverProcess = fork(join(__dirname, '../built/entry.js'), [], { - cwd: join(__dirname, '..'), + const serverProcess = fork(join(backendDir, 'built/entry.js'), [], { + cwd: backendDir, env: { ...process.env, NODE_ENV: 'production', @@ -209,16 +211,13 @@ async function measureMemory() { execArgv: [...process.execArgv, '--expose-gc'], }); - let serverReady = false; + const serverReady = waitForMessage( + serverProcess, + (message): message is 'ok' => message === 'ok', + 'server startup', + STARTUP_TIMEOUT, + ); - // Listen for the 'ok' message from the server indicating it's ready - serverProcess.on('message', (message) => { - if (message === 'ok') { - serverReady = true; - } - }); - - // Handle server output serverProcess.stdout?.on('data', (data) => { process.stderr.write(`[server stdout] ${data}`); }); @@ -227,7 +226,6 @@ async function measureMemory() { process.stderr.write(`[server stderr] ${data}`); }); - // Handle server error serverProcess.on('error', (err) => { process.stderr.write(`[server error] ${err}\n`); }); @@ -245,141 +243,55 @@ async function measureMemory() { if (message === 'gc unavailable') { throw new Error('GC is unavailable. Start the process with --expose-gc to enable this feature.'); } - - await setTimeout(1000); } - //function createRequest() { - // return new Promise((resolve, reject) => { - // const req = http.request({ - // host: 'localhost', - // port: 61812, - // path: '/api/meta', - // method: 'POST', - // }, (res) => { - // res.on('data', () => { }); - // res.on('end', () => { - // resolve(); - // }); - // }); - // req.on('error', (err) => { - // reject(err); - // }); - // req.end(); - // }); - //} - - // Wait for server to be ready or timeout const startupStartTime = Date.now(); - while (!serverReady) { - if (Date.now() - startupStartTime > STARTUP_TIMEOUT) { - serverProcess.kill('SIGTERM'); - throw new Error('Server startup timeout'); - } - await setTimeout(100); + try { + await serverReady; + } catch (err) { + serverProcess.kill('SIGTERM'); + throw err; } const startupTime = Date.now() - startupStartTime; process.stderr.write(`Server started in ${startupTime}ms\n`); - // Wait for memory to settle - await setTimeout(MEMORY_SETTLE_TIME); - - //const beforeGc = await getAllMemoryUsage(serverProcess); - await triggerGc(); - const memoryUsageAfterGC = await getAllMemoryUsage(serverProcess); - - //// create some http requests to simulate load - //await Promise.all( - // Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), - //); - - //await triggerGc(); - - //const afterRequest = await getAllMemoryUsage(serverProcess); + const afterGc = await getAllMemoryUsage(serverProcess); + process.stderr.write(`Memory ${afterGc.stability.converged ? 'stabilized' : 'did not stabilize'} after ${afterGc.stability.readingCount} readings over ${Math.round(afterGc.stability.elapsedMs)}ms\n`); const heapSnapshotAfterGc = await getHeapSnapshotStatistics(serverProcess); - // Stop the server - serverProcess.kill('SIGTERM'); - - // Wait for process to exit - let exited = false; - await new Promise((resolve) => { - serverProcess.on('exit', () => { - exited = true; - resolve(undefined); - }); - // Force kill after 10 seconds if not exited - setTimeout(10000).then(() => { - if (!exited) { - serverProcess.kill('SIGKILL'); - } - resolve(undefined); + const serverExited = new Promise(resolve => { + const timer = globalThis.setTimeout(() => { + serverProcess.kill('SIGKILL'); + resolve(); + }, 10000); + serverProcess.once('exit', () => { + globalThis.clearTimeout(timer); + resolve(); }); }); + serverProcess.kill('SIGTERM'); + await serverExited; - const result = { + return { timestamp: new Date().toISOString(), phases: { - //beforeGc, afterGc: { - memoryUsage: memoryUsageAfterGC, + memoryUsage: afterGc.memoryUsage, + memoryStability: afterGc.stability, heapSnapshot: heapSnapshotAfterGc, }, - //afterRequest, }, }; - - return result; } -export type MemoryReportRaw = { - timestamp: string; - sampleCount: number; - measurement: { - startupTimeoutMs: number; - memorySettleTimeMs: number; - ipcTimeoutMs: number; - requestCount: number; - heapSnapshot: { - enabled: boolean; - timeoutMs: number; - breakdownTopN: number; - }; - }; - samples: Awaited>[]; -}; +export type MemorySample = Awaited>; async function main() { - const results = []; - for (let i = 0; i < SAMPLE_COUNT; i++) { - process.stderr.write(`Starting sample ${i + 1}/${SAMPLE_COUNT}\n`); - const res = await measureMemory(); - results.push(res); - } - - const result: MemoryReportRaw = { - timestamp: new Date().toISOString(), - sampleCount: SAMPLE_COUNT, - measurement: { - startupTimeoutMs: STARTUP_TIMEOUT, - memorySettleTimeMs: MEMORY_SETTLE_TIME, - ipcTimeoutMs: IPC_TIMEOUT, - requestCount: REQUEST_COUNT, - heapSnapshot: { - enabled: HEAP_SNAPSHOT, - timeoutMs: HEAP_SNAPSHOT_TIMEOUT, - breakdownTopN: HEAP_SNAPSHOT_BREAKDOWN_TOP_N, - }, - }, - samples: results, - }; - - // Output as JSON to stdout - console.log(JSON.stringify(result, null, 2)); + console.log(JSON.stringify(await measureMemory(), null, 2)); } main().catch((err) => {