From c899aafeef33ac420a80f9ba01872bb9ee9cb532 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Tue, 23 Jun 2026 11:54:16 +0900 Subject: [PATCH] enhance(dev): tweak report-backend-memory --- .github/scripts/backend-memory-report.mjs | 78 +++++- .../measure-backend-memory-comparison.mjs | 224 ++++++++++++++++++ .github/workflows/get-backend-memory.yml | 61 +---- packages/backend/scripts/measure-memory.mjs | 26 +- 4 files changed, 323 insertions(+), 66 deletions(-) create mode 100644 .github/scripts/measure-backend-memory-comparison.mjs diff --git a/.github/scripts/backend-memory-report.mjs b/.github/scripts/backend-memory-report.mjs index d37ae98cc8..c06ca4cd3f 100644 --- a/.github/scripts/backend-memory-report.mjs +++ b/.github/scripts/backend-memory-report.mjs @@ -34,6 +34,11 @@ function formatMemory(valueKiB) { return `${formatNumber(valueKiB / 1024)} MB`; } +function formatMemoryWithSpread(valueKiB, spreadKiB) { + if (spreadKiB == null) return formatMemory(valueKiB); + return `${formatMemory(valueKiB)} ± ${formatMemory(spreadKiB)}`; +} + function formatPercent(value) { return `${formatNumber(value)}%`; } @@ -48,7 +53,7 @@ function formatMathText(text) { function formatColoredDiff(text, diff) { const color = diff > 0 ? 'orange' : 'green'; - return `$\\color{${color}}{\\text{${formatMathText(text)}}}$`; + return `$\\color{${color}}{\\text{${formatMathText(text).replaceAll('\\%', '\\\\%')}}}$`; } function formatDiff(baseKiB, headKiB) { @@ -73,6 +78,29 @@ function getMemoryValue(report, phase, metric) { return Number.isFinite(value) ? value : null; } +function median(values) { + const sorted = values.toSorted((a, b) => a - b); + const center = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[center]; + return Math.round((sorted[center - 1] + sorted[center]) / 2); +} + +function getSampleValues(report, phase, metric) { + if (!Array.isArray(report?.samples)) return []; + + return report.samples + .map(sample => getMemoryValue(sample, phase, metric)) + .filter(value => Number.isFinite(value)); +} + +function getSampleSpread(report, phase, metric) { + const values = getSampleValues(report, phase, metric); + if (values.length < 2) return null; + + const center = median(values); + return median(values.map(value => Math.abs(value - center))); +} + function renderTable(base, head, phase) { const lines = [ '| Metric | Base | Head | Δ | Δ (%) |', @@ -84,7 +112,10 @@ function renderTable(base, head, phase) { const headValue = getMemoryValue(head, phase, metric); if (baseValue == null || headValue == null) continue; - lines.push(`| ${metric} | ${formatMemory(baseValue)} | ${formatMemory(headValue)} | ${formatDiff(baseValue, headValue)} | ${formatDiffPercent(baseValue, headValue)} |`); + const baseSpread = getSampleSpread(base, phase, metric); + const headSpread = getSampleSpread(head, phase, metric); + + lines.push(`| ${metric} | ${formatMemoryWithSpread(baseValue, baseSpread)} | ${formatMemoryWithSpread(headValue, headSpread)} | ${formatDiff(baseValue, headValue)} | ${formatDiffPercent(baseValue, headValue)} |`); } return lines.join('\n'); @@ -99,7 +130,7 @@ function getDiffPercent(base, head, phase, metric) { } function getWarningMetric(base, head) { - for (const metric of ['Pss', 'VmRSS']) { + for (const metric of ['Pss', 'Private_Dirty', 'VmRSS']) { if (getMemoryValue(base, 'afterGc', metric) != null && getMemoryValue(head, 'afterGc', metric) != null) { return metric; } @@ -107,6 +138,22 @@ function getWarningMetric(base, head) { return null; } +function isBeyondSampleNoise(base, head, phase, metric) { + const baseValue = getMemoryValue(base, phase, metric); + const headValue = getMemoryValue(head, phase, metric); + if (baseValue == null || headValue == null) return false; + + const diff = headValue - baseValue; + if (diff <= 0) return false; + + const baseSpread = getSampleSpread(base, phase, metric); + const headSpread = getSampleSpread(head, phase, metric); + if (baseSpread == null || headSpread == null) return true; + + const combinedSpread = Math.hypot(baseSpread, headSpread); + return diff > combinedSpread * 3; +} + function workflowFooter() { const repository = process.env.GITHUB_REPOSITORY; const runId = process.env.GITHUB_RUN_ID; @@ -117,6 +164,21 @@ function workflowFooter() { return `[See workflow logs for details](https://github.com/${repository}/actions/runs/${runId})`; } +function measurementSummary(base, head) { + const baseCount = base?.sampleCount; + const headCount = head?.sampleCount; + const strategy = base?.comparison?.strategy; + if (baseCount == null || headCount == null) return null; + + if (strategy === 'interleaved-pairs') { + const rounds = base?.comparison?.rounds ?? baseCount; + const warmupRounds = base?.comparison?.warmupRounds ?? 0; + return `_Measured as ${rounds} interleaved base/head pairs after ${warmupRounds} warmup pair(s). Values are medians; ± is median absolute deviation._`; + } + + return `_Sample count: base ${baseCount}, head ${headCount}. Values are medians; ± is median absolute deviation._`; +} + const base = JSON.parse(await readFile(baseFile, 'utf8')); const head = JSON.parse(await readFile(headFile, 'utf8')); const lines = [ @@ -124,6 +186,12 @@ const lines = [ '', ]; +const summary = measurementSummary(base, head); +if (summary != null) { + lines.push(summary); + lines.push(''); +} + for (const phase of phases) { lines.push(`### ${phase.title}`); lines.push(renderTable(base, head, phase.key)); @@ -132,8 +200,8 @@ for (const phase of phases) { const warningMetric = getWarningMetric(base, head); const warningDiffPercent = warningMetric == null ? null : getDiffPercent(base, head, 'afterGc', warningMetric); -if (warningMetric != null && warningDiffPercent != null && warningDiffPercent > 5) { - lines.push(`⚠️ **Warning**: Memory usage (${warningMetric}) has increased by more than 5%. Please verify this is not an unintended change.`); +if (warningMetric != null && warningDiffPercent != null && warningDiffPercent > 5 && isBeyondSampleNoise(base, head, 'afterGc', warningMetric)) { + lines.push(`⚠️ **Warning**: Memory usage (${warningMetric}) has increased by more than 5% and exceeds the observed sample noise. Please verify this is not an unintended change.`); lines.push(''); } diff --git a/.github/scripts/measure-backend-memory-comparison.mjs b/.github/scripts/measure-backend-memory-comparison.mjs new file mode 100644 index 0000000000..da543570ac --- /dev/null +++ b/.github/scripts/measure-backend-memory-comparison.mjs @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +const phases = ['beforeGc', 'afterGc', 'afterRequest']; + +const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2); + +if (baseDirArg == null || headDirArg == null || baseOutputArg == null || headOutputArg == null) { + console.error('Usage: node .github/scripts/measure-backend-memory-comparison.mjs '); + process.exit(1); +} + +function readIntegerEnv(name, defaultValue, min) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`); + + const value = Number(rawValue); + if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`); + return value; +} + +function commandName(command) { + if (process.platform !== 'win32') return command; + if (command === 'pnpm') return 'pnpm.cmd'; + return command; +} + +function run(command, args, options = {}) { + return new Promise((resolvePromise, reject) => { + const child = spawn(commandName(command), args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', data => { + stdout += data; + if (options.logStdout) process.stderr.write(data); + }); + + child.stderr.on('data', data => { + stderr += data; + process.stderr.write(data); + }); + + child.on('error', reject); + + child.on('close', code => { + if (code === 0) { + resolvePromise(stdout); + } else { + reject(new Error(`${command} ${args.join(' ')} failed with exit code ${code}\n${stderr}`)); + } + }); + }); +} + +async function resetState(repoDir) { + const require = createRequire(join(repoDir, 'packages/backend/package.json')); + const pg = require('pg'); + const Redis = require('ioredis'); + + const postgres = new pg.Client({ + host: '127.0.0.1', + port: 54312, + database: 'postgres', + user: 'postgres', + }); + + await postgres.connect(); + try { + await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); + await postgres.query('CREATE DATABASE "test-misskey"'); + } finally { + await postgres.end(); + } + + const redis = new Redis({ host: '127.0.0.1', port: 56312 }); + try { + await redis.flushall(); + } finally { + redis.disconnect(); + } +} + +function median(values) { + const sorted = values.toSorted((a, b) => a - b); + const center = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[center]; + return Math.round((sorted[center - 1] + sorted[center]) / 2); +} + +function summarizeSamples(samples) { + const summary = {}; + + for (const phase of phases) { + summary[phase] = {}; + + const metricKeys = new Set(); + for (const sample of samples) { + for (const key of Object.keys(sample[phase] ?? {})) { + metricKeys.add(key); + } + } + + for (const key of metricKeys) { + const values = samples + .map(sample => sample[phase]?.[key]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) summary[phase][key] = median(values); + } + } + + return summary; +} + +async function measureRepo(label, repoDir, round, orderIndex) { + process.stderr.write(`[${label}] Resetting database and Redis\n`); + await resetState(repoDir); + + process.stderr.write(`[${label}] Running migrations\n`); + await run('pnpm', ['--filter', 'backend', 'migrate'], { + cwd: repoDir, + env: process.env, + logStdout: true, + }); + + process.stderr.write(`[${label}] Measuring memory\n`); + const stdout = await run('node', ['packages/backend/scripts/measure-memory.mjs'], { + cwd: repoDir, + env: { + ...process.env, + MK_MEMORY_SAMPLE_COUNT: '1', + }, + }); + + const report = JSON.parse(stdout); + const sample = report.samples?.[0] ?? { + timestamp: report.timestamp, + beforeGc: report.beforeGc, + afterGc: report.afterGc, + afterRequest: report.afterRequest, + }; + + return { + ...sample, + label, + round, + orderIndex, + }; +} + +async function main() { + const baseDir = resolve(baseDirArg); + const headDir = resolve(headDirArg); + const baseOutput = resolve(baseOutputArg); + const headOutput = resolve(headOutputArg); + const rounds = readIntegerEnv('MK_MEMORY_COMPARE_ROUNDS', 5, 1); + const warmupRounds = readIntegerEnv('MK_MEMORY_COMPARE_WARMUP_ROUNDS', 1, 0); + const startedAt = new Date().toISOString(); + + const repos = { + base: { + dir: baseDir, + samples: [], + }, + head: { + dir: headDir, + samples: [], + }, + }; + + for (let round = 1; round <= warmupRounds; round++) { + process.stderr.write(`Starting warmup round ${round}/${warmupRounds}\n`); + for (const label of ['base', 'head']) { + await measureRepo(label, repos[label].dir, -round, 0); + } + } + + for (let round = 1; round <= rounds; round++) { + const order = round % 2 === 1 ? ['base', 'head'] : ['head', 'base']; + process.stderr.write(`Starting measurement round ${round}/${rounds}: ${order.join(' -> ')}\n`); + + for (const [orderIndex, label] of order.entries()) { + const sample = await measureRepo(label, repos[label].dir, round, orderIndex); + repos[label].samples.push(sample); + } + } + + for (const label of ['base', 'head']) { + const report = { + timestamp: new Date().toISOString(), + sampleCount: repos[label].samples.length, + aggregation: 'median', + comparison: { + strategy: 'interleaved-pairs', + rounds, + warmupRounds, + startedAt, + }, + ...summarizeSamples(repos[label].samples), + samples: repos[label].samples, + }; + + await writeFile(label === 'base' ? baseOutput : headOutput, `${JSON.stringify(report, null, 2)}\n`); + } +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index b210fbfc5e..73a65635c0 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -85,62 +85,11 @@ jobs: - name: Build head working-directory: head run: pnpm build - - name: Measure base memory usage - working-directory: base - run: | - node --input-type=module <<'EOF' - import { createRequire } from 'node:module'; - import { join } from 'node:path'; - - const require = createRequire(join(process.cwd(), 'packages/backend/package.json')); - const pg = require('pg'); - const Redis = require('ioredis'); - - const postgres = new pg.Client({ - host: '127.0.0.1', - port: 54312, - database: 'postgres', - user: 'postgres', - }); - await postgres.connect(); - await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); - await postgres.query('CREATE DATABASE "test-misskey"'); - await postgres.end(); - - const redis = new Redis({ host: '127.0.0.1', port: 56312 }); - await redis.flushall(); - redis.disconnect(); - EOF - pnpm --filter backend migrate - node packages/backend/scripts/measure-memory.mjs > ../memory-base.json - - name: Measure head memory usage - working-directory: head - run: | - node --input-type=module <<'EOF' - import { createRequire } from 'node:module'; - import { join } from 'node:path'; - - const require = createRequire(join(process.cwd(), 'packages/backend/package.json')); - const pg = require('pg'); - const Redis = require('ioredis'); - - const postgres = new pg.Client({ - host: '127.0.0.1', - port: 54312, - database: 'postgres', - user: 'postgres', - }); - await postgres.connect(); - await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); - await postgres.query('CREATE DATABASE "test-misskey"'); - await postgres.end(); - - const redis = new Redis({ host: '127.0.0.1', port: 56312 }); - await redis.flushall(); - redis.disconnect(); - EOF - pnpm --filter backend migrate - node packages/backend/scripts/measure-memory.mjs > ../memory-head.json + - name: Measure backend memory usage + env: + MK_MEMORY_COMPARE_ROUNDS: 5 + MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1 + run: node head/.github/scripts/measure-backend-memory-comparison.mjs base head memory-base.json memory-head.json - name: Upload Artifact uses: actions/upload-artifact@v7 with: diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index 3a4d145894..beefcd7ed0 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -20,10 +20,21 @@ import * as fs from 'node:fs/promises'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); -const SAMPLE_COUNT = 3; // Number of samples to measure -const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup -const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle -const IPC_TIMEOUT = 30000; // 30 seconds timeout for IPC responses +function readIntegerEnv(name, defaultValue, min) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`); + + const value = Number(rawValue); + if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`); + return value; +} + +const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure +const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup +const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle +const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses +const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0); const procStatusKeys = { VmPeak: 0, @@ -272,7 +283,6 @@ async function measureMemory() { const afterGc = await getAllMemoryUsage(serverProcess); // create some http requests to simulate load - const REQUEST_COUNT = 10; await Promise.all( Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), ); @@ -324,6 +334,12 @@ async function main() { timestamp: new Date().toISOString(), sampleCount: SAMPLE_COUNT, aggregation: 'median', + measurement: { + startupTimeoutMs: STARTUP_TIMEOUT, + memorySettleTimeMs: MEMORY_SETTLE_TIME, + ipcTimeoutMs: IPC_TIMEOUT, + requestCount: REQUEST_COUNT, + }, ...summary, samples: results, };