diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index aa4bf5e3a4..0e53ab927d 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -10,6 +10,7 @@ on: - packages/backend/** - packages/misskey-js/** - .github/workflows/get-backend-memory.yml + - .github/workflows/report-backend-memory.yml jobs: get-memory-usage: @@ -17,15 +18,6 @@ jobs: permissions: contents: read - strategy: - matrix: - memory-json-name: [memory-base.json, memory-head.json] - include: - - memory-json-name: memory-base.json - ref: ${{ github.base_ref }} - - memory-json-name: memory-head.json - ref: refs/pull/${{ github.event.number }}/merge - services: postgres: image: postgres:18 @@ -40,37 +32,121 @@ jobs: - 56312:6379 steps: - - uses: actions/checkout@v6.0.2 + - name: Checkout base + uses: actions/checkout@v6.0.2 with: - ref: ${{ matrix.ref }} + ref: ${{ github.base_ref }} + path: base + submodules: true + - name: Checkout head + uses: actions/checkout@v6.0.2 + with: + ref: refs/pull/${{ github.event.number }}/merge + path: head submodules: true - name: Setup pnpm uses: pnpm/action-setup@v6.0.3 + with: + package_json_file: head/package.json - name: Use Node.js uses: actions/setup-node@v6.4.0 with: - node-version-file: '.node-version' + node-version-file: 'head/.node-version' cache: 'pnpm' - - run: pnpm i --frozen-lockfile - - name: Check pnpm-lock.yaml + cache-dependency-path: | + base/pnpm-lock.yaml + head/pnpm-lock.yaml + - name: Install base dependencies + working-directory: base + run: pnpm i --frozen-lockfile + - name: Check base pnpm-lock.yaml + working-directory: base run: git diff --exit-code pnpm-lock.yaml - - name: Copy Configure - run: cp .github/misskey/test.yml .config/default.yml - - name: Compile Configure - run: pnpm compile-config - - name: Build - run: pnpm build - - name: Run migrations - run: pnpm --filter backend migrate - - name: Measure memory usage + - name: Configure base + working-directory: base run: | - # Start the server and measure memory usage - node packages/backend/scripts/measure-memory.mjs > ${{ matrix.memory-json-name }} + cp .github/misskey/test.yml .config/default.yml + pnpm compile-config + - name: Build base + working-directory: base + run: pnpm build + - name: Install head dependencies + working-directory: head + run: pnpm i --frozen-lockfile + - name: Check head pnpm-lock.yaml + working-directory: head + run: git diff --exit-code pnpm-lock.yaml + - name: Configure head + working-directory: head + run: | + cp .github/misskey/test.yml .config/default.yml + pnpm compile-config + - name: Build head + working-directory: head + run: pnpm build + - name: Measure base memory usage + working-directory: base + run: | + node --input-type=module <<'EOF' + import { createRequire } from 'node:module'; + import { join } from 'node:path'; + + const require = createRequire(join(process.cwd(), 'packages/backend/package.json')); + const pg = require('pg'); + const Redis = require('ioredis'); + + const postgres = new pg.Client({ + host: '127.0.0.1', + port: 54312, + database: 'postgres', + user: 'postgres', + }); + await postgres.connect(); + await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); + await postgres.query('CREATE DATABASE "test-misskey"'); + await postgres.end(); + + const redis = new Redis({ host: '127.0.0.1', port: 56312 }); + await redis.flushall(); + redis.disconnect(); + EOF + pnpm --filter backend migrate + node packages/backend/scripts/measure-memory.mjs > ../memory-base.json + - name: Measure head memory usage + working-directory: head + run: | + node --input-type=module <<'EOF' + import { createRequire } from 'node:module'; + import { join } from 'node:path'; + + const require = createRequire(join(process.cwd(), 'packages/backend/package.json')); + const pg = require('pg'); + const Redis = require('ioredis'); + + const postgres = new pg.Client({ + host: '127.0.0.1', + port: 54312, + database: 'postgres', + user: 'postgres', + }); + await postgres.connect(); + await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); + await postgres.query('CREATE DATABASE "test-misskey"'); + await postgres.end(); + + const redis = new Redis({ host: '127.0.0.1', port: 56312 }); + await redis.flushall(); + redis.disconnect(); + EOF + pnpm --filter backend migrate + node packages/backend/scripts/measure-memory.mjs > ../memory-head.json - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: memory-artifact-${{ matrix.memory-json-name }} - path: ${{ matrix.memory-json-name }} + name: memory-artifact-results + path: | + memory-base.json + memory-head.json save-pr-number: runs-on: ubuntu-latest diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index 07850bf2e4..b310120c44 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -56,20 +56,25 @@ jobs: variation() { calc() { - BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0") - HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0") + BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // empty") + HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // empty") + + if [ -z "$BASE" ] || [ -z "$HEAD" ]; then + echo "null" + return + fi DIFF=$((HEAD - BASE)) if [ "$BASE" -gt 0 ]; then - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) + DIFF_PERCENT=$(awk -v diff="$DIFF" -v base="$BASE" 'BEGIN { printf "%.2f", (diff * 100) / base }') else - DIFF_PERCENT=0 + DIFF_PERCENT=0.00 fi # Convert KB to MB for readability - BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) - HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) + BASE_MB=$(awk -v value="$BASE" 'BEGIN { printf "%.2f", value / 1024 }') + HEAD_MB=$(awk -v value="$HEAD" 'BEGIN { printf "%.2f", value / 1024 }') + DIFF_MB=$(awk -v value="$DIFF" 'BEGIN { printf "%.2f", value / 1024 }') JSON=$(jq -c -n \ --argjson base "$BASE_MB" \ @@ -82,11 +87,20 @@ jobs: } JSON=$(jq -c -n \ + --argjson HeapUsed "$(calc $1 HeapUsed)" \ + --argjson HeapTotal "$(calc $1 HeapTotal)" \ + --argjson External "$(calc $1 External)" \ + --argjson ArrayBuffers "$(calc $1 ArrayBuffers)" \ + --argjson Pss "$(calc $1 Pss)" \ + --argjson Private_Dirty "$(calc $1 Private_Dirty)" \ + --argjson Private_Clean "$(calc $1 Private_Clean)" \ + --argjson Shared_Dirty "$(calc $1 Shared_Dirty)" \ + --argjson Shared_Clean "$(calc $1 Shared_Clean)" \ --argjson VmRSS "$(calc $1 VmRSS)" \ --argjson VmHWM "$(calc $1 VmHWM)" \ --argjson VmSize "$(calc $1 VmSize)" \ --argjson VmData "$(calc $1 VmData)" \ - '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}') + '{HeapUsed: $HeapUsed, HeapTotal: $HeapTotal, External: $External, ArrayBuffers: $ArrayBuffers, Pss: $Pss, Private_Dirty: $Private_Dirty, Private_Clean: $Private_Clean, Shared_Dirty: $Shared_Dirty, Shared_Clean: $Shared_Clean, VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}') echo "$JSON" } @@ -114,6 +128,10 @@ jobs: echo "|--------|------:|------:|------:|------:|" >> ./output.md line() { + if [ "$(echo "$RES" | jq -r ".${1}.${2} == null")" = "true" ]; then + return + fi + METRIC=$2 BASE=$(echo "$RES" | jq -r ".${1}.${2}.base") HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head") @@ -125,8 +143,8 @@ jobs: DIFF_PERCENT="+$DIFF_PERCENT" fi - # highlight VmRSS - if [ "$2" = "VmRSS" ]; then + # highlight the most useful process and OS memory metrics + if [ "$2" = "HeapUsed" ] || [ "$2" = "Pss" ]; then METRIC="**${METRIC}**" BASE="**${BASE}**" HEAD="**${HEAD}**" @@ -137,10 +155,11 @@ jobs: echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md } - line $1 VmRSS - line $1 VmHWM - line $1 VmSize - line $1 VmData + line $1 HeapUsed + line $1 HeapTotal + line $1 External + line $1 Pss + line $1 Private_Dirty } echo "### Before GC" >> ./output.md @@ -156,8 +175,9 @@ jobs: echo >> ./output.md # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then - echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md + WARNING_METRIC=$(echo "$RES" | jq -r 'if .afterGc.Pss != null then "Pss" elif .afterGc.VmRSS != null then "VmRSS" else empty end') + if [ -n "$WARNING_METRIC" ] && [ "$(echo "$RES" | jq -r ".afterGc.${WARNING_METRIC}.diff_percent | tonumber > 5")" = "true" ]; then + echo "⚠️ **Warning**: Memory usage (${WARNING_METRIC}) has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md echo >> ./output.md fi diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs index 7c058a131d..3a4d145894 100644 --- a/packages/backend/scripts/measure-memory.mjs +++ b/packages/backend/scripts/measure-memory.mjs @@ -23,8 +23,9 @@ const __dirname = dirname(__filename); const SAMPLE_COUNT = 3; // Number of samples to measure const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle +const IPC_TIMEOUT = 30000; // 30 seconds timeout for IPC responses -const keys = { +const procStatusKeys = { VmPeak: 0, VmSize: 0, VmHWM: 0, @@ -37,30 +38,152 @@ const keys = { VmSwap: 0, }; -async function getMemoryUsage(pid) { - const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); +const smapsRollupKeys = { + Pss: 0, + Shared_Clean: 0, + Shared_Dirty: 0, + Private_Clean: 0, + Private_Dirty: 0, + Swap: 0, + SwapPss: 0, +}; +const runtimeKeys = { + HeapTotal: 0, + HeapUsed: 0, + External: 0, + ArrayBuffers: 0, +}; + +const memoryKeys = { + ...procStatusKeys, + ...smapsRollupKeys, + ...runtimeKeys, +}; + +const phases = ['beforeGc', 'afterGc', 'afterRequest']; + +function parseMemoryFile(content, keys, path, required) { const result = {}; for (const key of Object.keys(keys)) { - const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); + const match = content.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); if (match) { result[key] = parseInt(match[1], 10); - } else { - throw new Error(`Failed to parse ${key} from /proc/${pid}/status`); + } else if (required) { + throw new Error(`Failed to parse ${key} from ${path}`); + } + } + return result; +} + +function bytesToKiB(value) { + return Math.round(value / 1024); +} + +async function getMemoryUsage(pid) { + const path = `/proc/${pid}/status`; + const status = await fs.readFile(path, 'utf-8'); + + return parseMemoryFile(status, procStatusKeys, path, true); +} + +async function getSmapsRollupMemoryUsage(pid) { + const path = `/proc/${pid}/smaps_rollup`; + try { + const smapsRollup = await fs.readFile(path, 'utf-8'); + return parseMemoryFile(smapsRollup, smapsRollupKeys, path, false); + } catch (err) { + if (err.code === 'ENOENT' || err.code === 'EACCES') { + process.stderr.write(`Failed to read ${path}: ${err.message}\n`); + return {}; + } + throw err; + } +} + +function waitForMessage(serverProcess, predicate, description, timeout = IPC_TIMEOUT) { + return new Promise((resolve, reject) => { + const timer = globalThis.setTimeout(() => { + serverProcess.off('message', onMessage); + reject(new Error(`Timed out waiting for ${description}`)); + }, timeout); + + const onMessage = (message) => { + if (!predicate(message)) return; + globalThis.clearTimeout(timer); + serverProcess.off('message', onMessage); + resolve(message); + }; + + serverProcess.on('message', onMessage); + }); +} + +async function getRuntimeMemoryUsage(serverProcess) { + const response = waitForMessage( + serverProcess, + message => message != null && typeof message === 'object' && message.type === 'memory usage', + 'memory usage', + ); + + serverProcess.send('memory usage'); + + const message = await response; + const memoryUsage = message.value; + + return { + HeapTotal: bytesToKiB(memoryUsage.heapTotal), + HeapUsed: bytesToKiB(memoryUsage.heapUsed), + External: bytesToKiB(memoryUsage.external), + ArrayBuffers: bytesToKiB(memoryUsage.arrayBuffers), + }; +} + +async function getAllMemoryUsage(serverProcess) { + const pid = serverProcess.pid; + return { + ...await getMemoryUsage(pid), + ...await getSmapsRollupMemoryUsage(pid), + ...await getRuntimeMemoryUsage(serverProcess), + }; +} + +function median(values) { + const sorted = values.toSorted((a, b) => a - b); + const center = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[center]; + return Math.round((sorted[center - 1] + sorted[center]) / 2); +} + +function summarizeResults(results) { + const summary = {}; + + for (const phase of phases) { + summary[phase] = {}; + for (const key of Object.keys(memoryKeys)) { + const values = results + .map(result => result[phase][key]) + .filter(value => Number.isFinite(value)); + + if (values.length > 0) { + summary[phase][key] = median(values); + } } } - return result; + return summary; } async function measureMemory() { // Start the Misskey backend server using fork to enable IPC - const serverProcess = fork(join(__dirname, '../built/entry.js'), ['expose-gc'], { + const serverProcess = fork(join(__dirname, '../built/entry.js'), [], { cwd: join(__dirname, '..'), env: { ...process.env, NODE_ENV: 'production', MK_DISABLE_CLUSTERING: '1', + MK_ONLY_SERVER: '1', + MK_NO_DAEMONS: '1', }, stdio: ['pipe', 'pipe', 'pipe', 'ipc'], execArgv: [...process.execArgv, '--expose-gc'], @@ -90,15 +213,18 @@ async function measureMemory() { }); async function triggerGc() { - const ok = new Promise((resolve) => { - serverProcess.once('message', (message) => { - if (message === 'gc ok') resolve(); - }); - }); + const ok = waitForMessage( + serverProcess, + message => message === 'gc ok' || message === 'gc unavailable', + 'GC completion', + ); serverProcess.send('gc'); - await ok; + const message = await ok; + if (message === 'gc unavailable') { + throw new Error('GC is unavailable. Start the process with --expose-gc to enable this feature.'); + } await setTimeout(1000); } @@ -139,13 +265,11 @@ async function measureMemory() { // Wait for memory to settle await setTimeout(MEMORY_SETTLE_TIME); - const pid = serverProcess.pid; - - const beforeGc = await getMemoryUsage(pid); + const beforeGc = await getAllMemoryUsage(serverProcess); await triggerGc(); - const afterGc = await getMemoryUsage(pid); + const afterGc = await getAllMemoryUsage(serverProcess); // create some http requests to simulate load const REQUEST_COUNT = 10; @@ -155,7 +279,7 @@ async function measureMemory() { await triggerGc(); - const afterRequest = await getMemoryUsage(pid); + const afterRequest = await getAllMemoryUsage(serverProcess); // Stop the server serverProcess.kill('SIGTERM'); @@ -187,35 +311,21 @@ async function measureMemory() { } async function main() { - // 直列の方が時間的に分散されて正確そうだから直列でやる const results = []; for (let i = 0; i < SAMPLE_COUNT; i++) { + process.stderr.write(`Starting sample ${i + 1}/${SAMPLE_COUNT}\n`); const res = await measureMemory(); results.push(res); } - // Calculate averages - const beforeGc = structuredClone(keys); - const afterGc = structuredClone(keys); - const afterRequest = structuredClone(keys); - for (const res of results) { - for (const key of Object.keys(keys)) { - beforeGc[key] += res.beforeGc[key]; - afterGc[key] += res.afterGc[key]; - afterRequest[key] += res.afterRequest[key]; - } - } - for (const key of Object.keys(keys)) { - beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT); - afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT); - afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT); - } + const summary = summarizeResults(results); const result = { timestamp: new Date().toISOString(), - beforeGc, - afterGc, - afterRequest, + sampleCount: SAMPLE_COUNT, + aggregation: 'median', + ...summary, + samples: results, }; // Output as JSON to stdout diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index b27e8c1d6b..8c1862f13e 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -6,6 +6,7 @@ import { NestFactory } from '@nestjs/core'; import { init } from 'slacc'; import { NestLogger } from '@/NestLogger.js'; +import { envOption } from '@/env.js'; import type { Config } from '@/config.js'; let slaccInitialized = false; @@ -31,7 +32,7 @@ export async function server() { const serverService = app.get(ServerService); await serverService.launch(); - if (process.env.NODE_ENV !== 'test') { + if (process.env.NODE_ENV !== 'test' && !envOption.noDaemons) { const { ChartManagementService } = await import('../core/chart/ChartManagementService.js'); const { QueueStatsService } = await import('../daemons/QueueStatsService.js'); const { ServerStatsService } = await import('../daemons/ServerStatsService.js'); @@ -54,7 +55,9 @@ export async function jobQueue() { }); jobQueue.get(QueueProcessorService).start(); - jobQueue.get(ChartManagementService).start(); + if (!envOption.noDaemons) { + jobQueue.get(ChartManagementService).start(); + } return jobQueue; } diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 6e37bf9e1c..b5f258e9ee 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -91,10 +91,20 @@ process.on('message', msg => { if (msg === 'gc') { if (global.gc != null) { logger.info('Manual GC triggered'); - global.gc(); + for (let i = 0; i < 3; i++) { + global.gc(); + } if (process.send != null) process.send('gc ok'); } else { logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.'); + if (process.send != null) process.send('gc unavailable'); + } + } else if (msg === 'memory usage') { + if (process.send != null) { + process.send({ + type: 'memory usage', + value: process.memoryUsage(), + }); } } });