wip

.github/workflows/get-backend-memory.yml

@@ -10,6 +10,7 @@ on:
       - packages/backend/**
       - packages/misskey-js/**
       - .github/workflows/get-backend-memory.yml
+      - .github/workflows/report-backend-memory.yml
 
 jobs:
   get-memory-usage:
@@ -17,15 +18,6 @@ jobs:
     permissions:
       contents: read
 
-    strategy:
-      matrix:
-        memory-json-name: [memory-base.json, memory-head.json]
-        include:
-          - memory-json-name: memory-base.json
-            ref: ${{ github.base_ref }}
-          - memory-json-name: memory-head.json
-            ref: refs/pull/${{ github.event.number }}/merge
-
     services:
       postgres:
         image: postgres:18
@@ -40,37 +32,113 @@ jobs:
           - 56312:6379
 
     steps:
-    - uses: actions/checkout@v6.0.2
+    - name: Checkout base
+      uses: actions/checkout@v6.0.2
       with:
-        ref: ${{ matrix.ref }}
+        ref: ${{ github.base_ref }}
+        path: base
+        submodules: true
+    - name: Checkout head
+      uses: actions/checkout@v6.0.2
+      with:
+        ref: refs/pull/${{ github.event.number }}/merge
+        path: head
         submodules: true
     - name: Setup pnpm
       uses: pnpm/action-setup@v6.0.3
+      with:
+        package_json_file: head/package.json
     - name: Use Node.js
       uses: actions/setup-node@v6.4.0
       with:
-        node-version-file: '.node-version'
+        node-version-file: 'head/.node-version'
         cache: 'pnpm'
-    - run: pnpm i --frozen-lockfile
-    - name: Check pnpm-lock.yaml
+        cache-dependency-path: |
+          base/pnpm-lock.yaml
+          head/pnpm-lock.yaml
+    - name: Install base dependencies
+      working-directory: base
+      run: pnpm i --frozen-lockfile
+    - name: Check base pnpm-lock.yaml
+      working-directory: base
       run: git diff --exit-code pnpm-lock.yaml
-    - name: Copy Configure
-      run: cp .github/misskey/test.yml .config/default.yml
-    - name: Compile Configure
-      run: pnpm compile-config
-    - name: Build
-      run: pnpm build
-    - name: Run migrations
-      run: pnpm --filter backend migrate
-    - name: Measure memory usage
+    - name: Configure base
+      working-directory: base
       run: |
-        # Start the server and measure memory usage
-        node packages/backend/scripts/measure-memory.mjs > ${{ matrix.memory-json-name }}
+        cp .github/misskey/test.yml .config/default.yml
+        pnpm compile-config
+    - name: Build base
+      working-directory: base
+      run: pnpm build
+    - name: Install head dependencies
+      working-directory: head
+      run: pnpm i --frozen-lockfile
+    - name: Check head pnpm-lock.yaml
+      working-directory: head
+      run: git diff --exit-code pnpm-lock.yaml
+    - name: Configure head
+      working-directory: head
+      run: |
+        cp .github/misskey/test.yml .config/default.yml
+        pnpm compile-config
+    - name: Build head
+      working-directory: head
+      run: pnpm build
+    - name: Measure base memory usage
+      working-directory: base
+      run: |
+        node --input-type=module <<'EOF'
+        import pg from 'pg';
+        import Redis from 'ioredis';
+
+        const postgres = new pg.Client({
+          host: '127.0.0.1',
+          port: 54312,
+          database: 'postgres',
+          user: 'postgres',
+        });
+        await postgres.connect();
+        await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
+        await postgres.query('CREATE DATABASE "test-misskey"');
+        await postgres.end();
+
+        const redis = new Redis({ host: '127.0.0.1', port: 56312 });
+        await redis.flushall();
+        redis.disconnect();
+        EOF
+        pnpm --filter backend migrate
+        node packages/backend/scripts/measure-memory.mjs > ../memory-base.json
+    - name: Measure head memory usage
+      working-directory: head
+      run: |
+        node --input-type=module <<'EOF'
+        import pg from 'pg';
+        import Redis from 'ioredis';
+
+        const postgres = new pg.Client({
+          host: '127.0.0.1',
+          port: 54312,
+          database: 'postgres',
+          user: 'postgres',
+        });
+        await postgres.connect();
+        await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
+        await postgres.query('CREATE DATABASE "test-misskey"');
+        await postgres.end();
+
+        const redis = new Redis({ host: '127.0.0.1', port: 56312 });
+        await redis.flushall();
+        redis.disconnect();
+        EOF
+        pnpm --filter backend migrate
+        node packages/backend/scripts/measure-memory.mjs > ../memory-head.json
     - name: Upload Artifact
       uses: actions/upload-artifact@v7
       with:
-        name: memory-artifact-${{ matrix.memory-json-name }}
-        path: ${{ matrix.memory-json-name }}
+        name: memory-artifact-results
+        path: |
+          memory-base.json
+          memory-head.json
 
   save-pr-number:
     runs-on: ubuntu-latest

.github/workflows/report-backend-memory.yml

@@ -56,20 +56,25 @@ jobs:
 
           variation() {
             calc() {
-              BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0")
-              HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0")
+              BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // empty")
+              HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // empty")
+
+              if [ -z "$BASE" ] || [ -z "$HEAD" ]; then
+                echo "null"
+                return
+              fi
 
               DIFF=$((HEAD - BASE))
               if [ "$BASE" -gt 0 ]; then
-                DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
+                DIFF_PERCENT=$(awk -v diff="$DIFF" -v base="$BASE" 'BEGIN { printf "%.2f", (diff * 100) / base }')
               else
-                DIFF_PERCENT=0
+                DIFF_PERCENT=0.00
               fi
 
               # Convert KB to MB for readability
-              BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
-              HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
-              DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
+              BASE_MB=$(awk -v value="$BASE" 'BEGIN { printf "%.2f", value / 1024 }')
+              HEAD_MB=$(awk -v value="$HEAD" 'BEGIN { printf "%.2f", value / 1024 }')
+              DIFF_MB=$(awk -v value="$DIFF" 'BEGIN { printf "%.2f", value / 1024 }')
 
               JSON=$(jq -c -n \
                 --argjson base "$BASE_MB" \
@@ -82,11 +87,20 @@ jobs:
             }
 
             JSON=$(jq -c -n \
+              --argjson HeapUsed "$(calc $1 HeapUsed)" \
+              --argjson HeapTotal "$(calc $1 HeapTotal)" \
+              --argjson External "$(calc $1 External)" \
+              --argjson ArrayBuffers "$(calc $1 ArrayBuffers)" \
+              --argjson Pss "$(calc $1 Pss)" \
+              --argjson Private_Dirty "$(calc $1 Private_Dirty)" \
+              --argjson Private_Clean "$(calc $1 Private_Clean)" \
+              --argjson Shared_Dirty "$(calc $1 Shared_Dirty)" \
+              --argjson Shared_Clean "$(calc $1 Shared_Clean)" \
               --argjson VmRSS "$(calc $1 VmRSS)" \
               --argjson VmHWM "$(calc $1 VmHWM)" \
               --argjson VmSize "$(calc $1 VmSize)" \
               --argjson VmData "$(calc $1 VmData)" \
-              '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
+              '{HeapUsed: $HeapUsed, HeapTotal: $HeapTotal, External: $External, ArrayBuffers: $ArrayBuffers, Pss: $Pss, Private_Dirty: $Private_Dirty, Private_Clean: $Private_Clean, Shared_Dirty: $Shared_Dirty, Shared_Clean: $Shared_Clean, VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
 
             echo "$JSON"
           }
@@ -114,6 +128,10 @@ jobs:
             echo "|--------|------:|------:|------:|------:|" >> ./output.md
 
             line() {
+              if [ "$(echo "$RES" | jq -r ".${1}.${2} == null")" = "true" ]; then
+                return
+              fi
+
               METRIC=$2
               BASE=$(echo "$RES" | jq -r ".${1}.${2}.base")
               HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head")
@@ -125,8 +143,8 @@ jobs:
                 DIFF_PERCENT="+$DIFF_PERCENT"
               fi
 
-              # highlight VmRSS
-              if [ "$2" = "VmRSS" ]; then
+              # highlight the most useful process and OS memory metrics
+              if [ "$2" = "HeapUsed" ] || [ "$2" = "Pss" ]; then
                 METRIC="**${METRIC}**"
                 BASE="**${BASE}**"
                 HEAD="**${HEAD}**"
@@ -137,6 +155,15 @@ jobs:
               echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md
             }
 
+            line $1 HeapUsed
+            line $1 HeapTotal
+            line $1 External
+            line $1 ArrayBuffers
+            line $1 Pss
+            line $1 Private_Dirty
+            line $1 Private_Clean
+            line $1 Shared_Dirty
+            line $1 Shared_Clean
             line $1 VmRSS
             line $1 VmHWM
             line $1 VmSize
@@ -156,8 +183,9 @@ jobs:
           echo >> ./output.md
 
           # Determine if this is a significant change (more than 5% increase)
-          if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then
-            echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
+          WARNING_METRIC=$(echo "$RES" | jq -r 'if .afterGc.Pss != null then "Pss" elif .afterGc.VmRSS != null then "VmRSS" else empty end')
+          if [ -n "$WARNING_METRIC" ] && [ "$(echo "$RES" | jq -r ".afterGc.${WARNING_METRIC}.diff_percent | tonumber > 5")" = "true" ]; then
+            echo "⚠️ **Warning**: Memory usage (${WARNING_METRIC}) has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
             echo >> ./output.md
           fi
 

packages/backend/scripts/measure-memory.mjs

@@ -23,8 +23,9 @@ const __dirname = dirname(__filename);
 const SAMPLE_COUNT = 3; // Number of samples to measure
 const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
 const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
+const IPC_TIMEOUT = 30000; // 30 seconds timeout for IPC responses
 
-const keys = {
+const procStatusKeys = {
 	VmPeak: 0,
 	VmSize: 0,
 	VmHWM: 0,
@@ -37,30 +38,152 @@ const keys = {
 	VmSwap: 0,
 };
 
-async function getMemoryUsage(pid) {
-	const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
+const smapsRollupKeys = {
+	Pss: 0,
+	Shared_Clean: 0,
+	Shared_Dirty: 0,
+	Private_Clean: 0,
+	Private_Dirty: 0,
+	Swap: 0,
+	SwapPss: 0,
+};
 
+const runtimeKeys = {
+	HeapTotal: 0,
+	HeapUsed: 0,
+	External: 0,
+	ArrayBuffers: 0,
+};
+
+const memoryKeys = {
+	...procStatusKeys,
+	...smapsRollupKeys,
+	...runtimeKeys,
+};
+
+const phases = ['beforeGc', 'afterGc', 'afterRequest'];
+
+function parseMemoryFile(content, keys, path, required) {
 	const result = {};
 	for (const key of Object.keys(keys)) {
-		const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
+		const match = content.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
 		if (match) {
 			result[key] = parseInt(match[1], 10);
-		} else {
-			throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
+		} else if (required) {
+			throw new Error(`Failed to parse ${key} from ${path}`);
+		}
+	}
+	return result;
+}
+
+function bytesToKiB(value) {
+	return Math.round(value / 1024);
+}
+
+async function getMemoryUsage(pid) {
+	const path = `/proc/${pid}/status`;
+	const status = await fs.readFile(path, 'utf-8');
+
+	return parseMemoryFile(status, procStatusKeys, path, true);
+}
+
+async function getSmapsRollupMemoryUsage(pid) {
+	const path = `/proc/${pid}/smaps_rollup`;
+	try {
+		const smapsRollup = await fs.readFile(path, 'utf-8');
+		return parseMemoryFile(smapsRollup, smapsRollupKeys, path, false);
+	} catch (err) {
+		if (err.code === 'ENOENT' || err.code === 'EACCES') {
+			process.stderr.write(`Failed to read ${path}: ${err.message}\n`);
+			return {};
+		}
+		throw err;
+	}
+}
+
+function waitForMessage(serverProcess, predicate, description, timeout = IPC_TIMEOUT) {
+	return new Promise((resolve, reject) => {
+		const timer = globalThis.setTimeout(() => {
+			serverProcess.off('message', onMessage);
+			reject(new Error(`Timed out waiting for ${description}`));
+		}, timeout);
+
+		const onMessage = (message) => {
+			if (!predicate(message)) return;
+			globalThis.clearTimeout(timer);
+			serverProcess.off('message', onMessage);
+			resolve(message);
+		};
+
+		serverProcess.on('message', onMessage);
+	});
+}
+
+async function getRuntimeMemoryUsage(serverProcess) {
+	const response = waitForMessage(
+		serverProcess,
+		message => message != null && typeof message === 'object' && message.type === 'memory usage',
+		'memory usage',
+	);
+
+	serverProcess.send('memory usage');
+
+	const message = await response;
+	const memoryUsage = message.value;
+
+	return {
+		HeapTotal: bytesToKiB(memoryUsage.heapTotal),
+		HeapUsed: bytesToKiB(memoryUsage.heapUsed),
+		External: bytesToKiB(memoryUsage.external),
+		ArrayBuffers: bytesToKiB(memoryUsage.arrayBuffers),
+	};
+}
+
+async function getAllMemoryUsage(serverProcess) {
+	const pid = serverProcess.pid;
+	return {
+		...await getMemoryUsage(pid),
+		...await getSmapsRollupMemoryUsage(pid),
+		...await getRuntimeMemoryUsage(serverProcess),
+	};
+}
+
+function median(values) {
+	const sorted = values.toSorted((a, b) => a - b);
+	const center = Math.floor(sorted.length / 2);
+	if (sorted.length % 2 === 1) return sorted[center];
+	return Math.round((sorted[center - 1] + sorted[center]) / 2);
+}
+
+function summarizeResults(results) {
+	const summary = {};
+
+	for (const phase of phases) {
+		summary[phase] = {};
+		for (const key of Object.keys(memoryKeys)) {
+			const values = results
+				.map(result => result[phase][key])
+				.filter(value => Number.isFinite(value));
+
+			if (values.length > 0) {
+				summary[phase][key] = median(values);
+			}
 		}
 	}
 
-	return result;
+	return summary;
 }
 
 async function measureMemory() {
 	// Start the Misskey backend server using fork to enable IPC
-	const serverProcess = fork(join(__dirname, '../built/entry.js'), ['expose-gc'], {
+	const serverProcess = fork(join(__dirname, '../built/entry.js'), [], {
 		cwd: join(__dirname, '..'),
 		env: {
 			...process.env,
 			NODE_ENV: 'production',
 			MK_DISABLE_CLUSTERING: '1',
+			MK_ONLY_SERVER: '1',
+			MK_NO_DAEMONS: '1',
 		},
 		stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
 		execArgv: [...process.execArgv, '--expose-gc'],
@@ -90,15 +213,18 @@ async function measureMemory() {
 	});
 
 	async function triggerGc() {
-		const ok = new Promise((resolve) => {
-			serverProcess.once('message', (message) => {
-				if (message === 'gc ok') resolve();
-			});
-		});
+		const ok = waitForMessage(
+			serverProcess,
+			message => message === 'gc ok' || message === 'gc unavailable',
+			'GC completion',
+		);
 
 		serverProcess.send('gc');
 
-		await ok;
+		const message = await ok;
+		if (message === 'gc unavailable') {
+			throw new Error('GC is unavailable. Start the process with --expose-gc to enable this feature.');
+		}
 
 		await setTimeout(1000);
 	}
@@ -139,13 +265,11 @@ async function measureMemory() {
 	// Wait for memory to settle
 	await setTimeout(MEMORY_SETTLE_TIME);
 
-	const pid = serverProcess.pid;
-
-	const beforeGc = await getMemoryUsage(pid);
+	const beforeGc = await getAllMemoryUsage(serverProcess);
 
 	await triggerGc();
 
-	const afterGc = await getMemoryUsage(pid);
+	const afterGc = await getAllMemoryUsage(serverProcess);
 
 	// create some http requests to simulate load
 	const REQUEST_COUNT = 10;
@@ -155,7 +279,7 @@ async function measureMemory() {
 
 	await triggerGc();
 
-	const afterRequest = await getMemoryUsage(pid);
+	const afterRequest = await getAllMemoryUsage(serverProcess);
 
 	// Stop the server
 	serverProcess.kill('SIGTERM');
@@ -187,35 +311,21 @@ async function measureMemory() {
 }
 
 async function main() {
-	// 直列の方が時間的に分散されて正確そうだから直列でやる
 	const results = [];
 	for (let i = 0; i < SAMPLE_COUNT; i++) {
+		process.stderr.write(`Starting sample ${i + 1}/${SAMPLE_COUNT}\n`);
 		const res = await measureMemory();
 		results.push(res);
 	}
 
-	// Calculate averages
-	const beforeGc = structuredClone(keys);
-	const afterGc = structuredClone(keys);
-	const afterRequest = structuredClone(keys);
-	for (const res of results) {
-		for (const key of Object.keys(keys)) {
-			beforeGc[key] += res.beforeGc[key];
-			afterGc[key] += res.afterGc[key];
-			afterRequest[key] += res.afterRequest[key];
-		}
-	}
-	for (const key of Object.keys(keys)) {
-		beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT);
-		afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT);
-		afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT);
-	}
+	const summary = summarizeResults(results);
 
 	const result = {
 		timestamp: new Date().toISOString(),
-		beforeGc,
-		afterGc,
-		afterRequest,
+		sampleCount: SAMPLE_COUNT,
+		aggregation: 'median',
+		...summary,
+		samples: results,
 	};
 
 	// Output as JSON to stdout

packages/backend/src/boot/common.ts

@@ -6,6 +6,7 @@
 import { NestFactory } from '@nestjs/core';
 import { init } from 'slacc';
 import { NestLogger } from '@/NestLogger.js';
+import { envOption } from '@/env.js';
 import type { Config } from '@/config.js';
 
 let slaccInitialized = false;
@@ -31,7 +32,7 @@ export async function server() {
 	const serverService = app.get(ServerService);
 	await serverService.launch();
 
-	if (process.env.NODE_ENV !== 'test') {
+	if (process.env.NODE_ENV !== 'test' && !envOption.noDaemons) {
 		const { ChartManagementService } = await import('../core/chart/ChartManagementService.js');
 		const { QueueStatsService } = await import('../daemons/QueueStatsService.js');
 		const { ServerStatsService } = await import('../daemons/ServerStatsService.js');
@@ -54,7 +55,9 @@ export async function jobQueue() {
 	});
 
 	jobQueue.get(QueueProcessorService).start();
-	jobQueue.get(ChartManagementService).start();
+	if (!envOption.noDaemons) {
+		jobQueue.get(ChartManagementService).start();
+	}
 
 	return jobQueue;
 }

packages/backend/src/boot/entry.ts

@@ -91,10 +91,20 @@ process.on('message', msg => {
 	if (msg === 'gc') {
 		if (global.gc != null) {
 			logger.info('Manual GC triggered');
-			global.gc();
+			for (let i = 0; i < 3; i++) {
+				global.gc();
+			}
 			if (process.send != null) process.send('gc ok');
 		} else {
 			logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.');
+			if (process.send != null) process.send('gc unavailable');
+		}
+	} else if (msg === 'memory usage') {
+		if (process.send != null) {
+			process.send({
+				type: 'memory usage',
+				value: process.memoryUsage(),
+			});
 		}
 	}
 });