wip

Create frontend-js-size.yml

.github/workflows/frontend-js-size-comment.yml

@@ -0,0 +1,86 @@
+name: Frontend JS size comment
+
+on:
+  workflow_run:
+    workflows:
+      - Frontend JS size
+    types:
+      - completed
+
+permissions:
+  actions: read
+  contents: read
+  issues: write
+  pull-requests: read
+
+jobs:
+  comment:
+    name: Comment frontend JS size
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download size report
+        uses: actions/download-artifact@v8
+        with:
+          name: frontend-js-size-report
+          path: frontend-js-size-report
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+
+      - name: Comment on pull request
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require('node:fs');
+
+            const marker = '<!-- misskey-frontend-js-size -->';
+            const body = fs.readFileSync('frontend-js-size-report/frontend-js-size-report.md', 'utf8');
+            if (!body.includes(marker)) {
+              core.setFailed('The frontend JS size report is missing the expected marker.');
+              return;
+            }
+
+            const { owner, repo } = context.repo;
+            const workflowRun = context.payload.workflow_run;
+            let issue_number = workflowRun.pull_requests?.[0]?.number;
+
+            if (issue_number == null) {
+              const { data: pullRequests } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
+                owner,
+                repo,
+                commit_sha: workflowRun.head_sha,
+              });
+              issue_number = pullRequests.find((pr) => pr.head.sha === workflowRun.head_sha)?.number
+                ?? pullRequests[0]?.number;
+            }
+
+            if (issue_number == null) {
+              core.info(`No pull request found for workflow run ${workflowRun.id}.`);
+              return;
+            }
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number,
+              per_page: 100,
+            });
+            const previous = comments.find((comment) =>
+              comment.user?.type === 'Bot' && comment.body?.includes(marker));
+
+            if (previous) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: previous.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number,
+                body,
+              });
+            }

.github/workflows/frontend-js-size.yml

@@ -0,0 +1,343 @@
+name: Frontend JS size
+
+on:
+  pull_request:
+    paths:
+      - packages/frontend/**
+      - packages/frontend-shared/**
+      - packages/frontend-builder/**
+      - packages/i18n/**
+      - packages/icons-subsetter/**
+      - packages/misskey-js/**
+      - packages/misskey-reversi/**
+      - packages/misskey-bubble-game/**
+      - package.json
+      - pnpm-lock.yaml
+      - pnpm-workspace.yaml
+      - .node-version
+      - .github/workflows/frontend-js-size.yml
+      - .github/workflows/frontend-js-size-comment.yml
+permissions:
+  contents: read
+  pull-requests: read
+
+concurrency:
+  group: frontend-js-size-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  measure:
+    name: Measure frontend JS size
+    runs-on: ubuntu-latest
+    env:
+      FRONTEND_JS_SIZE_LOCALE: ja-JP
+    steps:
+      - name: Checkout base
+        uses: actions/checkout@v6.0.2
+        with:
+          repository: ${{ github.event.pull_request.base.repo.full_name }}
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: before
+          submodules: true
+
+      - name: Checkout pull request
+        uses: actions/checkout@v6.0.2
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: after
+          submodules: true
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v6.0.3
+        with:
+          package_json_file: after/package.json
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6.4.0
+        with:
+          node-version-file: after/.node-version
+          cache: pnpm
+          cache-dependency-path: |
+            before/pnpm-lock.yaml
+            after/pnpm-lock.yaml
+
+      - name: Install dependencies for base
+        working-directory: before
+        run: pnpm i --frozen-lockfile
+
+      - name: Build frontend for base
+        working-directory: before
+        run: |
+          pnpm --filter "frontend^..." run build
+          pnpm --filter frontend run build
+
+      - name: Install dependencies for pull request
+        working-directory: after
+        run: pnpm i --frozen-lockfile
+
+      - name: Build frontend for pull request
+        working-directory: after
+        run: |
+          pnpm --filter "frontend^..." run build
+          pnpm --filter frontend run build
+
+      - name: Write report script
+        shell: bash
+        run: |
+          mkdir -p .github/tmp
+          cat > .github/tmp/frontend-js-size-report.mjs <<'NODE'
+          import { promises as fs } from 'node:fs';
+          import path from 'node:path';
+
+          const marker = '<!-- misskey-frontend-js-size -->';
+          const locale = process.env.FRONTEND_JS_SIZE_LOCALE || 'ja-JP';
+          const topLimit = 10;
+
+          function normalizePath(filePath) {
+            return filePath.split(path.sep).join('/');
+          }
+
+          async function exists(filePath) {
+            try {
+              await fs.access(filePath);
+              return true;
+            } catch {
+              return false;
+            }
+          }
+
+          async function fileSize(filePath) {
+            const stat = await fs.stat(filePath);
+            return stat.size;
+          }
+
+          async function* walk(dir) {
+            for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
+              const fullPath = path.join(dir, entry.name);
+              if (entry.isDirectory()) {
+                yield* walk(fullPath);
+              } else if (entry.isFile()) {
+                yield fullPath;
+              }
+            }
+          }
+
+          function formatBytes(size) {
+            if (size == null) return '-';
+            if (size < 1024) return `${size} B`;
+            if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KiB`;
+            return `${(size / 1024 / 1024).toFixed(2)} MiB`;
+          }
+
+          function formatDiff(diff) {
+            if (diff == null) return '-';
+            if (diff === 0) return '0 B';
+            const sign = diff > 0 ? '+' : '-';
+            return `${sign}${formatBytes(Math.abs(diff))}`;
+          }
+
+          function escapeCell(value) {
+            return String(value).replaceAll('|', '\\|').replaceAll('\n', '<br>');
+          }
+
+          function entryDisplayName(entry) {
+            if (!entry) return '';
+            return entry.displayName === entry.file
+              ? entry.displayName
+              : `${entry.displayName} (${entry.file})`;
+          }
+
+          function findEntryKey(manifest) {
+            const entries = Object.entries(manifest);
+            return entries.find(([key, chunk]) => key === 'src/_boot_.ts' || chunk.src === 'src/_boot_.ts')?.[0]
+              ?? entries.find(([, chunk]) => chunk.name === 'entry' && chunk.isEntry)?.[0]
+              ?? entries.find(([, chunk]) => chunk.isEntry)?.[0]
+              ?? null;
+          }
+
+          function stableChunkKey(manifestKey, chunk) {
+            return chunk.src ?? (chunk.name ? `chunk:${chunk.name}` : manifestKey);
+          }
+
+          function collectStartupKeys(manifest) {
+            const entryKey = findEntryKey(manifest);
+            const keys = new Set();
+            if (entryKey == null) return keys;
+
+            function visit(key) {
+              if (keys.has(key)) return;
+              const chunk = manifest[key];
+              if (!chunk || !chunk.file?.endsWith('.js')) return;
+              keys.add(stableChunkKey(key, chunk));
+              for (const importKey of chunk.imports ?? []) {
+                visit(importKey);
+              }
+            }
+
+            visit(entryKey);
+            return keys;
+          }
+
+          async function resolveBuiltFile(outDir, file) {
+            const originalPath = path.join(outDir, file);
+            if (file.startsWith('scripts/')) {
+              const localizedFile = file.slice('scripts/'.length);
+              const localizedPath = path.join(outDir, locale, localizedFile);
+              if (await exists(localizedPath)) {
+                return {
+                  absolutePath: localizedPath,
+                  relativePath: `${locale}/${localizedFile}`,
+                };
+              }
+            }
+            return {
+              absolutePath: originalPath,
+              relativePath: file,
+            };
+          }
+
+          async function collectReport(repoDir) {
+            const outDir = path.join(repoDir, 'built/_frontend_vite_');
+            const manifestPath = path.join(outDir, 'manifest.json');
+            const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8'));
+            const byKey = new Map();
+            const byFile = new Set();
+
+            for (const [key, chunk] of Object.entries(manifest)) {
+              if (!chunk.file?.endsWith('.js')) continue;
+              const builtFile = await resolveBuiltFile(outDir, chunk.file);
+              const size = await fileSize(builtFile.absolutePath);
+              const stableKey = stableChunkKey(key, chunk);
+              const displayName = chunk.src ?? chunk.name ?? key;
+              byKey.set(stableKey, {
+                key: stableKey,
+                displayName,
+                file: builtFile.relativePath,
+                size,
+              });
+              byFile.add(builtFile.relativePath);
+            }
+
+            for await (const fullPath of walk(outDir)) {
+              if (!fullPath.endsWith('.js')) continue;
+              const relativePath = normalizePath(path.relative(outDir, fullPath));
+              if (byFile.has(relativePath)) continue;
+              if (relativePath.startsWith('scripts/') || relativePath.startsWith(`${locale}/`)) continue;
+              const size = await fileSize(fullPath);
+              byKey.set(relativePath, {
+                key: relativePath,
+                displayName: relativePath,
+                file: relativePath,
+                size,
+              });
+            }
+
+            return {
+              manifest,
+              chunks: Object.fromEntries(byKey),
+              startupKeys: [...collectStartupKeys(manifest)],
+            };
+          }
+
+          function compareRows(keys, before, after) {
+            return keys.map((key) => {
+              const beforeEntry = before.chunks[key];
+              const afterEntry = after.chunks[key];
+              const beforeSize = beforeEntry?.size ?? null;
+              const afterSize = afterEntry?.size ?? null;
+              return {
+                key,
+                file: entryDisplayName(afterEntry ?? beforeEntry),
+                beforeSize,
+                afterSize,
+                diff: beforeSize == null || afterSize == null ? null : afterSize - beforeSize,
+                sortSize: Math.max(beforeSize ?? 0, afterSize ?? 0),
+              };
+            });
+          }
+
+          function markdownTable(rows) {
+            if (rows.length === 0) {
+              return '_No JavaScript chunks found._';
+            }
+
+            const lines = [
+              '| File | Size (before) | Size (after) | Size (diff) |',
+              '| --- | ---: | ---: | ---: |',
+            ];
+            for (const row of rows) {
+              lines.push(`| ${escapeCell(row.file)} | ${formatBytes(row.beforeSize)} | ${formatBytes(row.afterSize)} | ${formatDiff(row.diff)} |`);
+            }
+            return lines.join('\n');
+          }
+
+          function unionTopKeys(before, after) {
+            const allKeys = new Set([
+              ...Object.keys(before.chunks),
+              ...Object.keys(after.chunks),
+            ]);
+            return compareRows([...allKeys], before, after)
+              .sort((a, b) => b.sortSize - a.sortSize || a.file.localeCompare(b.file))
+              .slice(0, topLimit)
+              .map((row) => row.key);
+          }
+
+          const beforeDir = process.argv[2];
+          const afterDir = process.argv[3];
+          const outFile = process.argv[4];
+          const beforeSha = process.env.BASE_SHA;
+          const afterSha = process.env.HEAD_SHA;
+
+          const before = await collectReport(beforeDir);
+          const after = await collectReport(afterDir);
+
+          const topRows = compareRows(unionTopKeys(before, after), before, after)
+            .sort((a, b) => b.sortSize - a.sortSize || a.file.localeCompare(b.file));
+
+          const startupKeys = new Set([
+            ...before.startupKeys,
+            ...after.startupKeys,
+          ]);
+          const startupRows = compareRows([...startupKeys], before, after)
+            .sort((a, b) => b.sortSize - a.sortSize || a.file.localeCompare(b.file));
+
+          const body = [
+            marker,
+            '## Frontend JavaScript size',
+            '',
+            `Compared locale: \`${locale}\``,
+            `Before: \`${beforeSha}\``,
+            `After: \`${afterSha}\``,
+            '',
+            '### Top 10 largest JS chunks',
+            '',
+            markdownTable(topRows),
+            '',
+            '### Startup JS chunks',
+            '',
+            markdownTable(startupRows),
+            '',
+            '_Top 10 is sorted by max(before, after) size. Startup chunks are the Vite entry for `src/_boot_.ts` and its static imports._',
+            '',
+          ].join('\n');
+
+          await fs.writeFile(outFile, body);
+          NODE
+
+      - name: Generate size report
+        shell: bash
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          node .github/tmp/frontend-js-size-report.mjs before after frontend-js-size-report.md
+          cat frontend-js-size-report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload size report
+        uses: actions/upload-artifact@v7
+        with:
+          name: frontend-js-size-report
+          path: frontend-js-size-report.md
+          if-no-files-found: error
+          retention-days: 1

.github/workflows/get-backend-memory.yml

@@ -10,6 +10,7 @@ on:
       - packages/backend/**
       - packages/misskey-js/**
       - .github/workflows/get-backend-memory.yml
+      - .github/workflows/report-backend-memory.yml
 
 jobs:
   get-memory-usage:
@@ -17,15 +18,6 @@ jobs:
     permissions:
       contents: read
 
-    strategy:
-      matrix:
-        memory-json-name: [memory-base.json, memory-head.json]
-        include:
-          - memory-json-name: memory-base.json
-            ref: ${{ github.base_ref }}
-          - memory-json-name: memory-head.json
-            ref: refs/pull/${{ github.event.number }}/merge
-
     services:
       postgres:
         image: postgres:18
@@ -40,37 +32,113 @@ jobs:
           - 56312:6379
 
     steps:
-    - uses: actions/checkout@v6.0.2
+    - name: Checkout base
+      uses: actions/checkout@v6.0.2
       with:
-        ref: ${{ matrix.ref }}
+        ref: ${{ github.base_ref }}
+        path: base
+        submodules: true
+    - name: Checkout head
+      uses: actions/checkout@v6.0.2
+      with:
+        ref: refs/pull/${{ github.event.number }}/merge
+        path: head
         submodules: true
     - name: Setup pnpm
       uses: pnpm/action-setup@v6.0.3
+      with:
+        package_json_file: head/package.json
     - name: Use Node.js
       uses: actions/setup-node@v6.4.0
       with:
-        node-version-file: '.node-version'
+        node-version-file: 'head/.node-version'
         cache: 'pnpm'
-    - run: pnpm i --frozen-lockfile
-    - name: Check pnpm-lock.yaml
+        cache-dependency-path: |
+          base/pnpm-lock.yaml
+          head/pnpm-lock.yaml
+    - name: Install base dependencies
+      working-directory: base
+      run: pnpm i --frozen-lockfile
+    - name: Check base pnpm-lock.yaml
+      working-directory: base
       run: git diff --exit-code pnpm-lock.yaml
-    - name: Copy Configure
-      run: cp .github/misskey/test.yml .config/default.yml
-    - name: Compile Configure
-      run: pnpm compile-config
-    - name: Build
-      run: pnpm build
-    - name: Run migrations
-      run: pnpm --filter backend migrate
-    - name: Measure memory usage
+    - name: Configure base
+      working-directory: base
       run: |
-        # Start the server and measure memory usage
-        node packages/backend/scripts/measure-memory.mjs > ${{ matrix.memory-json-name }}
+        cp .github/misskey/test.yml .config/default.yml
+        pnpm compile-config
+    - name: Build base
+      working-directory: base
+      run: pnpm build
+    - name: Install head dependencies
+      working-directory: head
+      run: pnpm i --frozen-lockfile
+    - name: Check head pnpm-lock.yaml
+      working-directory: head
+      run: git diff --exit-code pnpm-lock.yaml
+    - name: Configure head
+      working-directory: head
+      run: |
+        cp .github/misskey/test.yml .config/default.yml
+        pnpm compile-config
+    - name: Build head
+      working-directory: head
+      run: pnpm build
+    - name: Measure base memory usage
+      working-directory: base
+      run: |
+        node --input-type=module <<'EOF'
+        import pg from 'pg';
+        import Redis from 'ioredis';
+
+        const postgres = new pg.Client({
+          host: '127.0.0.1',
+          port: 54312,
+          database: 'postgres',
+          user: 'postgres',
+        });
+        await postgres.connect();
+        await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
+        await postgres.query('CREATE DATABASE "test-misskey"');
+        await postgres.end();
+
+        const redis = new Redis({ host: '127.0.0.1', port: 56312 });
+        await redis.flushall();
+        redis.disconnect();
+        EOF
+        pnpm --filter backend migrate
+        node packages/backend/scripts/measure-memory.mjs > ../memory-base.json
+    - name: Measure head memory usage
+      working-directory: head
+      run: |
+        node --input-type=module <<'EOF'
+        import pg from 'pg';
+        import Redis from 'ioredis';
+
+        const postgres = new pg.Client({
+          host: '127.0.0.1',
+          port: 54312,
+          database: 'postgres',
+          user: 'postgres',
+        });
+        await postgres.connect();
+        await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
+        await postgres.query('CREATE DATABASE "test-misskey"');
+        await postgres.end();
+
+        const redis = new Redis({ host: '127.0.0.1', port: 56312 });
+        await redis.flushall();
+        redis.disconnect();
+        EOF
+        pnpm --filter backend migrate
+        node packages/backend/scripts/measure-memory.mjs > ../memory-head.json
     - name: Upload Artifact
       uses: actions/upload-artifact@v7
       with:
-        name: memory-artifact-${{ matrix.memory-json-name }}
-        path: ${{ matrix.memory-json-name }}
+        name: memory-artifact-results
+        path: |
+          memory-base.json
+          memory-head.json
 
   save-pr-number:
     runs-on: ubuntu-latest

.github/workflows/report-backend-memory.yml

@@ -56,20 +56,25 @@ jobs:
 
           variation() {
             calc() {
-              BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0")
-              HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0")
+              BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // empty")
+              HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // empty")
+
+              if [ -z "$BASE" ] || [ -z "$HEAD" ]; then
+                echo "null"
+                return
+              fi
 
               DIFF=$((HEAD - BASE))
               if [ "$BASE" -gt 0 ]; then
-                DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
+                DIFF_PERCENT=$(awk -v diff="$DIFF" -v base="$BASE" 'BEGIN { printf "%.2f", (diff * 100) / base }')
               else
-                DIFF_PERCENT=0
+                DIFF_PERCENT=0.00
               fi
 
               # Convert KB to MB for readability
-              BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
-              HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
-              DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
+              BASE_MB=$(awk -v value="$BASE" 'BEGIN { printf "%.2f", value / 1024 }')
+              HEAD_MB=$(awk -v value="$HEAD" 'BEGIN { printf "%.2f", value / 1024 }')
+              DIFF_MB=$(awk -v value="$DIFF" 'BEGIN { printf "%.2f", value / 1024 }')
 
               JSON=$(jq -c -n \
                 --argjson base "$BASE_MB" \
@@ -82,11 +87,20 @@ jobs:
             }
 
             JSON=$(jq -c -n \
+              --argjson HeapUsed "$(calc $1 HeapUsed)" \
+              --argjson HeapTotal "$(calc $1 HeapTotal)" \
+              --argjson External "$(calc $1 External)" \
+              --argjson ArrayBuffers "$(calc $1 ArrayBuffers)" \
+              --argjson Pss "$(calc $1 Pss)" \
+              --argjson Private_Dirty "$(calc $1 Private_Dirty)" \
+              --argjson Private_Clean "$(calc $1 Private_Clean)" \
+              --argjson Shared_Dirty "$(calc $1 Shared_Dirty)" \
+              --argjson Shared_Clean "$(calc $1 Shared_Clean)" \
               --argjson VmRSS "$(calc $1 VmRSS)" \
               --argjson VmHWM "$(calc $1 VmHWM)" \
               --argjson VmSize "$(calc $1 VmSize)" \
               --argjson VmData "$(calc $1 VmData)" \
-              '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
+              '{HeapUsed: $HeapUsed, HeapTotal: $HeapTotal, External: $External, ArrayBuffers: $ArrayBuffers, Pss: $Pss, Private_Dirty: $Private_Dirty, Private_Clean: $Private_Clean, Shared_Dirty: $Shared_Dirty, Shared_Clean: $Shared_Clean, VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
 
             echo "$JSON"
           }
@@ -114,6 +128,10 @@ jobs:
             echo "|--------|------:|------:|------:|------:|" >> ./output.md
 
             line() {
+              if [ "$(echo "$RES" | jq -r ".${1}.${2} == null")" = "true" ]; then
+                return
+              fi
+
               METRIC=$2
               BASE=$(echo "$RES" | jq -r ".${1}.${2}.base")
               HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head")
@@ -125,8 +143,8 @@ jobs:
                 DIFF_PERCENT="+$DIFF_PERCENT"
               fi
 
-              # highlight VmRSS
-              if [ "$2" = "VmRSS" ]; then
+              # highlight the most useful process and OS memory metrics
+              if [ "$2" = "HeapUsed" ] || [ "$2" = "Pss" ]; then
                 METRIC="**${METRIC}**"
                 BASE="**${BASE}**"
                 HEAD="**${HEAD}**"
@@ -137,6 +155,15 @@ jobs:
               echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md
             }
 
+            line $1 HeapUsed
+            line $1 HeapTotal
+            line $1 External
+            line $1 ArrayBuffers
+            line $1 Pss
+            line $1 Private_Dirty
+            line $1 Private_Clean
+            line $1 Shared_Dirty
+            line $1 Shared_Clean
             line $1 VmRSS
             line $1 VmHWM
             line $1 VmSize
@@ -156,8 +183,9 @@ jobs:
           echo >> ./output.md
 
           # Determine if this is a significant change (more than 5% increase)
-          if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then
-            echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
+          WARNING_METRIC=$(echo "$RES" | jq -r 'if .afterGc.Pss != null then "Pss" elif .afterGc.VmRSS != null then "VmRSS" else empty end')
+          if [ -n "$WARNING_METRIC" ] && [ "$(echo "$RES" | jq -r ".afterGc.${WARNING_METRIC}.diff_percent | tonumber > 5")" = "true" ]; then
+            echo "⚠️ **Warning**: Memory usage (${WARNING_METRIC}) has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
             echo >> ./output.md
           fi
 

packages/backend/scripts/measure-memory.mjs

@@ -23,8 +23,9 @@ const __dirname = dirname(__filename);
 const SAMPLE_COUNT = 3; // Number of samples to measure
 const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup
 const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle
+const IPC_TIMEOUT = 30000; // 30 seconds timeout for IPC responses
 
-const keys = {
+const procStatusKeys = {
 	VmPeak: 0,
 	VmSize: 0,
 	VmHWM: 0,
@@ -37,30 +38,152 @@ const keys = {
 	VmSwap: 0,
 };
 
-async function getMemoryUsage(pid) {
-	const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8');
+const smapsRollupKeys = {
+	Pss: 0,
+	Shared_Clean: 0,
+	Shared_Dirty: 0,
+	Private_Clean: 0,
+	Private_Dirty: 0,
+	Swap: 0,
+	SwapPss: 0,
+};
 
+const runtimeKeys = {
+	HeapTotal: 0,
+	HeapUsed: 0,
+	External: 0,
+	ArrayBuffers: 0,
+};
+
+const memoryKeys = {
+	...procStatusKeys,
+	...smapsRollupKeys,
+	...runtimeKeys,
+};
+
+const phases = ['beforeGc', 'afterGc', 'afterRequest'];
+
+function parseMemoryFile(content, keys, path, required) {
 	const result = {};
 	for (const key of Object.keys(keys)) {
-		const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
+		const match = content.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`));
 		if (match) {
 			result[key] = parseInt(match[1], 10);
-		} else {
-			throw new Error(`Failed to parse ${key} from /proc/${pid}/status`);
+		} else if (required) {
+			throw new Error(`Failed to parse ${key} from ${path}`);
+		}
+	}
+	return result;
+}
+
+function bytesToKiB(value) {
+	return Math.round(value / 1024);
+}
+
+async function getMemoryUsage(pid) {
+	const path = `/proc/${pid}/status`;
+	const status = await fs.readFile(path, 'utf-8');
+
+	return parseMemoryFile(status, procStatusKeys, path, true);
+}
+
+async function getSmapsRollupMemoryUsage(pid) {
+	const path = `/proc/${pid}/smaps_rollup`;
+	try {
+		const smapsRollup = await fs.readFile(path, 'utf-8');
+		return parseMemoryFile(smapsRollup, smapsRollupKeys, path, false);
+	} catch (err) {
+		if (err.code === 'ENOENT' || err.code === 'EACCES') {
+			process.stderr.write(`Failed to read ${path}: ${err.message}\n`);
+			return {};
+		}
+		throw err;
+	}
+}
+
+function waitForMessage(serverProcess, predicate, description, timeout = IPC_TIMEOUT) {
+	return new Promise((resolve, reject) => {
+		const timer = globalThis.setTimeout(() => {
+			serverProcess.off('message', onMessage);
+			reject(new Error(`Timed out waiting for ${description}`));
+		}, timeout);
+
+		const onMessage = (message) => {
+			if (!predicate(message)) return;
+			globalThis.clearTimeout(timer);
+			serverProcess.off('message', onMessage);
+			resolve(message);
+		};
+
+		serverProcess.on('message', onMessage);
+	});
+}
+
+async function getRuntimeMemoryUsage(serverProcess) {
+	const response = waitForMessage(
+		serverProcess,
+		message => message != null && typeof message === 'object' && message.type === 'memory usage',
+		'memory usage',
+	);
+
+	serverProcess.send('memory usage');
+
+	const message = await response;
+	const memoryUsage = message.value;
+
+	return {
+		HeapTotal: bytesToKiB(memoryUsage.heapTotal),
+		HeapUsed: bytesToKiB(memoryUsage.heapUsed),
+		External: bytesToKiB(memoryUsage.external),
+		ArrayBuffers: bytesToKiB(memoryUsage.arrayBuffers),
+	};
+}
+
+async function getAllMemoryUsage(serverProcess) {
+	const pid = serverProcess.pid;
+	return {
+		...await getMemoryUsage(pid),
+		...await getSmapsRollupMemoryUsage(pid),
+		...await getRuntimeMemoryUsage(serverProcess),
+	};
+}
+
+function median(values) {
+	const sorted = values.toSorted((a, b) => a - b);
+	const center = Math.floor(sorted.length / 2);
+	if (sorted.length % 2 === 1) return sorted[center];
+	return Math.round((sorted[center - 1] + sorted[center]) / 2);
+}
+
+function summarizeResults(results) {
+	const summary = {};
+
+	for (const phase of phases) {
+		summary[phase] = {};
+		for (const key of Object.keys(memoryKeys)) {
+			const values = results
+				.map(result => result[phase][key])
+				.filter(value => Number.isFinite(value));
+
+			if (values.length > 0) {
+				summary[phase][key] = median(values);
+			}
 		}
 	}
 
-	return result;
+	return summary;
 }
 
 async function measureMemory() {
 	// Start the Misskey backend server using fork to enable IPC
-	const serverProcess = fork(join(__dirname, '../built/entry.js'), ['expose-gc'], {
+	const serverProcess = fork(join(__dirname, '../built/entry.js'), [], {
 		cwd: join(__dirname, '..'),
 		env: {
 			...process.env,
 			NODE_ENV: 'production',
 			MK_DISABLE_CLUSTERING: '1',
+			MK_ONLY_SERVER: '1',
+			MK_NO_DAEMONS: '1',
 		},
 		stdio: ['pipe', 'pipe', 'pipe', 'ipc'],
 		execArgv: [...process.execArgv, '--expose-gc'],
@@ -90,15 +213,18 @@ async function measureMemory() {
 	});
 
 	async function triggerGc() {
-		const ok = new Promise((resolve) => {
-			serverProcess.once('message', (message) => {
-				if (message === 'gc ok') resolve();
-			});
-		});
+		const ok = waitForMessage(
+			serverProcess,
+			message => message === 'gc ok' || message === 'gc unavailable',
+			'GC completion',
+		);
 
 		serverProcess.send('gc');
 
-		await ok;
+		const message = await ok;
+		if (message === 'gc unavailable') {
+			throw new Error('GC is unavailable. Start the process with --expose-gc to enable this feature.');
+		}
 
 		await setTimeout(1000);
 	}
@@ -139,13 +265,11 @@ async function measureMemory() {
 	// Wait for memory to settle
 	await setTimeout(MEMORY_SETTLE_TIME);
 
-	const pid = serverProcess.pid;
-
-	const beforeGc = await getMemoryUsage(pid);
+	const beforeGc = await getAllMemoryUsage(serverProcess);
 
 	await triggerGc();
 
-	const afterGc = await getMemoryUsage(pid);
+	const afterGc = await getAllMemoryUsage(serverProcess);
 
 	// create some http requests to simulate load
 	const REQUEST_COUNT = 10;
@@ -155,7 +279,7 @@ async function measureMemory() {
 
 	await triggerGc();
 
-	const afterRequest = await getMemoryUsage(pid);
+	const afterRequest = await getAllMemoryUsage(serverProcess);
 
 	// Stop the server
 	serverProcess.kill('SIGTERM');
@@ -187,35 +311,21 @@ async function measureMemory() {
 }
 
 async function main() {
-	// 直列の方が時間的に分散されて正確そうだから直列でやる
 	const results = [];
 	for (let i = 0; i < SAMPLE_COUNT; i++) {
+		process.stderr.write(`Starting sample ${i + 1}/${SAMPLE_COUNT}\n`);
 		const res = await measureMemory();
 		results.push(res);
 	}
 
-	// Calculate averages
-	const beforeGc = structuredClone(keys);
-	const afterGc = structuredClone(keys);
-	const afterRequest = structuredClone(keys);
-	for (const res of results) {
-		for (const key of Object.keys(keys)) {
-			beforeGc[key] += res.beforeGc[key];
-			afterGc[key] += res.afterGc[key];
-			afterRequest[key] += res.afterRequest[key];
-		}
-	}
-	for (const key of Object.keys(keys)) {
-		beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT);
-		afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT);
-		afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT);
-	}
+	const summary = summarizeResults(results);
 
 	const result = {
 		timestamp: new Date().toISOString(),
-		beforeGc,
-		afterGc,
-		afterRequest,
+		sampleCount: SAMPLE_COUNT,
+		aggregation: 'median',
+		...summary,
+		samples: results,
 	};
 
 	// Output as JSON to stdout

packages/backend/src/boot/common.ts

@@ -6,6 +6,7 @@
 import { NestFactory } from '@nestjs/core';
 import { init } from 'slacc';
 import { NestLogger } from '@/NestLogger.js';
+import { envOption } from '@/env.js';
 import type { Config } from '@/config.js';
 
 let slaccInitialized = false;
@@ -31,7 +32,7 @@ export async function server() {
 	const serverService = app.get(ServerService);
 	await serverService.launch();
 
-	if (process.env.NODE_ENV !== 'test') {
+	if (process.env.NODE_ENV !== 'test' && !envOption.noDaemons) {
 		const { ChartManagementService } = await import('../core/chart/ChartManagementService.js');
 		const { QueueStatsService } = await import('../daemons/QueueStatsService.js');
 		const { ServerStatsService } = await import('../daemons/ServerStatsService.js');
@@ -54,7 +55,9 @@ export async function jobQueue() {
 	});
 
 	jobQueue.get(QueueProcessorService).start();
-	jobQueue.get(ChartManagementService).start();
+	if (!envOption.noDaemons) {
+		jobQueue.get(ChartManagementService).start();
+	}
 
 	return jobQueue;
 }

packages/backend/src/boot/entry.ts

@@ -91,10 +91,20 @@ process.on('message', msg => {
 	if (msg === 'gc') {
 		if (global.gc != null) {
 			logger.info('Manual GC triggered');
-			global.gc();
+			for (let i = 0; i < 3; i++) {
+				global.gc();
+			}
 			if (process.send != null) process.send('gc ok');
 		} else {
 			logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.');
+			if (process.send != null) process.send('gc unavailable');
+		}
+	} else if (msg === 'memory usage') {
+		if (process.send != null) {
+			process.send({
+				type: 'memory usage',
+				value: process.memoryUsage(),
+			});
 		}
 	}
 });

packages/frontend-embed/vite.config.ts

@@ -153,11 +153,10 @@ export function getConfig(): UserConfig {
 							name: 'vue',
 							test: /node_modules[\\/]vue/,
 						}, {
-							// split each i18n related module to each distinct module, deny hoisting
+							// split i18n related module to distinct module
 							name: 'i18n',
-							test: /i18n\.ts/,
-							minSize: 0,
-							maxSize: 1,
+							includeDependenciesRecursively: false,
+							test: /i18n\.ts|locale\.ts/,
 						}],
 					},
 					entryFileNames: `scripts/${localesHash}-[hash:8].js`,

packages/frontend/vite.config.ts

@@ -194,11 +194,10 @@ export function getConfig(): UserConfig {
 							name: 'photoswipe',
 							test: /node_modules[\\/]photoswipe/,
 						}, {
-							// split each i18n related module to each distinct module, deny hoisting
+							// split i18n related module to distinct module
 							name: 'i18n',
-							test: /i18n\.ts/,
-							minSize: 0,
-							maxSize: 1,
+							includeDependenciesRecursively: false,
+							test: /i18n\.ts|locale\.ts/,
 						}],
 					},
 					entryFileNames: `scripts/${localesHash}-[hash:8].js`,

pnpm-workspace.yaml

@@ -63,6 +63,8 @@ overrides:
   '@aiscript-dev/aiscript-languageserver': '-'
   chokidar: 5.0.0
   lodash: 4.18.1
+  # remove when vite is updated to versions with rolldown > 1.1.0
+  vite>rolldown: "~1.1.0"
 engineStrict: true
 saveExact: true
 shellEmulator: true