diff --git a/.github/scripts/backend-js-footprint-loader.mjs b/.github/scripts/backend-js-footprint-loader.mjs new file mode 100644 index 0000000000..cd2c0af2e6 --- /dev/null +++ b/.github/scripts/backend-js-footprint-loader.mjs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { appendFileSync, statSync } from 'node:fs'; +import { extname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const traceFile = process.env.MK_BACKEND_JS_FOOTPRINT_TRACE; +const jsExtensions = new Set(['.js', '.mjs', '.cjs']); + +function recordLoadedFile(kind, url, format) { + if (traceFile == null || !url.startsWith('file:')) return; + + let filePath; + try { + filePath = fileURLToPath(url); + } catch { + return; + } + + const extension = extname(filePath); + if (!jsExtensions.has(extension)) return; + + let size = null; + try { + size = statSync(filePath).size; + } catch { + return; + } + + appendFileSync(traceFile, `${JSON.stringify({ + kind, + format, + path: filePath, + size, + timestamp: Date.now(), + })}\n`); +} + +export async function load(url, context, nextLoad) { + const result = await nextLoad(url, context); + recordLoadedFile('esm', url, result.format ?? context.format ?? null); + return result; +} diff --git a/.github/scripts/backend-js-footprint-require.cjs b/.github/scripts/backend-js-footprint-require.cjs new file mode 100644 index 0000000000..1adab8bc05 --- /dev/null +++ b/.github/scripts/backend-js-footprint-require.cjs @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +'use strict'; + +const { appendFileSync, statSync } = require('node:fs'); +const Module = require('node:module'); +const { extname } = require('node:path'); + +const traceFile = process.env.MK_BACKEND_JS_FOOTPRINT_TRACE; +const jsExtensions = new Set(['.js', '.mjs', '.cjs']); + +function recordLoadedFile(kind, filePath, request) { + if (traceFile == null || typeof filePath !== 'string') return; + + const extension = extname(filePath); + if (!jsExtensions.has(extension) && extension !== '.node') return; + + let size = null; + try { + size = statSync(filePath).size; + } catch { + return; + } + + appendFileSync(traceFile, `${JSON.stringify({ + kind, + format: extension === '.node' ? 'native' : 'commonjs', + path: filePath, + request, + size, + timestamp: Date.now(), + })}\n`); +} + +const originalLoad = Module._load; +const originalResolveFilename = Module._resolveFilename; + +Module._load = function load(request, parent, isMain) { + const resolved = originalResolveFilename.call(this, request, parent, isMain); + const result = originalLoad.apply(this, arguments); + recordLoadedFile('cjs', resolved, request); + return result; +}; diff --git a/.github/scripts/backend-js-footprint.mjs b/.github/scripts/backend-js-footprint.mjs new file mode 100644 index 0000000000..3def41d554 --- /dev/null +++ b/.github/scripts/backend-js-footprint.mjs @@ -0,0 +1,474 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { fork, spawn } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { cpus, tmpdir } from 'node:os'; +import { dirname, extname, join, relative, resolve, sep } from 'node:path'; +import { setTimeout } from 'node:timers/promises'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { gzipSync } from 'node:zlib'; +import * as fs from 'node:fs/promises'; +import * as fsSync from 'node:fs'; +import * as http from 'node:http'; +import * as util from './utility.mts'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const [repoDirArg, outputFileArg] = process.argv.slice(2); + +const STARTUP_TIMEOUT = util.readIntegerEnv('MK_JS_FOOTPRINT_STARTUP_TIMEOUT_MS', 120000, 1); +const SETTLE_TIME = util.readIntegerEnv('MK_JS_FOOTPRINT_SETTLE_TIME_MS', 10000, 0); +const REQUEST_COUNT = util.readIntegerEnv('MK_JS_FOOTPRINT_REQUEST_COUNT', 10, 0); +const MAX_TABLE_ITEMS = util.readIntegerEnv('MK_JS_FOOTPRINT_MAX_ITEMS', 20, 1); + +const repoDir = resolve(repoDirArg); +const outputFile = resolve(outputFileArg); +const backendDir = join(repoDir, 'packages/backend'); +const backendBuiltDir = join(backendDir, 'built'); +const traceFile = join(tmpdir(), `misskey-backend-js-footprint-${process.pid}-${Date.now()}.jsonl`); +const require = createRequire(join(repoDir, 'package.json')); +const ts = require('typescript'); +const jsExtensions = new Set(['.js', '.mjs', '.cjs']); +const fileMetricCache = new Map(); +const packageInfoCache = new Map(); +const nativePackageNames = new Set(); + +function isInside(parent, child) { + const rel = relative(parent, child); + return rel === '' || (!rel.startsWith('..') && !rel.includes(`..${sep}`)); +} + +function bytesToKiB(value) { + return Math.round(value / 1024); +} + +async function resetState() { + const backendRequire = createRequire(join(backendDir, 'package.json')); + const pg = backendRequire('pg'); + const Redis = backendRequire('ioredis'); + + const postgres = new pg.Client({ + host: '127.0.0.1', + port: 54312, + database: 'postgres', + user: 'postgres', + }); + + await postgres.connect(); + try { + await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); + await postgres.query('CREATE DATABASE "test-misskey"'); + } finally { + await postgres.end(); + } + + const redis = new Redis({ host: '127.0.0.1', port: 56312 }); + try { + await redis.flushall(); + } finally { + redis.disconnect(); + } +} + +function createRequest() { + return new Promise((resolvePromise, reject) => { + const req = http.request({ + host: 'localhost', + port: 61812, + path: '/api/meta', + method: 'POST', + }, res => { + res.on('data', () => { }); + res.on('end', () => resolvePromise()); + }); + req.on('error', reject); + req.end(); + }); +} + +async function waitForServerReady(serverProcess) { + let serverReady = false; + serverProcess.on('message', message => { + if (message === 'ok') serverReady = true; + }); + + const startupStartTime = Date.now(); + while (!serverReady) { + if (Date.now() - startupStartTime > STARTUP_TIMEOUT) { + serverProcess.kill('SIGTERM'); + throw new Error('Server startup timeout'); + } + await setTimeout(100); + } +} + +async function stopServer(serverProcess) { + serverProcess.kill('SIGTERM'); + + let exited = false; + await new Promise(resolvePromise => { + serverProcess.on('exit', () => { + exited = true; + resolvePromise(undefined); + }); + + setTimeout(10000).then(() => { + if (!exited) serverProcess.kill('SIGKILL'); + resolvePromise(undefined); + }); + }); +} + +function getPackageNameFromPath(filePath) { + const normalized = util.normalizePath(filePath); + const marker = '/node_modules/'; + const index = normalized.lastIndexOf(marker); + if (index === -1) return null; + + const rest = normalized.slice(index + marker.length).split('/'); + if (rest[0] === '.pnpm') { + const nestedNodeModulesIndex = rest.indexOf('node_modules'); + if (nestedNodeModulesIndex === -1) return null; + const packageParts = rest.slice(nestedNodeModulesIndex + 1); + if (packageParts.length === 0) return null; + return packageParts[0].startsWith('@') ? packageParts.slice(0, 2).join('/') : packageParts[0]; + } + + return rest[0]?.startsWith('@') ? rest.slice(0, 2).join('/') : rest[0] ?? null; +} + +function findPackageDir(filePath, packageName) { + const normalizedPackageName = packageName.split('/').join(sep); + let current = dirname(filePath); + + while (current !== dirname(current)) { + if (current.endsWith(`${sep}${normalizedPackageName}`) && fsSync.existsSync(join(current, 'package.json'))) { + return current; + } + + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + + return null; +} + +function readPackageInfo(filePath) { + const externalPackageName = getPackageNameFromPath(filePath); + if (externalPackageName != null) { + const packageDir = findPackageDir(filePath, externalPackageName); + const cacheKey = packageDir ?? externalPackageName; + if (packageInfoCache.has(cacheKey)) return packageInfoCache.get(cacheKey); + + let version = null; + if (packageDir != null) { + try { + const packageJson = JSON.parse(fsSync.readFileSync(join(packageDir, 'package.json'), 'utf8')); + version = typeof packageJson.version === 'string' ? packageJson.version : null; + } catch { } + } + + const info = { + category: 'external', + name: externalPackageName, + version, + dir: packageDir, + }; + packageInfoCache.set(cacheKey, info); + return info; + } + + if (isInside(backendBuiltDir, filePath)) { + return { + category: 'internal', + name: 'backend', + version: null, + dir: backendDir, + }; + } + + return { + category: 'internal', + name: 'workspace', + version: null, + dir: repoDir, + }; +} + +function analyzeSource(filePath, source) { + const sourceFile = ts.createSourceFile(filePath, source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const metrics = { + astNodeCount: 0, + functionCount: 0, + classCount: 0, + stringLiteralBytes: 0, + }; + + function visit(node) { + metrics.astNodeCount += 1; + + if ( + ts.isFunctionDeclaration(node) || + ts.isFunctionExpression(node) || + ts.isArrowFunction(node) || + ts.isMethodDeclaration(node) || + ts.isConstructorDeclaration(node) || + ts.isGetAccessorDeclaration(node) || + ts.isSetAccessorDeclaration(node) + ) { + metrics.functionCount += 1; + } else if (ts.isClassDeclaration(node) || ts.isClassExpression(node)) { + metrics.classCount += 1; + } else if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) { + metrics.stringLiteralBytes += Buffer.byteLength(node.text); + } + + ts.forEachChild(node, visit); + } + + visit(sourceFile); + return metrics; +} + +function readFileMetrics(filePath) { + if (fileMetricCache.has(filePath)) return fileMetricCache.get(filePath); + + const source = fsSync.readFileSync(filePath); + const sourceText = source.toString('utf8'); + const astMetrics = analyzeSource(filePath, sourceText); + const packageInfo = readPackageInfo(filePath); + const metric = { + path: filePath, + displayPath: util.normalizePath(relative(repoDir, filePath)), + sourceBytes: source.byteLength, + gzipBytes: gzipSync(source).byteLength, + ...astMetrics, + package: packageInfo, + }; + + fileMetricCache.set(filePath, metric); + return metric; +} + +async function readTraceRecords() { + let content = ''; + try { + content = await fs.readFile(traceFile, 'utf8'); + } catch (err) { + if (err.code === 'ENOENT') return []; + throw err; + } + + const records = []; + for (const line of content.split('\n')) { + if (line.trim() === '') continue; + try { + records.push(JSON.parse(line)); + } catch { } + } + return records; +} + +function emptyTotals() { + return { + loadedJsModules: 0, + loadedJsSourceBytes: 0, + loadedJsGzipBytes: 0, + astNodeCount: 0, + functionCount: 0, + classCount: 0, + stringLiteralBytes: 0, + externalPackageCount: 0, + nativeAddonPackageCount: 0, + }; +} + +function addFileMetrics(target, metric) { + target.loadedJsModules += 1; + target.loadedJsSourceBytes += metric.sourceBytes; + target.loadedJsGzipBytes += metric.gzipBytes; + target.astNodeCount += metric.astNodeCount; + target.functionCount += metric.functionCount; + target.classCount += metric.classCount; + target.stringLiteralBytes += metric.stringLiteralBytes; +} + +function summarizeRecords(records, phase) { + const jsPaths = new Set(); + const nativePaths = new Set(); + + for (const record of records) { + if (typeof record.path !== 'string') continue; + + const extension = extname(record.path); + if (jsExtensions.has(extension)) { + jsPaths.add(resolve(record.path)); + } else if (extension === '.node') { + nativePaths.add(resolve(record.path)); + } + } + + for (const nativePath of nativePaths) { + const packageInfo = readPackageInfo(nativePath); + if (packageInfo.category === 'external') nativePackageNames.add(packageInfo.name); + } + + const totals = emptyTotals(); + const packages = new Map(); + const modules = []; + + for (const filePath of [...jsPaths].toSorted()) { + let metric; + try { + metric = readFileMetrics(filePath); + } catch (err) { + process.stderr.write(`Failed to analyze ${filePath}: ${err.message}\n`); + continue; + } + + addFileMetrics(totals, metric); + + const packageKey = metric.package.name; + if (!packages.has(packageKey)) { + packages.set(packageKey, { + name: metric.package.name, + version: metric.package.version, + category: metric.package.category, + sourceBytes: 0, + gzipBytes: 0, + modules: 0, + astNodeCount: 0, + functionCount: 0, + classCount: 0, + stringLiteralBytes: 0, + nativeAddon: false, + }); + } + + const packageSummary = packages.get(packageKey); + packageSummary.sourceBytes += metric.sourceBytes; + packageSummary.gzipBytes += metric.gzipBytes; + packageSummary.modules += 1; + packageSummary.astNodeCount += metric.astNodeCount; + packageSummary.functionCount += metric.functionCount; + packageSummary.classCount += metric.classCount; + packageSummary.stringLiteralBytes += metric.stringLiteralBytes; + + modules.push({ + path: metric.displayPath, + package: metric.package.name, + category: metric.package.category, + sourceBytes: metric.sourceBytes, + gzipBytes: metric.gzipBytes, + astNodeCount: metric.astNodeCount, + functionCount: metric.functionCount, + classCount: metric.classCount, + stringLiteralBytes: metric.stringLiteralBytes, + }); + } + + for (const packageName of nativePackageNames) { + const packageSummary = packages.get(packageName); + if (packageSummary != null) packageSummary.nativeAddon = true; + } + + const externalPackages = [...packages.values()].filter(packageSummary => packageSummary.category === 'external'); + totals.externalPackageCount = externalPackages.length; + totals.nativeAddonPackageCount = externalPackages.filter(packageSummary => packageSummary.nativeAddon).length; + + return { + totals: { + ...totals, + loadedJsSourceKiB: bytesToKiB(totals.loadedJsSourceBytes), + loadedJsGzipKiB: bytesToKiB(totals.loadedJsGzipBytes), + stringLiteralKiB: bytesToKiB(totals.stringLiteralBytes), + }, + packages: [...packages.values()].toSorted((a, b) => b.sourceBytes - a.sourceBytes), + modules: modules.toSorted((a, b) => b.sourceBytes - a.sourceBytes).slice(0, MAX_TABLE_ITEMS), + }; +} + +async function measureFootprint() { + await fs.writeFile(traceFile, ''); + + process.stderr.write('Resetting database and Redis\n'); + await resetState(); + + process.stderr.write('Running migrations\n'); + await util.run('pnpm', ['--filter', 'backend', 'migrate'], { + cwd: repoDir, + env: process.env, + logStdout: true, + }); + + const serverProcess = fork(join(backendBuiltDir, 'entry.js'), [], { + cwd: backendDir, + env: { + ...process.env, + NODE_ENV: 'production', + MK_DISABLE_CLUSTERING: '1', + MK_ONLY_SERVER: '1', + MK_NO_DAEMONS: '1', + MK_BACKEND_JS_FOOTPRINT_TRACE: traceFile, + }, + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv: [ + '--require', + join(__dirname, 'backend-js-footprint-require.cjs'), + '--experimental-loader', + pathToFileURL(join(__dirname, 'backend-js-footprint-loader.mjs')).href, + ], + }); + + serverProcess.stdout?.on('data', data => { + process.stderr.write(`[server stdout] ${data}`); + }); + + serverProcess.stderr?.on('data', data => { + process.stderr.write(`[server stderr] ${data}`); + }); + + serverProcess.on('error', err => { + process.stderr.write(`[server error] ${err}\n`); + }); + + try { + await waitForServerReady(serverProcess); + await setTimeout(SETTLE_TIME); + + //const startup = summarizeRecords(await readTraceRecords(), 'startup'); + + await Promise.all( + Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), + ); + await setTimeout(1000); + + const afterRequest = summarizeRecords(await readTraceRecords(), 'afterRequest'); + + return { + timestamp: new Date().toISOString(), + measurement: { + strategy: 'runtime-loader-trace', + startupTimeoutMs: STARTUP_TIMEOUT, + settleTimeMs: SETTLE_TIME, + requestCount: REQUEST_COUNT, + cpus: cpus().length, + }, + phases: { + //startup, + afterRequest, + }, + }; + } finally { + await stopServer(serverProcess); + await fs.rm(traceFile, { force: true }); + } +} + +const result = await measureFootprint(); +await fs.writeFile(outputFile, `${JSON.stringify(result, null, 2)}\n`); diff --git a/.github/scripts/backend-memory-report.mts b/.github/scripts/backend-memory-report.mts new file mode 100644 index 0000000000..a758beb979 --- /dev/null +++ b/.github/scripts/backend-memory-report.mts @@ -0,0 +1,425 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { readFile, writeFile } from 'node:fs/promises'; +import * as util from './utility.mts'; +import * as heapSnapshotUtil from './heap-snapshot-util.mts'; +import type { MemoryReport } from './measure-backend-memory-comparison.mts'; + +const [baseFile, headFile, outputFile, baseJsFootprintFile, headJsFootprintFile] = process.argv.slice(2); + +type RuntimeLoadedJsFootprintReport = { + phases: Record<'afterRequest', { + totals: { + loadedJsModules: number; + loadedJsSourceBytes: number; + loadedJsGzipBytes: number; + astNodeCount: number; + functionCount: number; + classCount: number; + stringLiteralBytes: number; + externalPackageCount: number; + nativeAddonPackageCount: number; + }; + modules: { + path: string; + package: string; + category: string; + sourceBytes: number; + gzipBytes: number; + astNodeCount: number; + functionCount: number; + classCount: number; + stringLiteralBytes: number; + }[]; + }>; +}; + +const memoryReportPhases = [ + { + key: 'afterGc', + title: 'After GC', + }, +] as const; + +const metrics = [ + 'HeapUsed', + 'Pss', + 'Private_Dirty', + 'VmRSS', + 'External', +] as const; + +function formatMemoryMb(valueKiB: number | null | undefined) { + if (valueKiB == null) return '-'; + return `${util.formatNumber(valueKiB / 1024)} MB`; +} + +function getMemoryValue(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) { + return report.summary[phase].memoryUsage[metric]; +} + +function getMemoryValueFromSample(sample: MemoryReport['samples'][number], phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) { + return sample.phases[phase].memoryUsage[metric]; +} + +function getSampleSpread(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) { + const values = report.samples.map(sample => getMemoryValueFromSample(sample, phase, metric)); + if (values.length < 2) return null; + + const center = util.median(values); + return util.median(values.map(value => Math.abs(value - center))); +} + +function renderMainTableForPhase(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key']) { + const lines = [ + '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + + function formatDeltaMemory(diffKiB: number) { + return util.formatColoredDelta(formatMemoryMb(Math.abs(diffKiB)), diffKiB); + } + + for (const metric of metrics) { + const baseValue = getMemoryValue(base, phase, metric); + const headValue = getMemoryValue(head, phase, metric); + + const baseSpread = getSampleSpread(base, phase, metric); + const headSpread = getSampleSpread(head, phase, metric); + const summary = util.pairedDeltaSummary(base.samples, head.samples, (sample) => getMemoryValueFromSample(sample, phase, metric)); + const percent = summary.median * 100 / baseValue; + const deltaMedian = summary == null ? '-' : `${formatDeltaMemory(summary.median)}
${util.formatDeltaPercent(percent).replaceAll('\\%', '\\\\%')}`; + + lines.push(`| **${metric}** | ${formatMemoryMb(baseValue)}
± ${formatMemoryMb(baseSpread)} | ${formatMemoryMb(headValue)}
± ${formatMemoryMb(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : formatMemoryMb(summary.mad)} | ${summary == null ? '-' : formatDeltaMemory(summary.min)} | ${summary == null ? '-' : formatDeltaMemory(summary.max)} |`); + } + + return lines.join('\n'); +} + +/* +function measurementSummary(base, head) { + const baseCount = base?.sampleCount; + const headCount = head?.sampleCount; + const strategy = base?.comparison?.strategy; + if (baseCount == null || headCount == null) return null; + + if (strategy === 'interleaved-pairs') { + const rounds = base?.comparison?.rounds ?? baseCount; + const warmupRounds = base?.comparison?.warmupRounds ?? 0; + return `_Measured as ${rounds} interleaved base/head pairs after ${warmupRounds} warmup pair(s). Values are medians; ± is median absolute deviation._`; + } + + return `_Sample count: base ${baseCount}, head ${headCount}. Values are medians; ± is median absolute deviation._`; +} +*/ + +function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) { + const baseHeapSnapshotReport = { + summary: base.summary.afterGc.heapSnapshot!, + samples: base.samples.map(sample => ({ + round: sample.round, + data: sample.phases.afterGc.heapSnapshot!, + })), + }; + + const headHeapSnapshotReport = { + summary: head.summary.afterGc.heapSnapshot!, + samples: head.samples.map(sample => ({ + round: sample.round, + data: sample.phases.afterGc.heapSnapshot!, + })), + }; + + const table = heapSnapshotUtil.renderHeapSnapshotTable(baseHeapSnapshotReport, headHeapSnapshotReport); + if (table == null) return null; + + const lines = [ + '### V8 Heap Snapshot Statistics', + '', + table, + '', + ]; + + for (const graph of [ + heapSnapshotUtil.renderHeapSnapshotSankey(baseHeapSnapshotReport, 'Base'), + heapSnapshotUtil.renderHeapSnapshotSankey(headHeapSnapshotReport, 'Head'), + ]) { + if (graph == null) continue; + lines.push(graph); + lines.push(''); + } + + return lines.join('\n'); +} + +function getJsFootprintValue(report: RuntimeLoadedJsFootprintReport, phase: 'afterRequest', key: keyof RuntimeLoadedJsFootprintReport['phases'][typeof phase]['totals']) { + const value = report.phases[phase].totals[key]; + return Number.isFinite(value) ? value : null; +} + +function renderJsFootprintMetricTable(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) { + const metricRows = [ + ['Loaded JS modules', 'loadedJsModules', util.formatNumber], + ['Loaded JS source', 'loadedJsSourceBytes', util.formatBytes], + //['Loaded JS gzip estimate', 'loadedJsGzipBytes', util.formatBytes], + //['AST nodes', 'astNodeCount', util.formatNumber], + //['Functions', 'functionCount', util.formatNumber], + //['Classes', 'classCount', util.formatNumber], + //['String literals', 'stringLiteralBytes', util.formatBytes], + ['External packages loaded', 'externalPackageCount', util.formatNumber], + ['Native addon packages', 'nativeAddonPackageCount', util.formatNumber], + ] as const; + + const lines = [ + '| Metric | Base | Head | Δ | Δ (%) |', + '| --- | ---: | ---: | ---: | ---: |', + ]; + + for (const [title, key, formatter] of metricRows) { + const baseValue = getJsFootprintValue(base, 'afterRequest', key); + const headValue = getJsFootprintValue(head, 'afterRequest', key); + if (baseValue == null || headValue == null) continue; + + lines.push(`| **${title}** | ${formatter(baseValue)} | ${formatter(headValue)} | ${util.formatColoredDelta(formatter(headValue - baseValue), headValue - baseValue)} | ${util.calcAndFormatDeltaPercent(baseValue, headValue).replaceAll('\\%', '\\\\%')} |`); + } + + return lines.join('\n'); +} + +/* +function renderJsFootprintPhaseTable(base, head) { + const lines = [ + '| Phase | Base modules | Head modules | Δ modules | Base source | Head source | Δ source |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + + for (const [phase, title] of [['startup', 'Startup'], ['afterRequest', 'After warmup requests']]) { + const baseModules = getJsFootprintValue(base, phase, 'loadedJsModules'); + const headModules = getJsFootprintValue(head, phase, 'loadedJsModules'); + const baseSource = getJsFootprintValue(base, phase, 'loadedJsSourceBytes'); + const headSource = getJsFootprintValue(head, phase, 'loadedJsSourceBytes'); + if (baseModules == null || headModules == null || baseSource == null || headSource == null) continue; + + lines.push(`| ${title} | ${util.formatNumber(baseModules)} | ${util.formatNumber(headModules)} | ${formatPlainDelta(baseModules, headModules)} | ${util.formatBytes(baseSource)} | ${util.formatBytes(headSource)} | ${formatPlainDelta(baseSource, headSource, util.formatBytes)} |`); + } + + return lines.join('\n'); +} +*/ + +function packageMap(report: RuntimeLoadedJsFootprintReport) { + const map = new Map(); + for (const packageSummary of report.phases.afterRequest.packages) { + if (packageSummary?.category !== 'external' || typeof packageSummary.name !== 'string') continue; + map.set(packageSummary.name, packageSummary); + } + return map; +} + +function packageDisplayName(packageSummary: { name: string; version?: string | null }) { + if (packageSummary.version == null) return packageSummary.name; + return `${packageSummary.name} ${packageSummary.version}`; +} + +function renderNewExternalPackages(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) { + const basePackages = packageMap(base); + const headPackages = packageMap(head); + const newPackages = [...headPackages.values()] + .filter(packageSummary => !basePackages.has(packageSummary.name)) + .toSorted((a, b) => b.sourceBytes - a.sourceBytes) + .slice(0, 10); + + if (newPackages.length === 0) return null; + + const lines = [ + '#### Newly Loaded External Packages', + '', + '| Package | Loaded JS | Modules | Notes |', + '| --- | ---: | ---: | --- |', + ]; + + for (const packageSummary of newPackages) { + lines.push(`| ${packageDisplayName(packageSummary)} | ${util.formatBytes(packageSummary.sourceBytes)} | ${util.formatNumber(packageSummary.modules)} | ${packageSummary.nativeAddon ? 'native addon' : ''} |`); + } + + return lines.join('\n'); +} + +function renderLargestPackageIncreases(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) { + const basePackages = packageMap(base); + const headPackages = packageMap(head); + const increases = [...headPackages.values()] + .map(headPackage => { + const basePackage = basePackages.get(headPackage.name); + const baseSourceBytes = basePackage?.sourceBytes ?? 0; + const baseModules = basePackage?.modules ?? 0; + return { + ...headPackage, + baseSourceBytes, + baseModules, + sourceDiff: headPackage.sourceBytes - baseSourceBytes, + moduleDiff: headPackage.modules - baseModules, + }; + }) + .filter(packageSummary => packageSummary.sourceDiff > 0) + .toSorted((a, b) => b.sourceDiff - a.sourceDiff) + .slice(0, 10); + + if (increases.length === 0) return null; + + const lines = [ + '#### Largest Package Increases', + '', + '| Package | Base | Head | Δ | Modules Δ |', + '| --- | ---: | ---: | ---: | ---: |', + ]; + + for (const packageSummary of increases) { + lines.push(`| ${packageDisplayName(packageSummary)} | ${util.formatBytes(packageSummary.baseSourceBytes)} | ${util.formatBytes(packageSummary.sourceBytes)} | ${util.formatColoredDelta(util.formatBytes(packageSummary.sourceBytes - packageSummary.baseSourceBytes), packageSummary.sourceBytes - packageSummary.baseSourceBytes)} | ${util.formatColoredDelta(util.formatNumber(packageSummary.modules - packageSummary.baseModules), packageSummary.modules - packageSummary.baseModules)} |`); + } + + return lines.join('\n'); +} + +function renderNewLoadedModules(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) { + function moduleMap(report: RuntimeLoadedJsFootprintReport) { + const map = new Map(); + for (const moduleSummary of report.phases.afterRequest.modules) { + if (typeof moduleSummary.path !== 'string') continue; + map.set(moduleSummary.path, moduleSummary); + } + return map; + } + + const baseModules = moduleMap(base); + const headModules = moduleMap(head); + const newModules = [...headModules.values()] + .filter(moduleSummary => !baseModules.has(moduleSummary.path)) + .toSorted((a, b) => b.sourceBytes - a.sourceBytes) + .slice(0, 10); + + if (newModules.length === 0) return null; + + const lines = [ + '#### Largest Newly Loaded Modules', + '', + '| Module | Package | Loaded JS |', + '| --- | --- | ---: |', + ]; + + for (const moduleSummary of newModules) { + lines.push(`| \`${moduleSummary.path}\` | ${moduleSummary.package} | ${util.formatBytes(moduleSummary.sourceBytes)} |`); + } + + return lines.join('\n'); +} + +function renderJsFootprintSection(base: RuntimeLoadedJsFootprintReport, head: RuntimeLoadedJsFootprintReport) { + const lines = [ + '### Runtime Loaded JS Footprint', + '', + '
Click to show', + '', + renderJsFootprintMetricTable(base, head), + '', + //'#### Load Phase Breakdown', + //'', + //renderJsFootprintPhaseTable(base, head), + //'', + ]; + + for (const block of [ + renderNewExternalPackages(base, head), + renderLargestPackageIncreases(base, head), + renderNewLoadedModules(base, head), + ]) { + if (block == null) continue; + lines.push(block); + lines.push(''); + } + + lines.push('
'); + lines.push(''); + + return lines.join('\n'); +} + +const base = JSON.parse(await readFile(baseFile, 'utf8')) as MemoryReport; +const head = JSON.parse(await readFile(headFile, 'utf8')) as MemoryReport; +const baseJsFootprint = JSON.parse(await readFile(baseJsFootprintFile, 'utf8')) as RuntimeLoadedJsFootprintReport; +const headJsFootprint = JSON.parse(await readFile(headJsFootprintFile, 'utf8')) as RuntimeLoadedJsFootprintReport; +const lines = [ + '## ⚙️ Backend Memory Usage Report', + '', +]; + +//const summary = measurementSummary(base, head); +//if (summary != null) { +// lines.push(summary); +// lines.push(''); +//} + +for (const phase of memoryReportPhases) { + lines.push(`### ${phase.title}`); + lines.push(renderMainTableForPhase(base, head, phase.key)); + lines.push(''); +} + +const heapSnapshotSection = renderHeapSnapshotSection(base, head); +if (heapSnapshotSection != null) { + lines.push(heapSnapshotSection); + lines.push(''); +} + +const jsFootprintSection = renderJsFootprintSection(baseJsFootprint, headJsFootprint); +if (jsFootprintSection != null) { + lines.push(jsFootprintSection); + lines.push(''); +} + +function getWarningMetric(base: MemoryReport, head: MemoryReport) { + for (const metric of ['Pss', 'Private_Dirty', 'VmRSS'] as const) { + if (getMemoryValue(base, 'afterGc', metric) != null && getMemoryValue(head, 'afterGc', metric) != null) { + return metric; + } + } + return null; +} + +function getDiffPercent(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) { + const baseValue = getMemoryValue(base, phase, metric); + const headValue = getMemoryValue(head, phase, metric); + if (baseValue == null || headValue == null || baseValue <= 0) return null; + + return ((headValue - baseValue) * 100) / baseValue; +} + +function isBeyondSampleNoise(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) { + const baseValue = getMemoryValue(base, phase, metric); + const headValue = getMemoryValue(head, phase, metric); + if (baseValue == null || headValue == null) return false; + + const diff = headValue - baseValue; + if (diff <= 0) return false; + + const baseSpread = getSampleSpread(base, phase, metric); + const headSpread = getSampleSpread(head, phase, metric); + if (baseSpread == null || headSpread == null) return true; + + const combinedSpread = Math.hypot(baseSpread, headSpread); + return diff > combinedSpread * 3; +} + +const warningMetric = getWarningMetric(base, head); +const warningDiffPercent = warningMetric == null ? null : getDiffPercent(base, head, 'afterGc', warningMetric); +if (warningMetric != null && warningDiffPercent != null && warningDiffPercent > 5 && isBeyondSampleNoise(base, head, 'afterGc', warningMetric)) { + lines.push(`⚠️ **Warning**: Memory usage (${warningMetric}) has increased by more than 5% and exceeds the observed sample noise. Please verify this is not an unintended change.`); + lines.push(''); +} + +lines.push(`[See workflow logs for details](https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID})`); + +await writeFile(outputFile, `${lines.join('\n')}\n`); diff --git a/.github/scripts/frontend-bundle-visualizer-report.mjs b/.github/scripts/frontend-bundle-visualizer-report.mjs deleted file mode 100644 index 366bf9aa53..0000000000 --- a/.github/scripts/frontend-bundle-visualizer-report.mjs +++ /dev/null @@ -1,276 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises'; - -const [beforeFile, afterFile, outputFile] = process.argv.slice(2); - -if (beforeFile == null || afterFile == null || outputFile == null) { - console.error('Usage: node .github/scripts/frontend-bundle-visualizer-report.mjs '); - process.exit(1); -} - -const byteFormatter = new Intl.NumberFormat('en-US'); -const numberFormatter = new Intl.NumberFormat('en-US'); - -function formatBytes(value) { - if (!Number.isFinite(value) || value <= 0) return '0 B'; - - const units = ['B', 'KiB', 'MiB', 'GiB']; - let unitIndex = 0; - let size = value; - while (size >= 1024 && unitIndex < units.length - 1) { - size /= 1024; - unitIndex += 1; - } - - const maximumFractionDigits = size >= 10 || unitIndex === 0 ? 0 : 1; - return `${byteFormatter.format(Number(size.toFixed(maximumFractionDigits)))} ${units[unitIndex]}`; -} - -function formatNumber(value) { - return numberFormatter.format(value); -} - -function formatPercent(value) { - return `${Math.round(value)}%`; -} - -function sharePercent(value, total) { - if (total === 0) return '0%'; - return formatPercent((value / total) * 100); -} - -function formatMathText(text) { - return text - .replaceAll('\\', '\\\\') - .replaceAll('{', '\\{') - .replaceAll('}', '\\}') - .replaceAll('%', '\\%'); -} - -function formatColoredDiff(text, diff) { - const color = diff > 0 ? 'orange' : 'green'; - return `$\\color{${color}}{\\text{${formatMathText(text)}}}$`; -} - -function formatDiff(before, after, formatter) { - const diff = after - before; - if (diff === 0) return formatter(0); - - const sign = diff > 0 ? '+' : '-'; - return formatColoredDiff(`${sign}${formatter(Math.abs(diff))}`, diff); -} - -function formatDiffPercent(before, after) { - if (before === 0 && after === 0) return '0%'; - if (before === 0) return '-'; - - const diff = after - before; - if (diff === 0) return '0%'; - - const sign = diff > 0 ? '+' : '-'; - return formatColoredDiff(`${sign}${formatPercent(Math.abs(diff / before) * 100)}`, diff); -} - -function tableCell(value) { - return String(value).replaceAll('|', '\\|').replaceAll('\r', ' ').replaceAll('\n', ' '); -} - -function code(value) { - const sanitized = String(value).replaceAll('\r', ' ').replaceAll('\n', ' '); - const backtickRuns = sanitized.match(/`+/g) ?? []; - const fenceLength = Math.max(1, ...backtickRuns.map((run) => run.length + 1)); - const fence = '`'.repeat(fenceLength); - const padding = sanitized.startsWith('`') || sanitized.endsWith('`') ? ' ' : ''; - - return `${fence}${padding}${sanitized}${padding}${fence}`; -} - -function tableCode(value) { - return tableCell(code(value)); -} - -function collectReport(data) { - const nodeParts = data.nodeParts ?? {}; - const nodeMetas = Object.values(data.nodeMetas ?? {}); - const moduleRows = []; - const bundleMap = new Map(); - - for (const meta of nodeMetas) { - const row = { - id: meta.id, - bundles: 0, - renderedLength: 0, - gzipLength: 0, - brotliLength: 0, - importedByCount: meta.importedBy?.length ?? 0, - importedCount: meta.imported?.length ?? 0, - }; - - for (const [bundleId, partUid] of Object.entries(meta.moduleParts ?? {})) { - const part = nodeParts[partUid]; - if (part == null) continue; - - row.bundles += 1; - row.renderedLength += part.renderedLength; - row.gzipLength += part.gzipLength; - row.brotliLength += part.brotliLength; - - const bundle = bundleMap.get(bundleId) ?? { - id: bundleId, - modules: 0, - renderedLength: 0, - gzipLength: 0, - brotliLength: 0, - }; - bundle.modules += 1; - bundle.renderedLength += part.renderedLength; - bundle.gzipLength += part.gzipLength; - bundle.brotliLength += part.brotliLength; - bundleMap.set(bundleId, bundle); - } - - if (row.bundles > 0) { - moduleRows.push(row); - } - } - - let staticImports = 0; - let dynamicImports = 0; - for (const meta of nodeMetas) { - for (const imported of meta.imported ?? []) { - if (imported.dynamic) { - dynamicImports += 1; - } else { - staticImports += 1; - } - } - } - - const bundleRows = [...bundleMap.values()].sort((a, b) => b.renderedLength - a.renderedLength); - const hotModules = [...moduleRows].sort((a, b) => b.renderedLength - a.renderedLength); - const totalRendered = moduleRows.reduce((sum, row) => sum + row.renderedLength, 0); - const totalGzip = moduleRows.reduce((sum, row) => sum + row.gzipLength, 0); - const totalBrotli = moduleRows.reduce((sum, row) => sum + row.brotliLength, 0); - - return { - options: data.options ?? {}, - summary: { - bundles: bundleRows.length, - modules: moduleRows.length, - entries: nodeMetas.filter((meta) => meta.isEntry).length, - externals: nodeMetas.filter((meta) => meta.isExternal).length, - staticImports, - dynamicImports, - }, - metrics: { - renderedLength: totalRendered, - gzipLength: totalGzip, - brotliLength: totalBrotli, - }, - hotModules, - }; -} - -function renderSummaryTable(before, after) { - const summary = [ - 'bundles', - 'modules', - 'entries', - //'externals', - 'staticImports', - 'dynamicImports', - ]; - - const metrics = [ - 'renderedLength', - 'gzipLength', - 'brotliLength', - ]; - - return [ - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ``, - ...summary.map((key) => ``), - ...metrics.map((key) => ``), - ``, - ``, - ``, - ...summary.map((key) => ``), - ...metrics.map((key) => ``), - ``, - ``, - ``, - ``, - ...summary.map((key) => ``), - ...metrics.map((key) => ``), - ``, - ``, - ``, - ...summary.map((key) => ``), - ...metrics.map((key) => ``), - ``, - ``, - `
BundlesModulesEntriesImportsSize
StaticDynamicRenderedGzipBrotli
Before${formatNumber(before.summary[key])}${formatBytes(before.metrics[key])}
After${formatNumber(after.summary[key])}${formatBytes(after.metrics[key])}
Δ${formatDiff(before.summary[key], after.summary[key], formatNumber)}${formatDiff(before.metrics[key], after.metrics[key], formatBytes)}
Δ (%)${formatDiffPercent(before.summary[key], after.summary[key])}${formatDiffPercent(before.metrics[key], after.metrics[key])}
`, - ]; -} - -const beforeData = JSON.parse(await readFile(beforeFile, 'utf8')); -const afterData = JSON.parse(await readFile(afterFile, 'utf8')); -const before = collectReport(beforeData); -const after = collectReport(afterData); -const lines = [ - '## Frontend Bundle Report', - '', - ...renderSummaryTable(before, after), - '', - '
', - 'Top 10', - '', -]; - -for (const row of after.hotModules.slice(0, 10)) { - lines.push(`- ${code(row.id)}: ${sharePercent(row.renderedLength, after.metrics.renderedLength)} (${formatBytes(row.renderedLength)})`); -} - -lines.push( - '', - '
', -); - -lines.push( - '', - '
', - 'Hot Modules (Self Size)', - '', - '| Module | Bundles | Rendered | Share | Gzip | Brotli | Imports | Imported By |', - '|---|---:|---:|---:|---:|---:|---:|---:|', -); - -for (const row of after.hotModules.slice(0, 15)) { - lines.push(`| ${tableCode(row.id)} | ${row.bundles} | ${formatBytes(row.renderedLength)} | ${sharePercent(row.renderedLength, after.metrics.renderedLength)} | ${formatBytes(row.gzipLength)} | ${formatBytes(row.brotliLength)} | ${row.importedCount} | ${row.importedByCount} |`); -} - -lines.push( - '', - '
', -); - -await writeFile(outputFile, `${lines.join('\n')}\n`); diff --git a/.github/scripts/frontend-js-size.mjs b/.github/scripts/frontend-js-size.mjs deleted file mode 100644 index 75744bfcba..0000000000 --- a/.github/scripts/frontend-js-size.mjs +++ /dev/null @@ -1,344 +0,0 @@ -import { promises as fs } from 'node:fs'; -import path from 'node:path'; - -const marker = ''; -const locale = process.env.FRONTEND_JS_SIZE_LOCALE || 'ja-JP'; - -function normalizePath(filePath) { - return filePath.split(path.sep).join('/'); -} - -async function exists(filePath) { - try { - await fs.access(filePath); - return true; - } catch { - return false; - } -} - -async function fileSize(filePath) { - const stat = await fs.stat(filePath); - return stat.size; -} - -async function* walk(dir) { - for (const entry of await fs.readdir(dir, { withFileTypes: true })) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - yield* walk(fullPath); - } else if (entry.isFile()) { - yield fullPath; - } - } -} - -function formatBytes(size) { - if (size == null) return '-'; - if (size < 1024) return `${size} B`; - if (size < 1024 * 1024) return `${stripTrailingZeros((size / 1024).toFixed(1))} KiB`; - return `${stripTrailingZeros((size / 1024 / 1024).toFixed(2))} MiB`; -} - -function stripTrailingZeros(value) { - return value.replace(/\.0+$/, '').replace(/(\.\d*?)0+$/, '$1'); -} - -function formatMathText(text) { - return text - .replaceAll('\\', '\\\\') - .replaceAll('{', '\\{') - .replaceAll('}', '\\}') - .replaceAll('%', '\\\\%'); -} - -function formatDiff(diff) { - if (diff == null) return '-'; - if (diff === 0) return '0 B'; - const sign = diff > 0 ? '+' : '-'; - const text = `${sign}${formatBytes(Math.abs(diff))}`; - const color = diff > 0 ? 'orange' : 'green'; - return `$\\color{${color}}{\\text{${formatMathText(text)}}}$`; -} - -function formatDiffPercent(beforeSize, afterSize) { - if (beforeSize == null || beforeSize === 0 || afterSize == null || afterSize === 0) return '-'; - const diff = afterSize - beforeSize; - if (diff === 0) return `0%`; - const percent = Math.round(diff / beforeSize * 100); - const color = diff > 0 ? 'orange' : 'green'; - return `$\\color{${color}}{\\text{${formatMathText(percent.toString() + '%')}}}$`; -} - -function escapeCell(value) { - return String(value).replaceAll('|', '\\|').replaceAll('\n', '
'); -} - -function entryDisplayName(entry) { - if (!entry) return ''; - return entry.displayName || entry.file; -} - -function findEntryKey(manifest) { - const entries = Object.entries(manifest); - return entries.find(([key, chunk]) => key === 'src/_boot_.ts' || chunk.src === 'src/_boot_.ts')?.[0] - ?? entries.find(([, chunk]) => chunk.name === 'entry' && chunk.isEntry)?.[0] - ?? entries.find(([, chunk]) => chunk.isEntry)?.[0] - ?? null; -} - -function stableChunkKey(manifestKey, chunk) { - return chunk.src ?? (chunk.name ? `chunk:${chunk.name}` : manifestKey); -} - -function collectStartupKeys(manifest) { - const entryKey = findEntryKey(manifest); - const keys = new Set(); - if (entryKey == null) return keys; - - function visit(key) { - if (keys.has(key)) return; - const chunk = manifest[key]; - if (!chunk || !chunk.file?.endsWith('.js')) return; - keys.add(stableChunkKey(key, chunk)); - for (const importKey of chunk.imports ?? []) { - visit(importKey); - } - } - - visit(entryKey); - return keys; -} - -async function resolveBuiltFile(outDir, file) { - if (file.startsWith('scripts/')) { - const localizedFile = file.slice('scripts/'.length); - const localizedPath = path.join(outDir, locale, localizedFile); - if (await exists(localizedPath)) { - return { - absolutePath: localizedPath, - relativePath: `${locale}/${localizedFile}`, - }; - } - - throw new Error(`Expected ${locale} localized chunk for ${file}, but ${localizedPath} was not found.`); - } - return { - absolutePath: path.join(outDir, file), - relativePath: file, - }; -} - -async function collectReport(repoDir) { - const outDir = path.join(repoDir, 'built/_frontend_vite_'); - const manifestPath = path.join(outDir, 'manifest.json'); - const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')); - const byKey = new Map(); - const byFile = new Set(); - - for (const [key, chunk] of Object.entries(manifest)) { - if (!chunk.file?.endsWith('.js')) continue; - const builtFile = await resolveBuiltFile(outDir, chunk.file); - const size = await fileSize(builtFile.absolutePath); - const stableKey = stableChunkKey(key, chunk); - const displayName = chunk.src ?? chunk.name ?? key; - byKey.set(stableKey, { - key: stableKey, - displayName, - file: builtFile.relativePath, - size, - }); - byFile.add(builtFile.relativePath); - } - - const localeDir = path.join(outDir, locale); - if (await exists(localeDir)) { - for await (const fullPath of walk(localeDir)) { - if (!fullPath.endsWith('.js')) continue; - const relativePath = normalizePath(path.relative(outDir, fullPath)); - if (byFile.has(relativePath)) continue; - const size = await fileSize(fullPath); - byKey.set(relativePath, { - key: relativePath, - displayName: relativePath, - file: relativePath, - size, - }); - } - } - - return { - manifest, - chunks: Object.fromEntries(byKey), - startupKeys: [...collectStartupKeys(manifest)], - }; -} - -function commonKeys(before, after) { - return Object.keys(before.chunks) - .filter((key) => after.chunks[key] != null); -} - -function addedKeys(before, after) { - return Object.keys(after.chunks) - .filter((key) => before.chunks[key] == null); -} - -function removedKeys(before, after) { - return Object.keys(before.chunks) - .filter((key) => after.chunks[key] == null); -} - -function getChunkComparisonRows(keys, before, after) { - return keys.map((key) => { - const beforeEntry = before.chunks[key]; - const afterEntry = after.chunks[key]; - const beforeSize = beforeEntry?.size ?? 0; - const afterSize = afterEntry?.size ?? 0; - return { - key, - name: entryDisplayName(beforeEntry ?? afterEntry), - chunkFile: beforeEntry?.file ?? afterEntry?.file, - beforeSize, - afterSize, - sortSize: Math.max(beforeSize, afterSize), - }; - }); -} - -function markdownTable(rows, total) { - if (rows.length === 0) return '_No data_'; - - const lines = [ - '| Chunk | Before | After | Δ | Δ (%) |', - '| --- | ---: | ---: | ---: | ---: |', - ]; - if (total != null) { - lines.push(`| (total) | ${formatBytes(total.beforeSize)} | ${formatBytes(total.afterSize)} | ${formatDiff(total.afterSize - total.beforeSize)} | ${formatDiffPercent(total.beforeSize, total.afterSize)} |`); - lines.push('| | | | | |'); - } - for (const row of rows) { - lines.push(`|
\`${escapeCell(row.name)}\` \`${escapeCell(row.chunkFile)}\`
| ${formatBytes(row.beforeSize)} | ${formatBytes(row.afterSize)} | ${formatDiff(row.afterSize - row.beforeSize)} | ${formatDiffPercent(row.beforeSize, row.afterSize)} |`); - } - return lines.join('\n'); -} - -function chunkRows(keys, report) { - return keys.map((key) => { - const entry = report.chunks[key]; - return { - key, - name: entryDisplayName(entry), - chunkFile: entry.file, - size: entry.size, - }; - }); -} - -function markdownChunkTable(rows) { - if (rows.length === 0) return '_No data_'; - - const lines = [ - '| Chunk | Size |', - '| --- | ---: |', - ]; - for (const row of rows) { - lines.push(`|
\`${escapeCell(row.name)}\` \`${escapeCell(row.chunkFile)}\`
| ${formatBytes(row.size)} |`); - } - return lines.join('\n'); -} - -const beforeDir = process.argv[2]; -const afterDir = process.argv[3]; -const outFile = process.argv[4]; -const beforeSha = process.env.BASE_SHA; -const afterSha = process.env.HEAD_SHA; - -const before = await collectReport(beforeDir); -const after = await collectReport(afterDir); - -const commonChunkKeys = commonKeys(before, after); -const comparisonRows = getChunkComparisonRows(commonChunkKeys, before, after); - -const diffRows = comparisonRows - .filter((row) => row.beforeSize !== row.afterSize) - .sort((a, b) => Math.abs(b.afterSize - b.beforeSize) - Math.abs(a.afterSize - a.beforeSize) - || (b.afterSize - b.beforeSize) - (a.afterSize - a.beforeSize) - || b.sortSize - a.sortSize - || a.name.localeCompare(b.name)) - .slice(0, 30); - -const diffTotal = { - beforeSize: comparisonRows.reduce((sum, row) => sum + row.beforeSize, 0), - afterSize: comparisonRows.reduce((sum, row) => sum + row.afterSize, 0), -}; - -const addedRows = chunkRows(addedKeys(before, after), after) - .sort((a, b) => b.size - a.size || a.name.localeCompare(b.name)); - -const removedRows = chunkRows(removedKeys(before, after), before) - .sort((a, b) => b.size - a.size || a.name.localeCompare(b.name)); - -const startupKeys = new Set([ - ...before.startupKeys, - ...after.startupKeys, -]); -const startupComparisonRows = getChunkComparisonRows([...startupKeys], before, after); -const startupRows = startupComparisonRows - .sort((a, b) => Math.abs(b.afterSize - b.beforeSize) - Math.abs(a.afterSize - a.beforeSize) - || (b.afterSize - b.beforeSize) - (a.afterSize - a.beforeSize) - || b.sortSize - a.sortSize - || a.name.localeCompare(b.name)); -const startupTotal = { - beforeSize: startupComparisonRows.reduce((sum, row) => sum + row.beforeSize, 0), - afterSize: startupComparisonRows.reduce((sum, row) => sum + row.afterSize, 0), -}; - -const largeRows = comparisonRows - .sort((a, b) => b.sortSize - a.sortSize || a.name.localeCompare(b.name)) - .slice(0, 30); - -const body = [ - marker, - `## Frontend chunk report (${locale})`, - '', - '
', - `Diffs`, - '', - markdownTable(diffRows, diffTotal), - '', - '
', - '', - '
', - `Added (${addedRows.length})`, - '', - markdownChunkTable(addedRows), - '', - '
', - '', - '
', - `Removed (${removedRows.length})`, - '', - markdownChunkTable(removedRows), - '', - '
', - '', - '
', - `Startup`, - '', - markdownTable(startupRows, startupTotal), - '', - `_Startup chunks are the Vite entry for \`src/_boot_.ts\` and its static imports._`, - '', - '
', - '', - '
', - `Largest`, - '', - markdownTable(largeRows), - '', - '
', - '', -].join('\n'); - -await fs.writeFile(outFile, body); diff --git a/.github/scripts/frontend-js-size.mts b/.github/scripts/frontend-js-size.mts new file mode 100644 index 0000000000..7bbc19ce0b --- /dev/null +++ b/.github/scripts/frontend-js-size.mts @@ -0,0 +1,503 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import * as util from './utility.mts'; + +const marker = ''; + +const locale = process.env.FRONTEND_JS_SIZE_LOCALE ?? 'ja-JP'; + +//function sharePercent(value, total) { +// if (total === 0) return '0%'; +// return Math.round((value / total) * 100) + '%'; +//} + +function escapeCell(value: string) { + return String(value).replaceAll('|', '\\|').replaceAll('\n', '
'); +} + +//function tableCell(value) { +// return String(value).replaceAll('|', '\\|').replaceAll('\r', ' ').replaceAll('\n', ' '); +//} + +//function code(value) { +// const sanitized = String(value).replaceAll('\r', ' ').replaceAll('\n', ' '); +// const backtickRuns = sanitized.match(/`+/g) ?? []; +// const fenceLength = Math.max(1, ...backtickRuns.map((run) => run.length + 1)); +// const fence = '`'.repeat(fenceLength); +// const padding = sanitized.startsWith('`') || sanitized.endsWith('`') ? ' ' : ''; +// +// return `${fence}${padding}${sanitized}${padding}${fence}`; +//} + +//function tableCode(value) { +// return tableCell(code(value)); +//} + +type Manifest = Record; + +type FileEntry = { + key: string; + displayName: string; + file: string; + size: number; +}; + +function entryDisplayName(entry: FileEntry) { + if (!entry) return ''; + return entry.displayName || entry.file; +} + +function findEntryKey(manifest: Manifest) { + const entries = Object.entries(manifest); + return entries.find(([key, chunk]) => key === 'src/_boot_.ts' || chunk.src === 'src/_boot_.ts')?.[0] + ?? entries.find(([, chunk]) => chunk.name === 'entry' && chunk.isEntry)?.[0] + ?? entries.find(([, chunk]) => chunk.isEntry)?.[0] + ?? null; +} + +function stableChunkKey(manifestKey: string, chunk: Manifest[string]) { + return chunk.src ?? (chunk.name ? `chunk:${chunk.name}` : manifestKey); +} + +function collectStartupKeys(manifest: Manifest) { + const entryKey = findEntryKey(manifest); + const keys = new Set(); + if (entryKey == null) return keys; + + function visit(key: string) { + if (keys.has(key)) return; + const chunk = manifest[key]; + if (!chunk || !chunk.file?.endsWith('.js')) return; + keys.add(stableChunkKey(key, chunk)); + for (const importKey of chunk.imports ?? []) { + visit(importKey); + } + } + + visit(entryKey); + return keys; +} + +async function resolveBuiltFile(outDir: string, file: string) { + if (file.startsWith('scripts/')) { + const localizedFile = file.slice('scripts/'.length); + const localizedPath = path.join(outDir, locale, localizedFile); + if (await util.fileExists(localizedPath)) { + return { + absolutePath: localizedPath, + relativePath: `${locale}/${localizedFile}`, + }; + } + + throw new Error(`Expected ${locale} localized chunk for ${file}, but ${localizedPath} was not found.`); + } + return { + absolutePath: path.join(outDir, file), + relativePath: file, + }; +} + +async function collectReport(repoDir: string) { + const outDir = path.join(repoDir, 'built/_frontend_vite_'); + const manifestPath = path.join(outDir, 'manifest.json'); + const manifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')) as Manifest; + const byKey = new Map(); + const byFile = new Set(); + + for (const [key, chunk] of Object.entries(manifest)) { + if (!chunk.file?.endsWith('.js')) continue; + const builtFile = await resolveBuiltFile(outDir, chunk.file); + const size = await util.fileSize(builtFile.absolutePath); + const stableKey = stableChunkKey(key, chunk); + const displayName = chunk.src ?? chunk.name ?? key; + byKey.set(stableKey, { + key: stableKey, + displayName, + file: builtFile.relativePath, + size, + }); + byFile.add(builtFile.relativePath); + } + + const localeDir = path.join(outDir, locale); + if (await util.fileExists(localeDir)) { + for await (const fullPath of util.traverseDirectory(localeDir)) { + if (!fullPath.endsWith('.js')) continue; + const relativePath = util.normalizePath(path.relative(outDir, fullPath)); + if (byFile.has(relativePath)) continue; + const size = await util.fileSize(fullPath); + byKey.set(relativePath, { + key: relativePath, + displayName: relativePath, + file: relativePath, + size, + }); + } + } + + return { + manifest, + chunks: Object.fromEntries(byKey), + startupKeys: [...collectStartupKeys(manifest)], + }; +} + +type VisualizerReport = { + nodeParts?: Record; + nodeMetas?: Record; + renderedLength: number; + gzipLength: number; + brotliLength: number; + }>; + options?: Record; +}; + + +function collectVisualizerReport(data: VisualizerReport) { + const nodeParts = data.nodeParts ?? {}; + const nodeMetas = Object.values(data.nodeMetas ?? {}); + const moduleRows = []; + const bundleMap = new Map(); + + for (const meta of nodeMetas) { + const row = { + id: meta.id, + bundles: 0, + renderedLength: 0, + gzipLength: 0, + brotliLength: 0, + importedByCount: meta.importedBy?.length ?? 0, + importedCount: meta.imported?.length ?? 0, + }; + + for (const [bundleId, partUid] of Object.entries(meta.moduleParts ?? {})) { + const part = nodeParts[partUid]; + if (part == null) continue; + + row.bundles += 1; + row.renderedLength += part.renderedLength; + row.gzipLength += part.gzipLength; + row.brotliLength += part.brotliLength; + + const bundle = bundleMap.get(bundleId) ?? { + id: bundleId, + modules: 0, + renderedLength: 0, + gzipLength: 0, + brotliLength: 0, + }; + bundle.modules += 1; + bundle.renderedLength += part.renderedLength; + bundle.gzipLength += part.gzipLength; + bundle.brotliLength += part.brotliLength; + bundleMap.set(bundleId, bundle); + } + + if (row.bundles > 0) { + moduleRows.push(row); + } + } + + let staticImports = 0; + let dynamicImports = 0; + for (const meta of nodeMetas) { + for (const imported of meta.imported ?? []) { + if (imported.dynamic) { + dynamicImports += 1; + } else { + staticImports += 1; + } + } + } + + const bundleRows = [...bundleMap.values()].sort((a, b) => b.renderedLength - a.renderedLength); + const hotModules = [...moduleRows].sort((a, b) => b.renderedLength - a.renderedLength); + const totalRendered = moduleRows.reduce((sum, row) => sum + row.renderedLength, 0); + const totalGzip = moduleRows.reduce((sum, row) => sum + row.gzipLength, 0); + const totalBrotli = moduleRows.reduce((sum, row) => sum + row.brotliLength, 0); + + return { + options: data.options ?? {}, + summary: { + bundles: bundleRows.length, + modules: moduleRows.length, + entries: nodeMetas.filter((meta) => meta.isEntry).length, + externals: nodeMetas.filter((meta) => meta.isExternal).length, + staticImports, + dynamicImports, + }, + metrics: { + renderedLength: totalRendered, + gzipLength: totalGzip, + brotliLength: totalBrotli, + }, + hotModules, + }; +} + +function renderVisualizerSummaryTable(before: ReturnType, after: ReturnType) { + const summary = [ + 'bundles', + 'modules', + 'entries', + //'externals', + 'staticImports', + 'dynamicImports', + ] as const; + + const metrics = [ + 'renderedLength', + 'gzipLength', + 'brotliLength', + ] as const; + + return [ + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ``, + ...summary.map((key) => ``), + ...metrics.map((key) => ``), + ``, + ``, + ``, + ...summary.map((key) => ``), + ...metrics.map((key) => ``), + ``, + ``, + ``, + ``, + ...summary.map((key) => ``), + ...metrics.map((key) => ``), + ``, + ``, + ``, + ...summary.map((key) => ``), + ...metrics.map((key) => ``), + ``, + ``, + `
BundlesModulesEntriesImportsSize
StaticDynamicRenderedGzipBrotli
Before${util.formatNumber(before.summary[key])}${util.formatBytes(before.metrics[key])}
After${util.formatNumber(after.summary[key])}${util.formatBytes(after.metrics[key])}
Δ${util.calcAndFormatDeltaNumber(before.summary[key], after.summary[key])}${util.calcAndFormatDeltaBytes(before.metrics[key], after.metrics[key])}
Δ (%)${util.calcAndFormatDeltaPercent(before.summary[key], after.summary[key])}${util.calcAndFormatDeltaPercent(before.metrics[key], after.metrics[key])}
`, + ]; +} + +function getChunkComparisonRows(keys: string[], before: Awaited>, after: Awaited>) { + return keys.map((key) => { + const beforeEntry = before.chunks[key]; + const afterEntry = after.chunks[key]; + const beforeSize = beforeEntry?.size ?? 0; + const afterSize = afterEntry?.size ?? 0; + return { + key, + name: entryDisplayName(beforeEntry ?? afterEntry), + chunkFile: beforeEntry?.file ?? afterEntry?.file, + beforeSize, + afterSize, + changeType: beforeEntry == null ? 'added' : afterEntry == null ? 'removed' : beforeSize !== afterSize ? 'updated' : 'unchanged', + sortSize: Math.max(beforeSize, afterSize), + }; + }); +} + +function summarizeChunkChanges(rows: ReturnType) { + return { + updated: rows.filter((row) => row.changeType === 'updated').length, + added: rows.filter((row) => row.changeType === 'added').length, + removed: rows.filter((row) => row.changeType === 'removed').length, + }; +} + +function formatChunkChangeSummary(label: string, summary: ReturnType) { + return `${label} (${summary.updated} updated, ${summary.added} added, ${summary.removed} removed)`; +} + +function compareChunkComparisonRows(a: ReturnType[number], b: ReturnType[number]) { + return Math.abs(b.afterSize - b.beforeSize) - Math.abs(a.afterSize - a.beforeSize) + || (b.afterSize - b.beforeSize) - (a.afterSize - a.beforeSize) + || b.sortSize - a.sortSize + || a.name.localeCompare(b.name); +} + +function chunkMarkdownTable(rows: ReturnType, total?: { beforeSize: number; afterSize: number }) { + if (rows.length === 0) return '_No data_'; + + const lines = [ + '| Chunk | Before | After | Δ | Δ (%) |', + '| --- | ---: | ---: | ---: | ---: |', + ]; + if (total != null) { + lines.push(`| (total) | ${util.formatBytes(total.beforeSize)} | ${util.formatBytes(total.afterSize)} | ${util.calcAndFormatDeltaBytes(total.beforeSize, total.afterSize)} | ${util.calcAndFormatDeltaPercent(total.beforeSize, total.afterSize).replaceAll('\\%', '\\\\%')} |`); + lines.push('| | | | | |'); + } + for (const row of rows) { + if (row.changeType === 'added') { + lines.push(`|
\`${escapeCell(row.name)}\` \`${escapeCell(row.chunkFile)}\`
| ${util.formatBytes(row.beforeSize)} | ${util.formatBytes(row.afterSize)} | ${util.calcAndFormatDeltaBytes(row.beforeSize, row.afterSize)} | $\\color{orange}{\\text{(+)}}$ |`); + } else if (row.changeType === 'removed') { + lines.push(`|
\`${escapeCell(row.name)}\` \`${escapeCell(row.chunkFile)}\`
| ${util.formatBytes(row.beforeSize)} | ${util.formatBytes(row.afterSize)} | ${util.calcAndFormatDeltaBytes(row.beforeSize, row.afterSize)} | $\\color{green}{\\text{(-)}}$ |`); + } else { + lines.push(`|
\`${escapeCell(row.name)}\` \`${escapeCell(row.chunkFile)}\`
| ${util.formatBytes(row.beforeSize)} | ${util.formatBytes(row.afterSize)} | ${util.calcAndFormatDeltaBytes(row.beforeSize, row.afterSize)} | ${util.calcAndFormatDeltaPercent(row.beforeSize, row.afterSize).replaceAll('\\%', '\\\\%')} |`); + } + } + return lines.join('\n'); +} + +function renderFrontendChunkReport(before: Awaited>, after: Awaited>) { + const commonChunkKeys = Object.keys(before.chunks).filter((key) => after.chunks[key] != null); + const addedChunkKeys = Object.keys(after.chunks).filter((key) => before.chunks[key] == null); + const removedChunkKeys = Object.keys(before.chunks).filter((key) => after.chunks[key] == null); + const allChunkKeys = [ + ...commonChunkKeys, + ...addedChunkKeys, + ...removedChunkKeys, + ]; + //const comparisonRows = getChunkComparisonRows(commonChunkKeys, before, after); + const allComparisonRows = getChunkComparisonRows(allChunkKeys, before, after); + + const changedRows = allComparisonRows.filter((row) => row.changeType !== 'unchanged'); + const diffSummary = summarizeChunkChanges(changedRows); + const diffTotal = { + beforeSize: allComparisonRows.reduce((sum, row) => sum + row.beforeSize, 0), + afterSize: allComparisonRows.reduce((sum, row) => sum + row.afterSize, 0), + }; + const diffRows = changedRows.sort(compareChunkComparisonRows).slice(0, 30); // TODO: 実際に30を超えて切り捨てられたrowがあった場合はその旨をmarkdown内に表示するようにする + + const startupKeys = new Set([ + ...before.startupKeys, + ...after.startupKeys, + ]); + const startupComparisonRows = getChunkComparisonRows([...startupKeys], before, after); + const startupRows = startupComparisonRows.sort(compareChunkComparisonRows); + const startupSummary = summarizeChunkChanges(startupComparisonRows); + const startupTotal = { + beforeSize: startupComparisonRows.reduce((sum, row) => sum + row.beforeSize, 0), + afterSize: startupComparisonRows.reduce((sum, row) => sum + row.afterSize, 0), + }; + + //const largeRows = comparisonRows + // .sort((a, b) => b.sortSize - a.sortSize || a.name.localeCompare(b.name)) + // .slice(0, 30); + + return [ + '
', + `${formatChunkChangeSummary('Chunk size diff', diffSummary)}`, + '', + chunkMarkdownTable(diffRows, diffTotal), + '', + '
', + '', + '
', + `${formatChunkChangeSummary('Startup chunk size', startupSummary)}`, + '', + chunkMarkdownTable(startupRows, startupTotal), + '', + `_Startup chunks are the Vite entry for \`src/_boot_.ts\` and its static imports._`, + '', + '
', + '', + //'
', + //`Largest`, + //'', + //markdownTable(largeRows), + //'', + //'
', + //'', + ].join('\n'); +} + +function renderFrontendBundleReport(before: ReturnType, after: ReturnType) { + const lines = [ + ...renderVisualizerSummaryTable(before, after), + '', + //'
', + //'Top 10', + //'', + ]; + + /* + for (const row of after.hotModules.slice(0, 10)) { + lines.push(`- ${code(row.id)}: ${sharePercent(row.renderedLength, after.metrics.renderedLength)} (${formatBytes(row.renderedLength)})`); + } + + lines.push( + '', + '
', + ); + + lines.push( + '', + '
', + 'Hot Modules (Self Size)', + '', + '| Module | Bundles | Rendered | Share | Gzip | Brotli | Imports | Imported By |', + '|---|---:|---:|---:|---:|---:|---:|---:|', + ); + + for (const row of after.hotModules.slice(0, 15)) { + lines.push(`| ${tableCode(row.id)} | ${row.bundles} | ${formatBytes(row.renderedLength)} | ${sharePercent(row.renderedLength, after.metrics.renderedLength)} | ${formatBytes(row.gzipLength)} | ${formatBytes(row.brotliLength)} | ${row.importedCount} | ${row.importedByCount} |`); + } + + lines.push( + '', + '
', + ); + */ + + return lines.join('\n'); +} + +const args = process.argv.slice(2); +const [beforeDir, afterDir, beforeStatsFile, afterStatsFile, outFile] = args; +const before = await collectReport(beforeDir); +const after = await collectReport(afterDir); +const beforeStats = JSON.parse(await fs.readFile(beforeStatsFile, 'utf8')) as VisualizerReport; +const afterStats = JSON.parse(await fs.readFile(afterStatsFile, 'utf8')) as VisualizerReport; +const beforeVisualizerReport = collectVisualizerReport(beforeStats); +const afterVisualizerReport = collectVisualizerReport(afterStats); +const visualizerArtifactLink = `[Open treemap HTML](${process.env.FRONTEND_BUNDLE_REPORT_ARTIFACT_URL})`; + +const body = [ + marker, + '', + `## 📦 Frontend Bundle Report`, + '', + renderFrontendChunkReport(before, after), + '', + '## Bundle Stats', + '', + renderFrontendBundleReport(beforeVisualizerReport, afterVisualizerReport), + '', + visualizerArtifactLink, +].join('\n'); + +await fs.writeFile(outFile, body); diff --git a/.github/scripts/heap-snapshot-util.mts b/.github/scripts/heap-snapshot-util.mts new file mode 100644 index 0000000000..d6bb6ee3c7 --- /dev/null +++ b/.github/scripts/heap-snapshot-util.mts @@ -0,0 +1,200 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// NOTE: このファイルはworkflow上でバックエンドからも参照されるため、side effectがあってはならない + +import * as util from './utility.mts'; + +export const heapSnapshotCategory = { + total: { label: 'Total', color: 'gray', colorHex: '#888888' }, + code: { label: 'Code', color: 'orange', colorHex: '#f28e2c' }, + strings: { label: 'Strings', color: 'red', colorHex: '#e15759' }, + jsArrays: { label: 'JS arrays', color: 'cyan', colorHex: '#76b7b2' }, + typedArrays: { label: 'Typed arrays', color: 'green', colorHex: '#59a14f' }, + systemObjects: { label: 'System objects', color: 'yellow', colorHex: '#edc949' }, + otherJsObjects: { label: 'Other JS objs', color: 'violet', colorHex: '#af7aa1' }, + otherNonJsObjects: { label: 'Other non-JS objs', color: 'pink', colorHex: '#ff9da7' }, +} as const satisfies Record; + +export type HeapSnapshotData = { + categories: Record; + nodeCounts: Record; + breakdowns?: Record>; +}; + +export type HeapSnapshotReport = { + summary: HeapSnapshotData; + samples: { + round: number; + data: HeapSnapshotData; + }[]; +}; + +function getHeapSnapshotCategoryValue(report: HeapSnapshotReport, category: keyof typeof heapSnapshotCategory) { + return report.summary.categories[category]; +} + +export function renderHeapSnapshotTable(base: HeapSnapshotReport, head: HeapSnapshotReport) { + const lines = [ + '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + const baseTotal = getHeapSnapshotCategoryValue(base, 'total'); + const headTotal = getHeapSnapshotCategoryValue(head, 'total'); + + function getHeapSnapshotSampleSpread(report: HeapSnapshotReport, category: keyof typeof heapSnapshotCategory) { + const values = report.samples + .map(sample => sample.data.categories[category]) + .filter(value => Number.isFinite(value)) as number[]; + if (values.length < 2) throw new Error(`Not enough samples for category ${category}`); + + const center = util.median(values); + return util.median(values.map(value => Math.abs(value - center))); + } + + for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) { + const baseValue = getHeapSnapshotCategoryValue(base, category); + const headValue = getHeapSnapshotCategoryValue(head, category); + const baseSpread = getHeapSnapshotSampleSpread(base, category); + const headSpread = getHeapSnapshotSampleSpread(head, category); + const summary = util.pairedDeltaSummary(base.samples, head.samples, (sample) => sample.data.categories[category]); + const percent = summary.median * 100 / baseValue; + + if (category === 'total') { + const deltaMedian = `${util.formatDeltaBytes(summary.median)}
${util.formatDeltaPercent(percent).replaceAll('\\%', '\\\\%')}`; + const baseText = `${util.formatBytes(baseValue)}
± ${util.formatBytes(baseSpread)}`; + const headText = `${util.formatBytes(headValue)}
± ${util.formatBytes(headSpread)}`; + const metricText = `$\\color{${heapSnapshotCategory[category].color}}{\\rule{8pt}{8pt}}$ **${heapSnapshotCategory[category].label}**`; + lines.push(`| ${metricText} | ${baseText} | ${headText} | ${deltaMedian} | ${util.formatBytes(summary.mad)} | ${util.formatDeltaBytes(summary.min)} | ${util.formatDeltaBytes(summary.max)} |`); + lines.push('| | | | | | | |'); + } else { + const deltaMedian = util.formatDeltaBytes(summary.median); + const baseText = util.formatBytes(baseValue); + const headText = util.formatBytes(headValue); + const basePercent = util.formatPercent((baseValue * 100) / baseTotal); + const headPercent = util.formatPercent((headValue * 100) / headTotal); + const metricText = `
$\\color{${heapSnapshotCategory[category].color}}{\\rule{8pt}{8pt}}$ **${heapSnapshotCategory[category].label}**${basePercent} → ${headPercent}
`; + lines.push(`| ${metricText} | ${baseText} | ${headText} | ${deltaMedian} | ${util.formatBytes(summary.mad)} | ${util.formatDeltaBytes(summary.min)} | ${util.formatDeltaBytes(summary.max)} |`); + } + } + + if (lines.length === 2) return null; + return lines.join('\n'); +} + +const heapSnapshotSankeyChildMinRatio = 0.3; +const heapSnapshotSankeyParentMinPercent = 10; + +function escapeCsvValue(value: string) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +export function renderHeapSnapshotSankey(report: HeapSnapshotReport, title: string) { + const total = getHeapSnapshotCategoryValue(report, 'total'); + if (total == null || total <= 0) return null; + + function getHeapSnapshotBreakdownEntries(category: keyof typeof heapSnapshotCategory) { + const breakdown = report.summary.breakdowns?.[category]; + if (breakdown == null || typeof breakdown !== 'object') return []; + + return Object.entries(breakdown) + .filter(([, value]) => Number.isFinite(value) && value > 0) + .toSorted((a, b) => b[1] - a[1]); + } + + function formatHeapSnapshotSankeyChildLabel(label: string) { + return String(label).replace(/^[^:]+:\s*/, ''); + } + + function formatSankeyPercentValue(value: number) { + const rounded = Math.round(value * 100) / 100; + if (rounded === 0 && value > 0) return '0.01'; + if (Number.isInteger(rounded)) return String(rounded); + return rounded.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); + } + + const categories = (Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) + .filter(category => category !== 'total') + .map(category => { + const value = getHeapSnapshotCategoryValue(report, category); + if (value == null || value <= 0) return null; + const breakdownEntries = getHeapSnapshotBreakdownEntries(category); + const breakdownTotal = breakdownEntries.reduce((sum, [, childValue]) => sum + childValue, 0); + const percent = (value * 100) / total; + const childEntries = []; + let otherPercent = 0; + + if (breakdownTotal > 0 && percent > heapSnapshotSankeyParentMinPercent) { + for (const [childName, childValue] of breakdownEntries) { + const childRatio = childValue / breakdownTotal; + const childPercent = percent * childRatio; + if (childRatio >= heapSnapshotSankeyChildMinRatio) { + childEntries.push([formatHeapSnapshotSankeyChildLabel(childName), childPercent]); + } else { + otherPercent += childPercent; + } + } + + if (childEntries.length > 0 && otherPercent > 0) { + childEntries.push(['Other', otherPercent]); + } + } + + return { + category, + percent, + childEntries, + }; + }) + .filter(value => value != null); + + if (categories.length === 0) return null; + + const nodeColors = { + [title]: heapSnapshotCategory.total.colorHex, + } as Record; + for (const { category, childEntries } of categories) { + const categoryColor = heapSnapshotCategory[category].colorHex; + nodeColors[category] = categoryColor; + + for (const [childName] of childEntries) { + nodeColors[childName] = categoryColor; + } + } + + const lines = [ + `
${title} heap snapshot composition`, + '', + '```mermaid', + `%%{init: ${JSON.stringify({ + sankey: { + showValues: false, + linkColor: 'target', + labelStyle: 'outlined', + nodeAlignment: 'center', + nodePadding: 10, + nodeColors: { + ...nodeColors, + 'Other': '#888888', + }, + }, + })}}%%`, + 'sankey-beta', + ]; + + for (const { category, percent, childEntries } of categories) { + lines.push(`${escapeCsvValue(title)},${escapeCsvValue(heapSnapshotCategory[category].label)},${formatSankeyPercentValue(percent)}`); + + for (const [childName, childPercent] of childEntries) { + lines.push(`${escapeCsvValue(heapSnapshotCategory[category].label)},${escapeCsvValue(childName)},${formatSankeyPercentValue(childPercent)}`); + } + } + + lines.push('```'); + lines.push(''); + lines.push('
'); + + return lines.join('\n'); +} diff --git a/.github/scripts/measure-backend-memory-comparison.mts b/.github/scripts/measure-backend-memory-comparison.mts new file mode 100644 index 0000000000..99ba4f64ae --- /dev/null +++ b/.github/scripts/measure-backend-memory-comparison.mts @@ -0,0 +1,259 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { createRequire } from 'node:module'; +import { writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import * as util from './utility.mts'; +import * as heapSnapshotUtil from './heap-snapshot-util.mts'; +import type { MemoryReportRaw } from '../../packages/backend/scripts/measure-memory.mts'; + +const phases = ['afterGc'] as const; + +export type MemoryReport = { + timestamp: string; + sampleCount: any; + aggregation: string; + measurement: { + startupTimeoutMs: any; + memorySettleTimeMs: any; + ipcTimeoutMs: any; + requestCount: any; + heapSnapshot: { + enabled: any; + timeoutMs: any; + breakdownTopN: any; + }; + }; + summary: Record; + heapSnapshot?: heapSnapshotUtil.HeapSnapshotData; + }>; + samples: (MemoryReportRaw['samples'][number] & { + round: number; + })[]; +}; + +const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2); + +const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = util.readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1); + +async function resetState(repoDir: string) { + const require = createRequire(join(repoDir, 'packages/backend/package.json')); + const pg = require('pg'); + const Redis = require('ioredis'); + + const postgres = new pg.Client({ + host: '127.0.0.1', + port: 54312, + database: 'postgres', + user: 'postgres', + }); + + await postgres.connect(); + try { + await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)'); + await postgres.query('CREATE DATABASE "test-misskey"'); + } finally { + await postgres.end(); + } + + const redis = new Redis({ host: '127.0.0.1', port: 56312 }); + try { + await redis.flushall(); + } finally { + redis.disconnect(); + } +} + +function summarizeHeapSnapshotBreakdowns(samples: MemoryReport['samples'], phase: typeof phases[number]) { + const breakdowns = {} as Record>; + + for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) { + if (category === 'total') continue; + + const childKeys = new Set(); + for (const sample of samples) { + for (const childKey of Object.keys(sample.phases[phase].heapSnapshot?.breakdowns?.[category] ?? {})) { + childKeys.add(childKey); + } + } + + const categoryBreakdown = {} as Record; + for (const childKey of childKeys) { + const values = samples + .map(sample => sample.phases[phase].heapSnapshot?.breakdowns?.[category]?.[childKey]) + .filter(value => Number.isFinite(value)) as number[]; + + if (values.length > 0) categoryBreakdown[childKey] = util.median(values); + } + + if (Object.keys(categoryBreakdown).length > 0) { + breakdowns[category] = collapseHeapSnapshotBreakdown(categoryBreakdown); + } + } + + return breakdowns; +} + +function collapseHeapSnapshotBreakdown(breakdown: Record) { + const entries = Object.entries(breakdown) + .filter(([, value]) => value > 0) + .toSorted((a, b) => b[1] - a[1]); + + const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N); + const otherValue = entries + .slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N) + .reduce((sum, [, value]) => sum + value, 0); + + const collapsed = Object.fromEntries(topEntries); + if (otherValue > 0) collapsed.Other = otherValue; + return collapsed; +} + +function summarizeSamples(samples: MemoryReport['samples']) { + const summary = {} as MemoryReport['summary']; + + for (const phase of phases) { + summary[phase] = { + memoryUsage: {}, + }; + + const metricKeys = new Set(); + for (const sample of samples) { + for (const key of Object.keys(sample.phases[phase].memoryUsage)) { + metricKeys.add(key); + } + } + + for (const key of metricKeys) { + const values = samples.map(sample => sample.phases[phase].memoryUsage[key]); + summary[phase].memoryUsage[key] = util.median(values); + } + + const heapSnapshotCategoryValues = {} as Record; + for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) { + const values = samples + .map(sample => sample.phases[phase].heapSnapshot?.categories?.[category]) + .filter(value => Number.isFinite(value)) as number[]; + + if (values.length > 0) heapSnapshotCategoryValues[category] = util.median(values); + } + + const heapSnapshotNodeCountValues = {} as Record; + for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) { + const values = samples + .map(sample => sample.phases[phase].heapSnapshot?.nodeCounts?.[category]) + .filter(value => Number.isFinite(value)) as number[]; + + if (values.length > 0) heapSnapshotNodeCountValues[category] = util.median(values); + } + + if (Object.keys(heapSnapshotCategoryValues).length > 0) { + const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(samples, phase); + + summary[phase].heapSnapshot = { + categories: heapSnapshotCategoryValues, + nodeCounts: heapSnapshotNodeCountValues, + ...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}), + }; + } + } + + return summary; +} + +async function measureRepo(label: string, repoDir: string, round: number) { + process.stderr.write(`[${label}] Resetting database and Redis\n`); + await resetState(repoDir); + + process.stderr.write(`[${label}] Running migrations\n`); + await util.run('pnpm', ['--filter', 'backend', 'migrate'], { + cwd: repoDir, + env: process.env, + logStdout: true, + }); + + process.stderr.write(`[${label}] Measuring memory\n`); + const measureEnv = { + ...process.env, + MK_MEMORY_SAMPLE_COUNT: '1', + } as NodeJS.ProcessEnv; + if (round <= 0) measureEnv.MK_MEMORY_HEAP_SNAPSHOT = '0'; + + const stdout = await util.run('node', ['packages/backend/scripts/measure-memory.mts'], { + cwd: repoDir, + env: measureEnv, + }); + + const report = JSON.parse(stdout) as MemoryReportRaw; + const sample = report.samples[0]; + + return sample; +} + +async function main() { + const baseDir = resolve(baseDirArg); + const headDir = resolve(headDirArg); + const baseOutput = resolve(baseOutputArg); + const headOutput = resolve(headOutputArg); + const rounds = util.readIntegerEnv('MK_MEMORY_COMPARE_ROUNDS', 5, 1); + const warmupRounds = util.readIntegerEnv('MK_MEMORY_COMPARE_WARMUP_ROUNDS', 1, 0); + const startedAt = new Date().toISOString(); + + const reports = { + base: { + dir: baseDir, + samples: [] as MemoryReport['samples'], + }, + head: { + dir: headDir, + samples: [] as MemoryReport['samples'], + }, + }; + + for (let round = 1; round <= warmupRounds; round++) { + process.stderr.write(`Starting warmup round ${round}/${warmupRounds}\n`); + for (const label of ['base', 'head'] as const) { + await measureRepo(label, reports[label].dir, -round); + } + } + + for (let round = 1; round <= rounds; round++) { + const order = round % 2 === 1 ? ['base', 'head'] as const : ['head', 'base'] as const; + process.stderr.write(`Starting measurement round ${round}/${rounds}: ${order.join(' -> ')}\n`); + + for (const [orderIndex, label] of order.entries()) { + const sample = await measureRepo(label, reports[label].dir, round); + reports[label].samples.push({ + ...sample, + round, + }); + } + } + + for (const label of ['base', 'head'] as const) { + const report = { + timestamp: new Date().toISOString(), + sampleCount: reports[label].samples.length, + aggregation: 'median', + comparison: { + strategy: 'interleaved-pairs', + rounds, + warmupRounds, + startedAt, + }, + summary: summarizeSamples(reports[label].samples), + samples: reports[label].samples, + }; + + await writeFile(label === 'base' ? baseOutput : headOutput, `${JSON.stringify(report, null, 2)}\n`); + } +} + +main().catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/.github/scripts/utility.mts b/.github/scripts/utility.mts new file mode 100644 index 0000000000..bb3b1d9401 --- /dev/null +++ b/.github/scripts/utility.mts @@ -0,0 +1,204 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// NOTE: このファイルはworkflow上でバックエンドからも参照されるため、side effectがあってはならない + +import { spawn } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; + +export function median(values: number[]) { + const sorted = values.toSorted((a, b) => a - b); + const center = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) return sorted[center]; + return Math.round((sorted[center - 1] + sorted[center]) / 2); +} + +export function mad(values: number[]) { + if (values.length < 2) throw new Error('Not enough samples to calculate MAD'); + + const center = median(values); + return median(values.map(value => Math.abs(value - center))); +} + +function getSamplesByRound(samples: T) { + const samplesByRound = new Map(); + for (const sample of samples) { + if (sample.round <= 0) continue; + samplesByRound.set(sample.round, sample); + } + return samplesByRound; +} + +export function pairedDeltaSummary(baseSamples: T, headSamples: T, getValue: (sample: T[number]) => number | null) { + const baseSamplesByRound = getSamplesByRound(baseSamples); + const headSamplesByRound = getSamplesByRound(headSamples); + const values = []; + + for (const [round, baseSample] of baseSamplesByRound) { + const headSample = headSamplesByRound.get(round); + if (headSample == null) continue; + + const baseValue = getValue(baseSample); + const headValue = getValue(headSample); + if (baseValue == null || headValue == null) continue; + + values.push(headValue - baseValue); + } + + return { + median: median(values), + mad: mad(values), + min: Math.min(...values), + max: Math.max(...values), + samples: values.length, + }; +} + +export function normalizePath(filePath: string) { + return filePath.split(path.sep).join('/'); +} + +export async function fileExists(filePath: string) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +export async function fileSize(filePath: string) { + const stat = await fs.stat(filePath); + return stat.size; +} + +export async function* traverseDirectory(dir: string): AsyncGenerator { + for (const entry of await fs.readdir(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + yield* traverseDirectory(fullPath); + } else if (entry.isFile()) { + yield fullPath; + } + } +} + +export function escapeLatex(text: string) { + return text + .replaceAll('\\', '\\\\') + .replaceAll('{', '\\{') + .replaceAll('}', '\\}') + .replaceAll('%', '\\%'); +} + +export function formatColoredDelta(text: string, delta: number) { + if (delta === 0) return text; + const color = delta > 0 ? 'orange' : 'green'; + const sign = delta > 0 ? '+' : '-'; + return `$\\color{${color}}{\\text{${sign}${escapeLatex(text)}}}$`; +} + +const numberFormatter = new Intl.NumberFormat('en-US', { + maximumFractionDigits: 1, +}); + +export function formatNumber(value: number) { + return numberFormatter.format(value); +} + +export function formatBytes(value: number) { + if (value === 0) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB']; + let unitIndex = 0; + let size = value; + while (size >= 1024 && unitIndex < units.length - 1) { + size /= 1024; + unitIndex += 1; + } + + const maximumFractionDigits = size >= 10 || unitIndex === 0 ? 0 : 1; + return `${numberFormatter.format(Number(size.toFixed(maximumFractionDigits)))} ${units[unitIndex]}`; +} + +export function calcAndFormatDeltaNumber(before: number, after: number) { + if (before == null || after == null) return '-'; + const delta = after - before; + return formatColoredDelta(formatNumber(Math.abs(delta)), delta); +} + +export function formatDeltaBytes(deltaBytes: number) { + return formatColoredDelta(formatBytes(Math.abs(deltaBytes)), deltaBytes); +} + +export function calcAndFormatDeltaBytes(before: number, after: number) { + if (before == null || after == null) return '-'; + const delta = after - before; + return formatDeltaBytes(delta); +} + +export function formatPercent(value: number) { + return `${formatNumber(value)}%`; +} + +export function formatDeltaPercent(deltaPercent: number) { + if (deltaPercent === 0) return '0%'; + return formatColoredDelta(formatPercent(Math.abs(deltaPercent)), deltaPercent); +} + +export function calcAndFormatDeltaPercent(before: number, after: number) { + if (before == null || before === 0 || after == null || after === 0) return '-'; + const delta = after - before; + return formatDeltaPercent(delta / before * 100); +} + +export function commandName(command: string) { + if (process.platform !== 'win32') return command; + if (command === 'pnpm') return 'pnpm.cmd'; + return command; +} + +export function readIntegerEnv(name: string, defaultValue: number, min: number) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`); + + const value = Number(rawValue); + if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`); + return value; +} + +export function run(command: string, args: string[], options: { cwd?: string; env?: NodeJS.ProcessEnv; logStdout?: boolean } = {}) { + return new Promise((resolvePromise, reject) => { + const child = spawn(commandName(command), args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', data => { + stdout += data; + if (options.logStdout) process.stderr.write(data); + }); + + child.stderr.on('data', data => { + stderr += data; + process.stderr.write(data); + }); + + child.on('error', reject); + + child.on('close', code => { + if (code === 0) { + resolvePromise(stdout); + } else { + reject(new Error(`${command} ${args.join(' ')} failed with exit code ${code}\n${stderr}`)); + } + }); + }); +} diff --git a/.github/workflows/frontend-bundle-report-comment.yml b/.github/workflows/frontend-bundle-report-comment.yml index 29bc00a5f4..b93f8a320f 100644 --- a/.github/workflows/frontend-bundle-report-comment.yml +++ b/.github/workflows/frontend-bundle-report-comment.yml @@ -25,8 +25,8 @@ on: - pnpm-lock.yaml - pnpm-workspace.yaml - .node-version - - .github/scripts/frontend-js-size.mjs - - .github/scripts/frontend-bundle-visualizer-report.mjs + - .github/scripts/utility.mts + - .github/scripts/frontend-js-size.mts - .github/workflows/frontend-bundle-report.yml - .github/workflows/frontend-bundle-report-comment.yml @@ -104,6 +104,9 @@ jobs: }); const report = artifacts.find((artifact) => artifact.name === artifactName && !artifact.expired); if (report) return { done: true, run }; + + core.info(`Frontend bundle report run ${run.id} did not produce ${artifactName}.`); + return { done: true, run: null }; } return { done: false, run: null }; @@ -126,8 +129,30 @@ jobs: core.warning(`Timed out waiting for ${artifactName} from ${workflow_id} for ${headSha}.`); - - name: Download bundle report from workflow_run + - name: Find bundle report artifact if: github.event_name == 'workflow_run' + id: find-report-artifact + uses: actions/github-script@v9 + with: + script: | + const artifactName = 'frontend-bundle-report'; + const { owner, repo } = context.repo; + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner, + repo, + run_id: context.payload.workflow_run.id, + per_page: 100, + }); + const report = artifacts.find((artifact) => artifact.name === artifactName && !artifact.expired); + if (report) { + core.setOutput('exists', 'true'); + } else { + core.info(`Workflow run ${context.payload.workflow_run.id} did not produce ${artifactName}.`); + core.setOutput('exists', 'false'); + } + + - name: Download bundle report from workflow_run + if: github.event_name == 'workflow_run' && steps.find-report-artifact.outputs.exists == 'true' uses: actions/download-artifact@v8 with: name: frontend-bundle-report @@ -147,7 +172,7 @@ jobs: run-id: ${{ steps.find-report-run.outputs.run-id }} - name: Comment on pull request - if: github.event_name == 'workflow_run' || steps.find-report-run.outputs.run-id != '' + if: (github.event_name == 'workflow_run' && steps.find-report-artifact.outputs.exists == 'true') || steps.find-report-run.outputs.run-id != '' uses: actions/github-script@v9 with: github-token: ${{ secrets.FRONTEND_BUNDLE_REPORT_COMMENT_TOKEN || secrets.FRONTEND_JS_SIZE_COMMENT_TOKEN || secrets.FRONTEND_BUNDLE_VISUALIZER_COMMENT_TOKEN || github.token }} @@ -160,30 +185,25 @@ jobs: const reportMarkers = [jsSizeMarker, visualizerMarker]; const reportDir = path.join(process.env.RUNNER_TEMP, 'frontend-bundle-report'); const jsSizeReportPath = path.join(reportDir, 'frontend-js-size-report.md'); - const visualizerReportPath = path.join(reportDir, 'frontend-bundle-visualizer-report.md'); const prNumberPath = path.join(reportDir, 'pr-number.txt'); const headShaPath = path.join(reportDir, 'head-sha.txt'); const workflowRun = context.payload.workflow_run; const pullRequest = context.payload.pull_request; - const headSha = workflowRun?.head_sha ?? pullRequest?.head?.sha ?? null; + const eventHeadSha = workflowRun?.head_sha ?? pullRequest?.head?.sha ?? null; const { owner, repo } = context.repo; if (!fs.existsSync(jsSizeReportPath)) { core.setFailed('The frontend bundle report artifact does not contain frontend-js-size-report.md.'); return; } - if (!fs.existsSync(visualizerReportPath)) { - core.setFailed('The frontend bundle report artifact does not contain frontend-bundle-visualizer-report.md.'); - return; - } const artifactHeadSha = fs.existsSync(headShaPath) ? fs.readFileSync(headShaPath, 'utf8').trim() : null; - if (headSha != null && artifactHeadSha != null && artifactHeadSha !== headSha) { - core.setFailed(`The artifact head SHA (${artifactHeadSha}) does not match the workflow head SHA (${headSha}).`); - return; + if (eventHeadSha != null && artifactHeadSha != null && artifactHeadSha !== eventHeadSha) { + core.info(`The artifact head SHA (${artifactHeadSha}) differs from the event head SHA (${eventHeadSha}). Using artifact metadata for PR validation.`); } + const reportHeadSha = artifactHeadSha ?? eventHeadSha; const artifactPrNumber = fs.existsSync(prNumberPath) ? Number(fs.readFileSync(prNumberPath, 'utf8').trim()) @@ -203,25 +223,29 @@ jobs: } } - const pullRequestsForCommit = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, { - owner, - repo, - commit_sha: headSha, - per_page: 100, - }); - for (const pullRequest of pullRequestsForCommit) { - associatedPullRequests.set(pullRequest.number, pullRequest); + if (reportHeadSha != null) { + const pullRequestsForCommit = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, { + owner, + repo, + commit_sha: reportHeadSha, + per_page: 100, + }); + for (const pullRequest of pullRequestsForCommit) { + associatedPullRequests.set(pullRequest.number, pullRequest); + } } if (Number.isInteger(artifactPrNumber) && associatedPullRequests.has(artifactPrNumber)) { issue_number = artifactPrNumber; + } else if (Number.isInteger(artifactPrNumber) && associatedPullRequests.size === 0) { + issue_number = artifactPrNumber; } else if (!Number.isInteger(artifactPrNumber) && associatedPullRequests.size === 1) { issue_number = [...associatedPullRequests.keys()][0]; } else if (Number.isInteger(artifactPrNumber)) { - core.setFailed(`The artifact pull request number (${artifactPrNumber}) is not associated with ${headSha}.`); + core.setFailed(`The artifact pull request number (${artifactPrNumber}) is not associated with ${reportHeadSha}.`); return; } else { - core.setFailed(`Could not determine the pull request associated with ${headSha}.`); + core.setFailed(`Could not determine the pull request associated with ${reportHeadSha}.`); return; } } else { @@ -229,16 +253,23 @@ jobs: return; } + const currentPullRequest = await github.rest.pulls.get({ + owner, + repo, + pull_number: issue_number, + }); + const currentHeadSha = currentPullRequest.data.head?.sha; + if (reportHeadSha != null && currentHeadSha != null && reportHeadSha !== currentHeadSha) { + core.info(`The report head SHA (${reportHeadSha}) is not the current pull request head SHA (${currentHeadSha}). Skipping stale frontend bundle report.`); + return; + } + const jsSizeReport = fs.readFileSync(jsSizeReportPath, 'utf8').trim(); if (!jsSizeReport.includes(jsSizeMarker)) { core.setFailed('The frontend JS size report is missing the expected marker.'); return; } - const visualizerReport = fs.readFileSync(visualizerReportPath, 'utf8').trim(); - let body = [ - jsSizeReport, - visualizerReport, - ].join('\n\n') + '\n'; + let body = `${jsSizeReport}\n`; const maxCommentLength = 65_000; if (body.length > maxCommentLength) { diff --git a/.github/workflows/frontend-bundle-report.yml b/.github/workflows/frontend-bundle-report.yml index 7b0f70451e..7625f4696d 100644 --- a/.github/workflows/frontend-bundle-report.yml +++ b/.github/workflows/frontend-bundle-report.yml @@ -20,8 +20,8 @@ on: - pnpm-lock.yaml - pnpm-workspace.yaml - .node-version - - .github/scripts/frontend-js-size.mjs - - .github/scripts/frontend-bundle-visualizer-report.mjs + - .github/scripts/utility.mts + - .github/scripts/frontend-js-size.mts - .github/workflows/frontend-bundle-report.yml - .github/workflows/frontend-bundle-report-comment.yml @@ -56,21 +56,25 @@ jobs: path: after submodules: true - - name: Backport visualizer tooling to base if needed + - name: Check base visualizer support + id: check-base-visualizer shell: bash run: | - if ! grep -q 'FRONTEND_BUNDLE_VISUALIZER' before/packages/frontend/vite.config.ts; then - cp after/packages/frontend/package.json before/packages/frontend/package.json - cp after/packages/frontend/vite.config.ts before/packages/frontend/vite.config.ts - cp after/pnpm-lock.yaml before/pnpm-lock.yaml + if grep -q 'FRONTEND_BUNDLE_VISUALIZER' before/packages/frontend/vite.config.ts; then + echo 'supported=true' >> "$GITHUB_OUTPUT" + else + echo 'supported=false' >> "$GITHUB_OUTPUT" + echo 'Base commit does not support frontend bundle visualizer. Skipping frontend bundle report.' >> "$GITHUB_STEP_SUMMARY" fi - name: Setup pnpm + if: steps.check-base-visualizer.outputs.supported == 'true' uses: pnpm/action-setup@v6.0.3 with: package_json_file: after/package.json - name: Setup Node.js + if: steps.check-base-visualizer.outputs.supported == 'true' uses: actions/setup-node@v6.4.0 with: node-version-file: after/.node-version @@ -80,67 +84,84 @@ jobs: after/pnpm-lock.yaml - name: Install dependencies for base + if: steps.check-base-visualizer.outputs.supported == 'true' working-directory: before run: pnpm i --frozen-lockfile - name: Build frontend dependencies for base + if: steps.check-base-visualizer.outputs.supported == 'true' working-directory: before run: pnpm --filter "frontend^..." run build - name: Prepare report output + if: steps.check-base-visualizer.outputs.supported == 'true' run: mkdir -p "$RUNNER_TEMP/frontend-bundle-report" - name: Build frontend report for base + if: steps.check-base-visualizer.outputs.supported == 'true' working-directory: before env: FRONTEND_BUNDLE_VISUALIZER: 'true' - FRONTEND_BUNDLE_VISUALIZER_TEMPLATE: raw-data FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/before-stats.json run: pnpm --filter frontend run build - name: Install dependencies for pull request + if: steps.check-base-visualizer.outputs.supported == 'true' working-directory: after run: pnpm i --frozen-lockfile - name: Build frontend dependencies for pull request + if: steps.check-base-visualizer.outputs.supported == 'true' working-directory: after run: pnpm --filter "frontend^..." run build - name: Build frontend report for pull request + if: steps.check-base-visualizer.outputs.supported == 'true' working-directory: after env: FRONTEND_BUNDLE_VISUALIZER: 'true' - FRONTEND_BUNDLE_VISUALIZER_TEMPLATE: raw-data FRONTEND_BUNDLE_VISUALIZER_FILE: ${{ runner.temp }}/frontend-bundle-report/after-stats.json + FRONTEND_BUNDLE_VISUALIZER_HTML_FILE: ${{ runner.temp }}/frontend-bundle-report/frontend-bundle-visualizer.html run: pnpm --filter frontend run build + - name: Upload bundle visualizer + if: steps.check-base-visualizer.outputs.supported == 'true' + id: upload-bundle-visualizer + uses: actions/upload-artifact@v7 + with: + name: frontend-bundle-visualizer + path: ${{ runner.temp }}/frontend-bundle-report/frontend-bundle-visualizer.html + if-no-files-found: error + archive: false + retention-days: 7 + - name: Generate report markdown + if: steps.check-base-visualizer.outputs.supported == 'true' shell: bash env: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} + FRONTEND_BUNDLE_REPORT_ARTIFACT_URL: ${{ steps.upload-bundle-visualizer.outputs.artifact-url }} run: | REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report" - node after/.github/scripts/frontend-js-size.mjs before after "$REPORT_DIR/frontend-js-size-report.md" - node after/.github/scripts/frontend-bundle-visualizer-report.mjs "$REPORT_DIR/before-stats.json" "$REPORT_DIR/after-stats.json" "$REPORT_DIR/frontend-bundle-visualizer-report.md" + node after/.github/scripts/frontend-js-size.mts before after "$REPORT_DIR/before-stats.json" "$REPORT_DIR/after-stats.json" "$REPORT_DIR/frontend-js-size-report.md" printf '%s\n' "$PR_NUMBER" > "$REPORT_DIR/pr-number.txt" printf '%s\n' "$BASE_SHA" > "$REPORT_DIR/base-sha.txt" printf '%s\n' "$HEAD_SHA" > "$REPORT_DIR/head-sha.txt" printf '%s\n' "${{ github.event.pull_request.html_url }}" > "$REPORT_DIR/pr-url.txt" - name: Check report + if: steps.check-base-visualizer.outputs.supported == 'true' run: | REPORT_DIR="$RUNNER_TEMP/frontend-bundle-report" test -s "$REPORT_DIR/before-stats.json" test -s "$REPORT_DIR/after-stats.json" test -s "$REPORT_DIR/frontend-js-size-report.md" - test -s "$REPORT_DIR/frontend-bundle-visualizer-report.md" cat "$REPORT_DIR/frontend-js-size-report.md" >> "$GITHUB_STEP_SUMMARY" - printf '\n\n' >> "$GITHUB_STEP_SUMMARY" - cat "$REPORT_DIR/frontend-bundle-visualizer-report.md" >> "$GITHUB_STEP_SUMMARY" - name: Upload bundle report + if: steps.check-base-visualizer.outputs.supported == 'true' uses: actions/upload-artifact@v7 with: name: frontend-bundle-report diff --git a/.github/workflows/get-backend-memory.yml b/.github/workflows/get-backend-memory.yml index aa4bf5e3a4..d25e3db1d9 100644 --- a/.github/workflows/get-backend-memory.yml +++ b/.github/workflows/get-backend-memory.yml @@ -9,7 +9,14 @@ on: paths: - packages/backend/** - packages/misskey-js/** + - .github/scripts/utility.mts + - .github/scripts/backend-memory-report.mts + - .github/scripts/measure-backend-memory-comparison.mts + - .github/scripts/backend-js-footprint.mjs + - .github/scripts/backend-js-footprint-loader.mjs + - .github/scripts/backend-js-footprint-require.cjs - .github/workflows/get-backend-memory.yml + - .github/workflows/report-backend-memory.yml jobs: get-memory-usage: @@ -17,15 +24,6 @@ jobs: permissions: contents: read - strategy: - matrix: - memory-json-name: [memory-base.json, memory-head.json] - include: - - memory-json-name: memory-base.json - ref: ${{ github.base_ref }} - - memory-json-name: memory-head.json - ref: refs/pull/${{ github.event.number }}/merge - services: postgres: image: postgres:18 @@ -40,37 +38,77 @@ jobs: - 56312:6379 steps: - - uses: actions/checkout@v6.0.2 + - name: Checkout base + uses: actions/checkout@v6.0.2 with: - ref: ${{ matrix.ref }} + ref: ${{ github.base_ref }} + path: base + submodules: true + - name: Checkout head + uses: actions/checkout@v6.0.2 + with: + ref: refs/pull/${{ github.event.number }}/merge + path: head submodules: true - name: Setup pnpm uses: pnpm/action-setup@v6.0.3 + with: + package_json_file: head/package.json - name: Use Node.js uses: actions/setup-node@v6.4.0 with: - node-version-file: '.node-version' + node-version-file: 'head/.node-version' cache: 'pnpm' - - run: pnpm i --frozen-lockfile - - name: Check pnpm-lock.yaml + cache-dependency-path: | + base/pnpm-lock.yaml + head/pnpm-lock.yaml + - name: Install base dependencies + working-directory: base + run: pnpm i --frozen-lockfile + - name: Check base pnpm-lock.yaml + working-directory: base run: git diff --exit-code pnpm-lock.yaml - - name: Copy Configure - run: cp .github/misskey/test.yml .config/default.yml - - name: Compile Configure - run: pnpm compile-config - - name: Build - run: pnpm build - - name: Run migrations - run: pnpm --filter backend migrate - - name: Measure memory usage + - name: Configure base + working-directory: base run: | - # Start the server and measure memory usage - node packages/backend/scripts/measure-memory.mjs > ${{ matrix.memory-json-name }} + cp .github/misskey/test.yml .config/default.yml + pnpm compile-config + - name: Build base + working-directory: base + run: pnpm build + - name: Install head dependencies + working-directory: head + run: pnpm i --frozen-lockfile + - name: Check head pnpm-lock.yaml + working-directory: head + run: git diff --exit-code pnpm-lock.yaml + - name: Configure head + working-directory: head + run: | + cp .github/misskey/test.yml .config/default.yml + pnpm compile-config + - name: Build head + working-directory: head + run: pnpm build + - name: Measure backend memory usage + env: + MK_MEMORY_COMPARE_ROUNDS: 5 + MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1 + MK_MEMORY_HEAP_SNAPSHOT: 1 + run: node head/.github/scripts/measure-backend-memory-comparison.mts base head memory-base.json memory-head.json + - name: Measure backend loaded JS footprint + run: | + node head/.github/scripts/backend-js-footprint.mjs base js-footprint-base.json + node head/.github/scripts/backend-js-footprint.mjs head js-footprint-head.json - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: memory-artifact-${{ matrix.memory-json-name }} - path: ${{ matrix.memory-json-name }} + name: memory-artifact-results + path: | + memory-base.json + memory-head.json + js-footprint-base.json + js-footprint-head.json save-pr-number: runs-on: ubuntu-latest diff --git a/.github/workflows/report-backend-memory.yml b/.github/workflows/report-backend-memory.yml index 07850bf2e4..157ad3b68f 100644 --- a/.github/workflows/report-backend-memory.yml +++ b/.github/workflows/report-backend-memory.yml @@ -11,9 +11,14 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' }} permissions: + actions: read + contents: read pull-requests: write steps: + - name: Checkout + uses: actions/checkout@v6.0.2 + - name: Download artifact uses: actions/github-script@v9 with: @@ -48,120 +53,13 @@ jobs: run: cat ./artifacts/memory-base.json - name: Output head run: cat ./artifacts/memory-head.json - - name: Compare memory usage - id: compare - run: | - BASE_MEMORY=$(cat ./artifacts/memory-base.json) - HEAD_MEMORY=$(cat ./artifacts/memory-head.json) - - variation() { - calc() { - BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0") - HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0") - - DIFF=$((HEAD - BASE)) - if [ "$BASE" -gt 0 ]; then - DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc) - else - DIFF_PERCENT=0 - fi - - # Convert KB to MB for readability - BASE_MB=$(echo "scale=2; $BASE / 1024" | bc) - HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc) - DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc) - - JSON=$(jq -c -n \ - --argjson base "$BASE_MB" \ - --argjson head "$HEAD_MB" \ - --argjson diff "$DIFF_MB" \ - --argjson diff_percent "$DIFF_PERCENT" \ - '{base: $base, head: $head, diff: $diff, diff_percent: $diff_percent}') - - echo "$JSON" - } - - JSON=$(jq -c -n \ - --argjson VmRSS "$(calc $1 VmRSS)" \ - --argjson VmHWM "$(calc $1 VmHWM)" \ - --argjson VmSize "$(calc $1 VmSize)" \ - --argjson VmData "$(calc $1 VmData)" \ - '{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}') - - echo "$JSON" - } - - JSON=$(jq -c -n \ - --argjson beforeGc "$(variation beforeGc)" \ - --argjson afterGc "$(variation afterGc)" \ - --argjson afterRequest "$(variation afterRequest)" \ - '{beforeGc: $beforeGc, afterGc: $afterGc, afterRequest: $afterRequest}') - - echo "res=$JSON" >> "$GITHUB_OUTPUT" + - name: Output base JS footprint + run: cat ./artifacts/js-footprint-base.json + - name: Output head JS footprint + run: cat ./artifacts/js-footprint-head.json - id: build-comment name: Build memory comment - env: - RES: ${{ steps.compare.outputs.res }} - run: | - HEADER="## Backend memory usage comparison" - FOOTER="[See workflow logs for details](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})" - - echo "$HEADER" > ./output.md - echo >> ./output.md - - table() { - echo "| Metric | base (MB) | head (MB) | Diff (MB) | Diff (%) |" >> ./output.md - echo "|--------|------:|------:|------:|------:|" >> ./output.md - - line() { - METRIC=$2 - BASE=$(echo "$RES" | jq -r ".${1}.${2}.base") - HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head") - DIFF=$(echo "$RES" | jq -r ".${1}.${2}.diff") - DIFF_PERCENT=$(echo "$RES" | jq -r ".${1}.${2}.diff_percent") - - if (( $(echo "$DIFF_PERCENT > 0" | bc -l) )); then - DIFF="+$DIFF" - DIFF_PERCENT="+$DIFF_PERCENT" - fi - - # highlight VmRSS - if [ "$2" = "VmRSS" ]; then - METRIC="**${METRIC}**" - BASE="**${BASE}**" - HEAD="**${HEAD}**" - DIFF="**${DIFF}**" - DIFF_PERCENT="**${DIFF_PERCENT}**" - fi - - echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md - } - - line $1 VmRSS - line $1 VmHWM - line $1 VmSize - line $1 VmData - } - - echo "### Before GC" >> ./output.md - table beforeGc - echo >> ./output.md - - echo "### After GC" >> ./output.md - table afterGc - echo >> ./output.md - - echo "### After Request" >> ./output.md - table afterRequest - echo >> ./output.md - - # Determine if this is a significant change (more than 5% increase) - if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then - echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md - echo >> ./output.md - fi - - echo "$FOOTER" >> ./output.md + run: node .github/scripts/backend-memory-report.mts ./artifacts/memory-base.json ./artifacts/memory-head.json ./output.md ./artifacts/js-footprint-base.json ./artifacts/js-footprint-head.json - uses: thollander/actions-comment-pull-request@v3 with: pr-number: ${{ steps.load-pr-num.outputs.pr-number }} diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index b8eb0e2493..198451c0e9 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -19,12 +19,6 @@ on: - .github/workflows/test-backend.yml - .github/misskey/test.yml workflow_dispatch: - inputs: - force_ffmpeg_cache_update: - description: 'Force update ffmpeg cache' - required: false - default: false - type: boolean jobs: unit: @@ -62,36 +56,9 @@ jobs: submodules: true - name: Setup pnpm uses: pnpm/action-setup@v6.0.3 - - name: Get current date - id: current-date - run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - name: Setup and Restore ffmpeg/ffprobe Cache - id: cache-ffmpeg - uses: actions/cache@v5 - with: - path: | - /usr/local/bin/ffmpeg - /usr/local/bin/ffprobe - # daily cache - key: ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }} - restore-keys: | - ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }} - name: Install FFmpeg - if: steps.cache-ffmpeg.outputs.cache-hit != 'true' || github.event.inputs.force_ffmpeg_cache_update == true run: | - for i in {1..3}; do - echo "Attempt $i: Installing FFmpeg..." - curl -s -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz && \ - tar -xf ffmpeg.tar.xz && \ - mv ffmpeg-*-static/ffmpeg /usr/local/bin/ && \ - mv ffmpeg-*-static/ffprobe /usr/local/bin/ && \ - rm -rf ffmpeg.tar.xz ffmpeg-*-static/ && \ - break || sleep 10 - if [ $i -eq 3 ]; then - echo "Failed to install FFmpeg after 3 attempts" - exit 1 - fi - done + sudo apt install -y ffmpeg - name: Use Node.js uses: actions/setup-node@v6.4.0 with: diff --git a/.github/workflows/test-federation.yml b/.github/workflows/test-federation.yml index a8b7831f44..0433931531 100644 --- a/.github/workflows/test-federation.yml +++ b/.github/workflows/test-federation.yml @@ -15,12 +15,6 @@ on: - packages/misskey-js/** - .github/workflows/test-federation.yml workflow_dispatch: - inputs: - force_ffmpeg_cache_update: - description: 'Force update ffmpeg cache' - required: false - default: false - type: boolean jobs: test: @@ -37,36 +31,9 @@ jobs: submodules: true - name: Setup pnpm uses: pnpm/action-setup@v6.0.3 - - name: Get current date - id: current-date - run: echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - name: Setup and Restore ffmpeg/ffprobe Cache - id: cache-ffmpeg - uses: actions/cache@v5 - with: - path: | - /usr/local/bin/ffmpeg - /usr/local/bin/ffprobe - # daily cache - key: ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }} - restore-keys: | - ${{ runner.os }}-ffmpeg-${{ steps.current-date.outputs.today }} - name: Install FFmpeg - if: steps.cache-ffmpeg.outputs.cache-hit != 'true' || github.event.inputs.force_ffmpeg_cache_update == true run: | - for i in {1..3}; do - echo "Attempt $i: Installing FFmpeg..." - curl -s -L https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o ffmpeg.tar.xz && \ - tar -xf ffmpeg.tar.xz && \ - mv ffmpeg-*-static/ffmpeg /usr/local/bin/ && \ - mv ffmpeg-*-static/ffprobe /usr/local/bin/ && \ - rm -rf ffmpeg.tar.xz ffmpeg-*-static/ && \ - break || sleep 10 - if [ $i -eq 3 ]; then - echo "Failed to install FFmpeg after 3 attempts" - exit 1 - fi - done + sudo apt install -y ffmpeg - name: Use Node.js uses: actions/setup-node@v6.4.0 with: diff --git a/.node-version b/.node-version index b8ffd70759..91d5f6ff8e 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -22.15.0 +22.18.0 diff --git a/packages/backend/scripts/measure-memory.mjs b/packages/backend/scripts/measure-memory.mjs deleted file mode 100644 index 7c058a131d..0000000000 --- a/packages/backend/scripts/measure-memory.mjs +++ /dev/null @@ -1,231 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -/** - * This script starts the Misskey backend server, waits for it to be ready, - * measures memory usage, and outputs the result as JSON. - * - * Usage: node scripts/measure-memory.mjs - */ - -import { fork } from 'node:child_process'; -import { setTimeout } from 'node:timers/promises'; -import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; -import * as http from 'node:http'; -import * as fs from 'node:fs/promises'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -const SAMPLE_COUNT = 3; // Number of samples to measure -const STARTUP_TIMEOUT = 120000; // 120 seconds timeout for server startup -const MEMORY_SETTLE_TIME = 10000; // Wait 10 seconds after startup for memory to settle - -const keys = { - VmPeak: 0, - VmSize: 0, - VmHWM: 0, - VmRSS: 0, - VmData: 0, - VmStk: 0, - VmExe: 0, - VmLib: 0, - VmPTE: 0, - VmSwap: 0, -}; - -async function getMemoryUsage(pid) { - const status = await fs.readFile(`/proc/${pid}/status`, 'utf-8'); - - const result = {}; - for (const key of Object.keys(keys)) { - const match = status.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); - if (match) { - result[key] = parseInt(match[1], 10); - } else { - throw new Error(`Failed to parse ${key} from /proc/${pid}/status`); - } - } - - return result; -} - -async function measureMemory() { - // Start the Misskey backend server using fork to enable IPC - const serverProcess = fork(join(__dirname, '../built/entry.js'), ['expose-gc'], { - cwd: join(__dirname, '..'), - env: { - ...process.env, - NODE_ENV: 'production', - MK_DISABLE_CLUSTERING: '1', - }, - stdio: ['pipe', 'pipe', 'pipe', 'ipc'], - execArgv: [...process.execArgv, '--expose-gc'], - }); - - let serverReady = false; - - // Listen for the 'ok' message from the server indicating it's ready - serverProcess.on('message', (message) => { - if (message === 'ok') { - serverReady = true; - } - }); - - // Handle server output - serverProcess.stdout?.on('data', (data) => { - process.stderr.write(`[server stdout] ${data}`); - }); - - serverProcess.stderr?.on('data', (data) => { - process.stderr.write(`[server stderr] ${data}`); - }); - - // Handle server error - serverProcess.on('error', (err) => { - process.stderr.write(`[server error] ${err}\n`); - }); - - async function triggerGc() { - const ok = new Promise((resolve) => { - serverProcess.once('message', (message) => { - if (message === 'gc ok') resolve(); - }); - }); - - serverProcess.send('gc'); - - await ok; - - await setTimeout(1000); - } - - function createRequest() { - return new Promise((resolve, reject) => { - const req = http.request({ - host: 'localhost', - port: 61812, - path: '/api/meta', - method: 'POST', - }, (res) => { - res.on('data', () => { }); - res.on('end', () => { - resolve(); - }); - }); - req.on('error', (err) => { - reject(err); - }); - req.end(); - }); - } - - // Wait for server to be ready or timeout - const startupStartTime = Date.now(); - while (!serverReady) { - if (Date.now() - startupStartTime > STARTUP_TIMEOUT) { - serverProcess.kill('SIGTERM'); - throw new Error('Server startup timeout'); - } - await setTimeout(100); - } - - const startupTime = Date.now() - startupStartTime; - process.stderr.write(`Server started in ${startupTime}ms\n`); - - // Wait for memory to settle - await setTimeout(MEMORY_SETTLE_TIME); - - const pid = serverProcess.pid; - - const beforeGc = await getMemoryUsage(pid); - - await triggerGc(); - - const afterGc = await getMemoryUsage(pid); - - // create some http requests to simulate load - const REQUEST_COUNT = 10; - await Promise.all( - Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), - ); - - await triggerGc(); - - const afterRequest = await getMemoryUsage(pid); - - // Stop the server - serverProcess.kill('SIGTERM'); - - // Wait for process to exit - let exited = false; - await new Promise((resolve) => { - serverProcess.on('exit', () => { - exited = true; - resolve(undefined); - }); - // Force kill after 10 seconds if not exited - setTimeout(10000).then(() => { - if (!exited) { - serverProcess.kill('SIGKILL'); - } - resolve(undefined); - }); - }); - - const result = { - timestamp: new Date().toISOString(), - beforeGc, - afterGc, - afterRequest, - }; - - return result; -} - -async function main() { - // 直列の方が時間的に分散されて正確そうだから直列でやる - const results = []; - for (let i = 0; i < SAMPLE_COUNT; i++) { - const res = await measureMemory(); - results.push(res); - } - - // Calculate averages - const beforeGc = structuredClone(keys); - const afterGc = structuredClone(keys); - const afterRequest = structuredClone(keys); - for (const res of results) { - for (const key of Object.keys(keys)) { - beforeGc[key] += res.beforeGc[key]; - afterGc[key] += res.afterGc[key]; - afterRequest[key] += res.afterRequest[key]; - } - } - for (const key of Object.keys(keys)) { - beforeGc[key] = Math.round(beforeGc[key] / SAMPLE_COUNT); - afterGc[key] = Math.round(afterGc[key] / SAMPLE_COUNT); - afterRequest[key] = Math.round(afterRequest[key] / SAMPLE_COUNT); - } - - const result = { - timestamp: new Date().toISOString(), - beforeGc, - afterGc, - afterRequest, - }; - - // Output as JSON to stdout - console.log(JSON.stringify(result, null, 2)); -} - -main().catch((err) => { - console.error(JSON.stringify({ - error: err.message, - timestamp: new Date().toISOString(), - })); - process.exit(1); -}); diff --git a/packages/backend/scripts/measure-memory.mts b/packages/backend/scripts/measure-memory.mts new file mode 100644 index 0000000000..bc26fe2398 --- /dev/null +++ b/packages/backend/scripts/measure-memory.mts @@ -0,0 +1,535 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { ChildProcess, fork } from 'node:child_process'; +import { setTimeout } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +//import * as http from 'node:http'; +import * as fs from 'node:fs/promises'; +import { heapSnapshotCategory, type HeapSnapshotData } from '../../../.github/scripts/heap-snapshot-util.mts'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +function readIntegerEnv(name, defaultValue, min) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (!/^\d+$/.test(rawValue)) throw new Error(`${name} must be an integer`); + + const value = Number(rawValue); + if (!Number.isSafeInteger(value) || value < min) throw new Error(`${name} must be >= ${min}`); + return value; +} + +function readBooleanEnv(name, defaultValue) { + const rawValue = process.env[name]; + if (rawValue == null || rawValue === '') return defaultValue; + if (rawValue === '1' || rawValue === 'true') return true; + if (rawValue === '0' || rawValue === 'false') return false; + throw new Error(`${name} must be one of: 1, 0, true, false`); +} + +const SAMPLE_COUNT = readIntegerEnv('MK_MEMORY_SAMPLE_COUNT', 3, 1); // Number of samples to measure +const STARTUP_TIMEOUT = readIntegerEnv('MK_MEMORY_STARTUP_TIMEOUT_MS', 120000, 1); // Timeout for server startup +const MEMORY_SETTLE_TIME = readIntegerEnv('MK_MEMORY_SETTLE_TIME_MS', 10000, 0); // Wait after startup for memory to settle +const IPC_TIMEOUT = readIntegerEnv('MK_MEMORY_IPC_TIMEOUT_MS', 30000, 1); // Timeout for IPC responses +const REQUEST_COUNT = readIntegerEnv('MK_MEMORY_REQUEST_COUNT', 10, 0); +const HEAP_SNAPSHOT = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false); +const HEAP_SNAPSHOT_TIMEOUT = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_TIMEOUT_MS', 120000, 1); +const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1); + +const procStatusKeys = ['VmPeak', 'VmSize', 'VmHWM', 'VmRSS', 'VmData', 'VmStk', 'VmExe', 'VmLib', 'VmPTE', 'VmSwap'] as const; +const smapsRollupKeys = ['Pss', 'Shared_Clean', 'Shared_Dirty', 'Private_Clean', 'Private_Dirty', 'Swap', 'SwapPss'] as const; + +const typedArrayNames = new Set([ + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'system / JSArrayBufferData', +]); + +const otherJsNodeTypes = new Set([ + 'object', + 'closure', + 'regexp', + 'number', + 'symbol', + 'bigint', +]); + +function parseMemoryFile(content: string, keys: KS, path: string, required: boolean): Record { + const result = {} as Record; + for (const _key of keys) { + const key = _key as KS[number]; + const match = content.match(new RegExp(`${key}:\\s+(\\d+)\\s+kB`)); + if (match) { + result[key] = parseInt(match[1], 10); + } else if (required) { + throw new Error(`Failed to parse ${key} from ${path}`); + } + } + return result; +} + +function bytesToKiB(value: number) { + return Math.round(value / 1024); +} + +function isTypedArrayNode(type, name) { + return typedArrayNames.has(name) || + (type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray'))); +} + +function isSystemNode(type, name) { + return type === 'hidden' || + type === 'synthetic' || + type === 'object shape' || + name.startsWith('system /') || + name.startsWith('(system '); +} + +function classifyHeapSnapshotNode(type, name): keyof typeof heapSnapshotCategory { + if (type === 'code') return 'code'; + if (type === 'string' || type === 'concatenated string' || type === 'sliced string') return 'strings'; + if (isTypedArrayNode(type, name)) return 'typedArrays'; + if (type === 'array' || (type === 'object' && name === 'Array')) return 'jsArrays'; + if (isSystemNode(type, name)) return 'systemObjects'; + if (otherJsNodeTypes.has(type)) return 'otherJsObjects'; + return 'otherNonJsObjects'; +} + +function sanitizeHeapSnapshotBreakdownLabel(value, fallback = 'unknown') { + const label = String(value ?? '').replace(/\s+/g, ' ').trim(); + if (label === '') return fallback; + if (label.length <= 80) return label; + return `${label.slice(0, 77)}...`; +} + +function classifyHeapSnapshotBreakdown(category: keyof typeof heapSnapshotCategory, type, name) { + if (category === 'strings') return type; + + if (category === 'jsArrays') { + if (type === 'array') return 'array nodes'; + if (type === 'object' && name === 'Array') return 'Array objects'; + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`); + } + + if (category === 'typedArrays') { + if (name === 'system / JSArrayBufferData') return 'ArrayBuffer data'; + if (name === 'Uint8Array') return 'Uint8Array / Buffer'; + if (typedArrayNames.has(name)) return name; + if (type === 'native' && name.includes('ArrayBuffer')) return 'native ArrayBuffer'; + if (type === 'native' && name.includes('TypedArray')) return 'native TypedArray'; + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`); + } + + if (category === 'systemObjects') { + if (name.startsWith('system /')) return sanitizeHeapSnapshotBreakdownLabel(name); + if (name.startsWith('(system ')) return sanitizeHeapSnapshotBreakdownLabel(name); + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); + } + + if (category === 'otherJsObjects') { + if (type === 'object') return sanitizeHeapSnapshotBreakdownLabel(`object: ${name}`, 'object: unknown'); + return type; + } + + if (category === 'otherNonJsObjects') { + if (type === 'native') return sanitizeHeapSnapshotBreakdownLabel(`native: ${name}`, 'native: unknown'); + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); + } + + if (category === 'code') { + const lowerName = name.toLowerCase(); + if (lowerName.includes('bytecode')) return 'bytecode'; + if (lowerName.includes('builtin')) return 'builtins'; + if (lowerName.includes('regexp')) return 'regexp code'; + if (lowerName.includes('stub')) return 'stubs'; + return sanitizeHeapSnapshotBreakdownLabel(`code: ${name}`, 'code: unknown'); + } + + return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); +} + +function collapseHeapSnapshotBreakdown(breakdowns: Record>) { + const collapsed = {} as Record>; + + for (const [category, children] of Object.entries(breakdowns)) { + const entries = Object.entries(children) + .filter(([, value]) => value > 0) + .toSorted((a, b) => b[1] - a[1]); + + const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N); + const otherValue = entries + .slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N) + .reduce((sum, [, value]) => sum + value, 0); + + const categoryBreakdown = Object.fromEntries(topEntries); + if (otherValue > 0) categoryBreakdown.Other = otherValue; + if (Object.keys(categoryBreakdown).length > 0) collapsed[category] = categoryBreakdown; + } + + return collapsed; +} + +function analyzeHeapSnapshot(snapshot) { + const meta = snapshot?.snapshot?.meta; + const nodes = snapshot?.nodes; + const strings = snapshot?.strings; + if (meta == null || !Array.isArray(nodes) || !Array.isArray(strings)) { + throw new Error('Invalid heap snapshot format'); + } + + const nodeFields = meta.node_fields; + if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields'); + + const typeOffset = nodeFields.indexOf('type'); + const nameOffset = nodeFields.indexOf('name'); + const selfSizeOffset = nodeFields.indexOf('self_size'); + if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0) { + throw new Error('Heap snapshot is missing required node fields'); + } + + const nodeTypeNames = meta.node_types?.[typeOffset]; + if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types'); + + function createEmptyHeapSnapshotCategoryMap() { + return Object.fromEntries(Object.keys(heapSnapshotCategory).map(category => [category, 0])) as Record; + } + + const fieldCount = nodeFields.length; + const categories = createEmptyHeapSnapshotCategoryMap(); + const nodeCounts = createEmptyHeapSnapshotCategoryMap(); + const breakdowns = Object.fromEntries( + (Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) + .filter(category => category !== 'total') + .map(category => [category, {}]), + ); + + function addValue(map: Record, key: string, value: number) { + map[key] = (map[key] ?? 0) + value; + } + + for (let offset = 0; offset < nodes.length; offset += fieldCount) { + const type = nodeTypeNames[nodes[offset + typeOffset]] ?? 'unknown'; + const name = strings[nodes[offset + nameOffset]] ?? ''; + const selfSize = nodes[offset + selfSizeOffset] ?? 0; + const category = classifyHeapSnapshotNode(type, name); + + categories[category] += selfSize; + categories.total += selfSize; + nodeCounts[category]++; + nodeCounts.total++; + addValue(breakdowns[category], classifyHeapSnapshotBreakdown(category, type, name), selfSize); + } + + return { + categories, + nodeCounts, + breakdowns: collapseHeapSnapshotBreakdown(breakdowns), + }; +} + +async function getMemoryUsage(pid: number) { + const path = `/proc/${pid}/status`; + const status = await fs.readFile(path, 'utf-8'); + return parseMemoryFile(status, procStatusKeys, path, true); +} + +async function getSmapsRollupMemoryUsage(pid: number) { + const path = `/proc/${pid}/smaps_rollup`; + const smapsRollup = await fs.readFile(path, 'utf-8'); + return parseMemoryFile(smapsRollup, smapsRollupKeys, path, false); +} + +function waitForMessage(serverProcess, predicate, description, timeout = IPC_TIMEOUT) { + return new Promise((resolve, reject) => { + const timer = globalThis.setTimeout(() => { + serverProcess.off('message', onMessage); + reject(new Error(`Timed out waiting for ${description}`)); + }, timeout); + + const onMessage = (message) => { + if (!predicate(message)) return; + globalThis.clearTimeout(timer); + serverProcess.off('message', onMessage); + resolve(message); + }; + + serverProcess.on('message', onMessage); + }); +} + +async function getRuntimeMemoryUsage(serverProcess: ChildProcess) { + const response = waitForMessage( + serverProcess, + message => message != null && typeof message === 'object' && message.type === 'memory usage', + 'memory usage', + ); + + serverProcess.send('memory usage'); + + const message = await response; + const memoryUsage = message.value; + + return { + HeapTotal: bytesToKiB(memoryUsage.heapTotal), + HeapUsed: bytesToKiB(memoryUsage.heapUsed), + External: bytesToKiB(memoryUsage.external), + ArrayBuffers: bytesToKiB(memoryUsage.arrayBuffers), + }; +} + +async function getHeapSnapshotStatistics(serverProcess: ChildProcess): Promise { + if (!HEAP_SNAPSHOT) return null; + + const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`); + const response = waitForMessage( + serverProcess, + message => message != null && typeof message === 'object' && (message.type === 'heap snapshot' || message.type === 'heap snapshot error'), + 'heap snapshot', + HEAP_SNAPSHOT_TIMEOUT, + ); + + serverProcess.send({ + type: 'heap snapshot', + path: snapshotPath, + }); + + const message = await response; + if (message.type === 'heap snapshot error') { + throw new Error(`Failed to write heap snapshot: ${message.message}`); + } + + const writtenPath = typeof message.path === 'string' ? message.path : snapshotPath; + + try { + const snapshot = JSON.parse(await fs.readFile(writtenPath, 'utf-8')); + return analyzeHeapSnapshot(snapshot); + } finally { + await fs.unlink(writtenPath).catch(err => { + process.stderr.write(`Failed to delete heap snapshot ${writtenPath}: ${err.message}\n`); + }); + } +} + +async function getAllMemoryUsage(serverProcess: ChildProcess) { + const pid = serverProcess.pid!; + return { + ...await getMemoryUsage(pid), + ...await getSmapsRollupMemoryUsage(pid), + ...await getRuntimeMemoryUsage(serverProcess), + }; +} + +async function measureMemory() { + // Start the Misskey backend server using fork to enable IPC + const serverProcess = fork(join(__dirname, '../built/entry.js'), [], { + cwd: join(__dirname, '..'), + env: { + ...process.env, + NODE_ENV: 'production', + MK_DISABLE_CLUSTERING: '1', + MK_ONLY_SERVER: '1', + MK_NO_DAEMONS: '1', + }, + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv: [...process.execArgv, '--expose-gc'], + }); + + let serverReady = false; + + // Listen for the 'ok' message from the server indicating it's ready + serverProcess.on('message', (message) => { + if (message === 'ok') { + serverReady = true; + } + }); + + // Handle server output + serverProcess.stdout?.on('data', (data) => { + process.stderr.write(`[server stdout] ${data}`); + }); + + serverProcess.stderr?.on('data', (data) => { + process.stderr.write(`[server stderr] ${data}`); + }); + + // Handle server error + serverProcess.on('error', (err) => { + process.stderr.write(`[server error] ${err}\n`); + }); + + async function triggerGc() { + const ok = waitForMessage( + serverProcess, + message => message === 'gc ok' || message === 'gc unavailable', + 'GC completion', + ); + + serverProcess.send('gc'); + + const message = await ok; + if (message === 'gc unavailable') { + throw new Error('GC is unavailable. Start the process with --expose-gc to enable this feature.'); + } + + await setTimeout(1000); + } + + //function createRequest() { + // return new Promise((resolve, reject) => { + // const req = http.request({ + // host: 'localhost', + // port: 61812, + // path: '/api/meta', + // method: 'POST', + // }, (res) => { + // res.on('data', () => { }); + // res.on('end', () => { + // resolve(); + // }); + // }); + // req.on('error', (err) => { + // reject(err); + // }); + // req.end(); + // }); + //} + + // Wait for server to be ready or timeout + const startupStartTime = Date.now(); + while (!serverReady) { + if (Date.now() - startupStartTime > STARTUP_TIMEOUT) { + serverProcess.kill('SIGTERM'); + throw new Error('Server startup timeout'); + } + await setTimeout(100); + } + + const startupTime = Date.now() - startupStartTime; + process.stderr.write(`Server started in ${startupTime}ms\n`); + + // Wait for memory to settle + await setTimeout(MEMORY_SETTLE_TIME); + + //const beforeGc = await getAllMemoryUsage(serverProcess); + + await triggerGc(); + + const memoryUsageAfterGC = await getAllMemoryUsage(serverProcess); + + //// create some http requests to simulate load + //await Promise.all( + // Array.from({ length: REQUEST_COUNT }).map(() => createRequest()), + //); + + //await triggerGc(); + + //const afterRequest = await getAllMemoryUsage(serverProcess); + + const heapSnapshotAfterGc = await getHeapSnapshotStatistics(serverProcess); + + // Stop the server + serverProcess.kill('SIGTERM'); + + // Wait for process to exit + let exited = false; + await new Promise((resolve) => { + serverProcess.on('exit', () => { + exited = true; + resolve(undefined); + }); + // Force kill after 10 seconds if not exited + setTimeout(10000).then(() => { + if (!exited) { + serverProcess.kill('SIGKILL'); + } + resolve(undefined); + }); + }); + + const result = { + timestamp: new Date().toISOString(), + phases: { + //beforeGc, + afterGc: { + memoryUsage: memoryUsageAfterGC, + heapSnapshot: heapSnapshotAfterGc, + }, + //afterRequest, + }, + }; + + return result; +} + +export type MemoryReportRaw = { + timestamp: string; + sampleCount: number; + measurement: { + startupTimeoutMs: number; + memorySettleTimeMs: number; + ipcTimeoutMs: number; + requestCount: number; + heapSnapshot: { + enabled: boolean; + timeoutMs: number; + breakdownTopN: number; + }; + }; + samples: Awaited>[]; +}; + +async function main() { + const results = []; + for (let i = 0; i < SAMPLE_COUNT; i++) { + process.stderr.write(`Starting sample ${i + 1}/${SAMPLE_COUNT}\n`); + const res = await measureMemory(); + results.push(res); + } + + const result: MemoryReportRaw = { + timestamp: new Date().toISOString(), + sampleCount: SAMPLE_COUNT, + measurement: { + startupTimeoutMs: STARTUP_TIMEOUT, + memorySettleTimeMs: MEMORY_SETTLE_TIME, + ipcTimeoutMs: IPC_TIMEOUT, + requestCount: REQUEST_COUNT, + heapSnapshot: { + enabled: HEAP_SNAPSHOT, + timeoutMs: HEAP_SNAPSHOT_TIMEOUT, + breakdownTopN: HEAP_SNAPSHOT_BREAKDOWN_TOP_N, + }, + }, + samples: results, + }; + + // Output as JSON to stdout + console.log(JSON.stringify(result, null, 2)); +} + +main().catch((err) => { + console.error(JSON.stringify({ + error: err.message, + timestamp: new Date().toISOString(), + })); + process.exit(1); +}); diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index b27e8c1d6b..8c1862f13e 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -6,6 +6,7 @@ import { NestFactory } from '@nestjs/core'; import { init } from 'slacc'; import { NestLogger } from '@/NestLogger.js'; +import { envOption } from '@/env.js'; import type { Config } from '@/config.js'; let slaccInitialized = false; @@ -31,7 +32,7 @@ export async function server() { const serverService = app.get(ServerService); await serverService.launch(); - if (process.env.NODE_ENV !== 'test') { + if (process.env.NODE_ENV !== 'test' && !envOption.noDaemons) { const { ChartManagementService } = await import('../core/chart/ChartManagementService.js'); const { QueueStatsService } = await import('../daemons/QueueStatsService.js'); const { ServerStatsService } = await import('../daemons/ServerStatsService.js'); @@ -54,7 +55,9 @@ export async function jobQueue() { }); jobQueue.get(QueueProcessorService).start(); - jobQueue.get(ChartManagementService).start(); + if (!envOption.noDaemons) { + jobQueue.get(ChartManagementService).start(); + } return jobQueue; } diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 6e37bf9e1c..9fcb4d73ca 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -9,6 +9,7 @@ import cluster from 'node:cluster'; import { EventEmitter } from 'node:events'; +import { writeHeapSnapshot } from 'node:v8'; import chalk from 'chalk'; import Xev from 'xev'; import Logger from '@/logger.js'; @@ -91,10 +92,35 @@ process.on('message', msg => { if (msg === 'gc') { if (global.gc != null) { logger.info('Manual GC triggered'); - global.gc(); + for (let i = 0; i < 3; i++) { + global.gc(); + } if (process.send != null) process.send('gc ok'); } else { logger.warn('Manual GC requested but gc is not available. Start the process with --expose-gc to enable this feature.'); + if (process.send != null) process.send('gc unavailable'); + } + } else if (msg === 'memory usage') { + if (process.send != null) { + process.send({ + type: 'memory usage', + value: process.memoryUsage(), + }); + } + } else if (msg != null && typeof msg === 'object' && 'type' in msg && msg.type === 'heap snapshot' && 'path' in msg && typeof msg.path === 'string') { + if (process.send != null) { + try { + const path = writeHeapSnapshot(msg.path); + process.send({ + type: 'heap snapshot', + path, + }); + } catch (err) { + process.send({ + type: 'heap snapshot error', + message: err instanceof Error ? err.message : String(err), + }); + } } } }); diff --git a/packages/frontend/vite.config.ts b/packages/frontend/vite.config.ts index b1413be0a4..8cf725b1a1 100644 --- a/packages/frontend/vite.config.ts +++ b/packages/frontend/vite.config.ts @@ -27,27 +27,29 @@ const extensions = ['.ts', '.tsx', '.js', '.jsx', '.mjs', '.json', '.json5', '.s function getBundleVisualizerPlugin(): PluginOption[] { if (process.env.FRONTEND_BUNDLE_VISUALIZER !== 'true') return []; - const template = process.env.FRONTEND_BUNDLE_VISUALIZER_TEMPLATE === 'markdown' - ? 'markdown' - : process.env.FRONTEND_BUNDLE_VISUALIZER_TEMPLATE === 'raw-data' - ? 'raw-data' - : 'treemap'; - const defaultFilename = template === 'markdown' - ? path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/report.md') - : template === 'raw-data' - ? path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/stats.json') - : path.resolve(__dirname, '../../built/_frontend_bundle_visualizer_/stats.html'); - - return [ + const visualizerOptions = { + title: 'Misskey frontend bundle visualizer', + gzipSize: true, + brotliSize: true, + projectRoot: path.resolve(__dirname, '../..'), + }; + const plugins = [ visualizer({ - filename: process.env.FRONTEND_BUNDLE_VISUALIZER_FILE ?? defaultFilename, - title: 'Misskey frontend bundle visualizer', - template, - gzipSize: true, - brotliSize: true, - projectRoot: path.resolve(__dirname, '../..'), + ...visualizerOptions, + filename: process.env.FRONTEND_BUNDLE_VISUALIZER_FILE, + template: 'raw-data', }) as PluginOption, ]; + + if (process.env.FRONTEND_BUNDLE_VISUALIZER_HTML_FILE != null) { + plugins.push(visualizer({ + ...visualizerOptions, + filename: process.env.FRONTEND_BUNDLE_VISUALIZER_HTML_FILE, + template: 'treemap', + }) as PluginOption); + } + + return plugins; } /**