From 96a454ee3a5e91084cff1c68ac19988d17d15a34 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:17:43 +0900 Subject: [PATCH] wip --- .github/scripts/frontend-browser-report.mts | 352 ++++++ .../measure-frontend-browser-comparison.mts | 1061 +++++++++++++++++ ...rontend-browser-metrics-report-comment.yml | 44 + .../frontend-browser-metrics-report.yml | 173 +++ 4 files changed, 1630 insertions(+) create mode 100644 .github/scripts/frontend-browser-report.mts create mode 100644 .github/scripts/measure-frontend-browser-comparison.mts create mode 100644 .github/workflows/frontend-browser-metrics-report-comment.yml create mode 100644 .github/workflows/frontend-browser-metrics-report.yml diff --git a/.github/scripts/frontend-browser-report.mts b/.github/scripts/frontend-browser-report.mts new file mode 100644 index 0000000000..92265e1632 --- /dev/null +++ b/.github/scripts/frontend-browser-report.mts @@ -0,0 +1,352 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { readFile, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import * as util from './utility.mts'; +import * as heapSnapshotUtil from './heap-snapshot-util.mts'; +import type { HeapSnapshotData } from './heap-snapshot-util.mts'; + +export type BrowserMeasurement = { + label: string; + timestamp: string; + url: string; + scenario: string; + durationMs: number; + network: { + requestCount: number; + finishedRequestCount: number; + failedRequestCount: number; + cachedRequestCount: number; + serviceWorkerRequestCount: number; + totalEncodedBytes: number; + totalDecodedBodyBytes: number; + sameOriginEncodedBytes: number; + thirdPartyEncodedBytes: number; + byResourceType: Record; + largestRequests: { + url: string; + method: string; + resourceType: string; + status?: number; + encodedBytes: number; + decodedBodyBytes: number; + }[]; + failedRequests: { + url: string; + method: string; + resourceType: string; + errorText?: string; + status?: number; + }[]; + }; + performance: { + cdpMetrics: Record; + runtimeHeap?: { + usedSize: number; + totalSize: number; + }; + webVitals: { + firstPaintMs?: number; + firstContentfulPaintMs?: number; + domContentLoadedEventEndMs?: number; + loadEventEndMs?: number; + longTaskCount: number; + longTaskDurationMs: number; + maxLongTaskDurationMs: number; + resourceEntryCount: number; + domElements: number; + }; + }; + heapSnapshot: HeapSnapshotData; +}; + +export type BrowserMeasurementSample = BrowserMeasurement & { + round: number; +}; + +export type BrowserMetricsReport = { + label: string; + timestamp: string; + url: string; + scenario: string; + sampleCount: number; + aggregation: 'median'; + summary: BrowserMeasurement; + samples: BrowserMeasurementSample[]; +}; + +function escapeCell(value: string) { + return String(value).replaceAll('|', '\\|').replaceAll('\n', '
'); +} + +function truncate(value: string, maxLength = 140) { + if (value.length <= maxLength) return value; + return `${value.slice(0, maxLength - 3)}...`; +} + +function formatMs(value: number | null | undefined) { + if (value == null || !Number.isFinite(value)) return '-'; + if (value >= 1_000) return `${util.formatNumber(value / 1_000)} s`; + return `${util.formatNumber(value)} ms`; +} + +function formatSecondsAsMs(value: number | null | undefined) { + if (value == null || !Number.isFinite(value)) return '-'; + return formatMs(value * 1_000); +} + +function formatDelta(value: number, formatter: (value: number) => string) { + if (value === 0) return formatter(0); + return util.formatColoredDelta(formatter(Math.abs(value)), value); +} + +function finiteValues(values: (number | null | undefined)[]) { + return values.filter(value => Number.isFinite(value)) as number[]; +} + +function sampleSpread(report: BrowserMetricsReport, getValue: (sample: BrowserMeasurementSample) => number | null | undefined) { + const values = finiteValues(report.samples.map(sample => getValue(sample))); + if (values.length < 2) return null; + + const center = util.median(values); + return util.median(values.map(value => Math.abs(value - center))); +} + +function formatValueWithSpread(report: BrowserMetricsReport, value: number, getSampleValue: (sample: BrowserMeasurementSample) => number | null | undefined, formatter: (value: number) => string) { + const spread = sampleSpread(report, getSampleValue); + if (spread == null) return formatter(value); + return `${formatter(value)}
± ${formatter(spread)}`; +} + +function pairedDelta(reportBase: BrowserMetricsReport, reportHead: BrowserMetricsReport, getValue: (sample: BrowserMeasurementSample) => number | null | undefined) { + try { + return util.pairedDeltaSummary(reportBase.samples, reportHead.samples, sample => getValue(sample) ?? null); + } catch { + return null; + } +} + +function metricRow( + label: string, + base: BrowserMetricsReport, + head: BrowserMetricsReport, + getSummaryValue: (summary: BrowserMeasurement) => number | null | undefined, + getSampleValue: (sample: BrowserMeasurementSample) => number | null | undefined, + formatter: (value: number) => string, +) { + const baseValue = getSummaryValue(base.summary); + const headValue = getSummaryValue(head.summary); + if (baseValue == null || headValue == null || !Number.isFinite(baseValue) || !Number.isFinite(headValue)) return null; + + const summary = pairedDelta(base, head, getSampleValue); + const medianDelta = summary?.median ?? (headValue - baseValue); + const deltaMedian = `${formatDelta(medianDelta, formatter)}
${util.calcAndFormatDeltaPercent(baseValue, headValue).replaceAll('\\%', '\\\\%')}`; + + return `| **${label}** | ${formatValueWithSpread(base, baseValue, getSampleValue, formatter)} | ${formatValueWithSpread(head, headValue, getSampleValue, formatter)} | ${deltaMedian} | ${summary == null ? '-' : formatter(summary.mad)} | ${summary == null ? '-' : formatDelta(summary.min, formatter)} | ${summary == null ? '-' : formatDelta(summary.max, formatter)} |`; +} + +function resourceTypeBytes(report: BrowserMeasurement, resourceTypes: string[]) { + return resourceTypes.reduce((sum, resourceType) => sum + (report.network.byResourceType[resourceType]?.encodedBytes ?? 0), 0); +} + +function resourceTypeSampleBytes(sample: BrowserMeasurementSample, resourceTypes: string[]) { + return resourceTypeBytes(sample, resourceTypes); +} + +function getMetric(report: BrowserMeasurement, key: string) { + return report.performance.cdpMetrics[key]; +} + +function renderSummaryTable(base: BrowserMetricsReport, head: BrowserMetricsReport) { + const rows = [ + metricRow('Scenario duration', base, head, summary => summary.durationMs, sample => sample.durationMs, formatMs), + metricRow('Requests', base, head, summary => summary.network.requestCount, sample => sample.network.requestCount, util.formatNumber), + metricRow('Failed requests', base, head, summary => summary.network.failedRequestCount, sample => sample.network.failedRequestCount, util.formatNumber), + metricRow('Encoded network', base, head, summary => summary.network.totalEncodedBytes, sample => sample.network.totalEncodedBytes, util.formatBytes), + metricRow('Decoded body', base, head, summary => summary.network.totalDecodedBodyBytes, sample => sample.network.totalDecodedBodyBytes, util.formatBytes), + metricRow('Same-origin encoded', base, head, summary => summary.network.sameOriginEncodedBytes, sample => sample.network.sameOriginEncodedBytes, util.formatBytes), + metricRow('Third-party encoded', base, head, summary => summary.network.thirdPartyEncodedBytes, sample => sample.network.thirdPartyEncodedBytes, util.formatBytes), + metricRow('Script encoded', base, head, summary => resourceTypeBytes(summary, ['Script']), sample => resourceTypeSampleBytes(sample, ['Script']), util.formatBytes), + metricRow('Stylesheet encoded', base, head, summary => resourceTypeBytes(summary, ['Stylesheet']), sample => resourceTypeSampleBytes(sample, ['Stylesheet']), util.formatBytes), + metricRow('Fetch/XHR encoded', base, head, summary => resourceTypeBytes(summary, ['Fetch', 'XHR']), sample => resourceTypeSampleBytes(sample, ['Fetch', 'XHR']), util.formatBytes), + metricRow('Image encoded', base, head, summary => resourceTypeBytes(summary, ['Image']), sample => resourceTypeSampleBytes(sample, ['Image']), util.formatBytes), + metricRow('Font encoded', base, head, summary => resourceTypeBytes(summary, ['Font']), sample => resourceTypeSampleBytes(sample, ['Font']), util.formatBytes), + metricRow('First contentful paint', base, head, summary => summary.performance.webVitals.firstContentfulPaintMs, sample => sample.performance.webVitals.firstContentfulPaintMs, formatMs), + metricRow('Load event end', base, head, summary => summary.performance.webVitals.loadEventEndMs, sample => sample.performance.webVitals.loadEventEndMs, formatMs), + metricRow('Long tasks', base, head, summary => summary.performance.webVitals.longTaskCount, sample => sample.performance.webVitals.longTaskCount, util.formatNumber), + metricRow('Long task duration', base, head, summary => summary.performance.webVitals.longTaskDurationMs, sample => sample.performance.webVitals.longTaskDurationMs, formatMs), + metricRow('Max long task', base, head, summary => summary.performance.webVitals.maxLongTaskDurationMs, sample => sample.performance.webVitals.maxLongTaskDurationMs, formatMs), + metricRow('JS heap used', base, head, summary => summary.performance.runtimeHeap?.usedSize ?? getMetric(summary, 'JSHeapUsedSize'), sample => sample.performance.runtimeHeap?.usedSize ?? getMetric(sample, 'JSHeapUsedSize'), util.formatBytes), + metricRow('JS heap total', base, head, summary => summary.performance.runtimeHeap?.totalSize ?? getMetric(summary, 'JSHeapTotalSize'), sample => sample.performance.runtimeHeap?.totalSize ?? getMetric(sample, 'JSHeapTotalSize'), util.formatBytes), + metricRow('V8 heap snapshot total', base, head, summary => summary.heapSnapshot.categories.total, sample => sample.heapSnapshot.categories.total, util.formatBytes), + metricRow('DOM elements', base, head, summary => summary.performance.webVitals.domElements, sample => sample.performance.webVitals.domElements, util.formatNumber), + metricRow('CDP nodes', base, head, summary => getMetric(summary, 'Nodes'), sample => getMetric(sample, 'Nodes'), util.formatNumber), + metricRow('JS event listeners', base, head, summary => getMetric(summary, 'JSEventListeners'), sample => getMetric(sample, 'JSEventListeners'), util.formatNumber), + metricRow('Layout count', base, head, summary => getMetric(summary, 'LayoutCount'), sample => getMetric(sample, 'LayoutCount'), util.formatNumber), + metricRow('Recalc style count', base, head, summary => getMetric(summary, 'RecalcStyleCount'), sample => getMetric(sample, 'RecalcStyleCount'), util.formatNumber), + metricRow('Script duration', base, head, summary => getMetric(summary, 'ScriptDuration'), sample => getMetric(sample, 'ScriptDuration'), formatSecondsAsMs), + metricRow('Task duration', base, head, summary => getMetric(summary, 'TaskDuration'), sample => getMetric(sample, 'TaskDuration'), formatSecondsAsMs), + ].filter(row => row != null); + + return [ + '| Metric | Base median | Head median | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ...rows, + ].join('\n'); +} + +function renderResourceTypeTable(base: BrowserMetricsReport, head: BrowserMetricsReport) { + const preferredOrder = ['Document', 'Script', 'Stylesheet', 'Fetch', 'XHR', 'Image', 'Font', 'Media', 'WebSocket', 'EventSource', 'Other']; + const keys = [...new Set([ + ...preferredOrder, + ...Object.keys(base.summary.network.byResourceType), + ...Object.keys(head.summary.network.byResourceType), + ])].filter(key => base.summary.network.byResourceType[key] != null || head.summary.network.byResourceType[key] != null); + + const lines = [ + '| Type | Base reqs | Head reqs | Δ reqs | Base encoded | Head encoded | Δ encoded |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + + for (const key of keys) { + const baseRow = base.summary.network.byResourceType[key] ?? { requests: 0, encodedBytes: 0 }; + const headRow = head.summary.network.byResourceType[key] ?? { requests: 0, encodedBytes: 0 }; + lines.push(`| **${key}** | ${util.formatNumber(baseRow.requests)} | ${util.formatNumber(headRow.requests)} | ${formatDelta(headRow.requests - baseRow.requests, util.formatNumber)} | ${util.formatBytes(baseRow.encodedBytes)} | ${util.formatBytes(headRow.encodedBytes)} | ${formatDelta(headRow.encodedBytes - baseRow.encodedBytes, util.formatBytes)} |`); + } + + return lines.join('\n'); +} + +function renderHeapSnapshotTable(base: BrowserMetricsReport, head: BrowserMetricsReport) { + const lines = [ + '| Category | Base median | Head median | Δ median | Δ MAD | Base nodes | Head nodes |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + + for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) { + const baseValue = base.summary.heapSnapshot.categories[category]; + const headValue = head.summary.heapSnapshot.categories[category]; + const baseNodeCount = base.summary.heapSnapshot.nodeCounts[category]; + const headNodeCount = head.summary.heapSnapshot.nodeCounts[category]; + const categoryInfo = heapSnapshotUtil.heapSnapshotCategory[category]; + const summary = pairedDelta(base, head, sample => sample.heapSnapshot.categories[category]); + const metricText = `$\\color{${categoryInfo.color}}{\\rule{8pt}{8pt}}$ **${categoryInfo.label}**`; + lines.push(`| ${metricText} | ${util.formatBytes(baseValue)} | ${util.formatBytes(headValue)} | ${formatDelta(summary?.median ?? (headValue - baseValue), util.formatBytes)} | ${summary == null ? '-' : util.formatBytes(summary.mad)} | ${util.formatNumber(baseNodeCount)} | ${util.formatNumber(headNodeCount)} |`); + } + + return lines.join('\n'); +} + +function renderLargestRequests(report: BrowserMetricsReport, title: string) { + if (report.summary.network.largestRequests.length === 0) return null; + + const lines = [ + `
${title}`, + '', + '| Resource | Type | Status | Encoded | Decoded |', + '| --- | --- | ---: | ---: | ---: |', + ]; + + for (const request of report.summary.network.largestRequests.slice(0, 10)) { + lines.push(`| \`${escapeCell(truncate(request.url))}\` | ${escapeCell(request.resourceType)} | ${request.status ?? '-'} | ${util.formatBytes(request.encodedBytes)} | ${util.formatBytes(request.decodedBodyBytes)} |`); + } + + lines.push('', '
'); + return lines.join('\n'); +} + +function renderFailedRequests(report: BrowserMetricsReport, title: string) { + if (report.summary.network.failedRequests.length === 0) return null; + + const lines = [ + `
${title}`, + '', + '| Resource | Type | Status | Error |', + '| --- | --- | ---: | --- |', + ]; + + for (const request of report.summary.network.failedRequests.slice(0, 20)) { + lines.push(`| \`${escapeCell(truncate(request.url))}\` | ${escapeCell(request.resourceType)} | ${request.status ?? '-'} | ${escapeCell(request.errorText ?? '')} |`); + } + + lines.push('', '
'); + return lines.join('\n'); +} + +function renderHeadHeapSankey(head: BrowserMetricsReport) { + return heapSnapshotUtil.renderHeapSnapshotSankey({ + summary: head.summary.heapSnapshot, + samples: head.samples.map(sample => ({ + round: sample.round, + data: sample.heapSnapshot, + })), + }, 'Head browser'); +} + +export function renderFrontendBrowserReport(base: BrowserMetricsReport, head: BrowserMetricsReport, options: { + headHeapSnapshotUrl?: string; +} = {}) { + const headHeapSnapshotUrl = options.headHeapSnapshotUrl; + const sampleSummary = base.sampleCount === head.sampleCount + ? `${base.sampleCount} samples per side` + : `${base.sampleCount} base sample(s), ${head.sampleCount} head sample(s)`; + const lines = [ + '## Frontend Browser Metrics', + '', + renderSummaryTable(base, head), + '', + `_Measured ${sampleSummary} with fresh headless Chrome profiles, browser cache disabled, service workers bypassed, and forced V8 GC before each heap snapshot. Values are medians; ± is median absolute deviation. Scenario: sign up, dismiss the initial account setup dialog, create the first timeline note, then wait until that note is visible._`, + '', + '
', + 'Requests by resource type', + '', + renderResourceTypeTable(base, head), + '', + '
', + '', + '
', + 'V8 heap snapshot statistics', + '', + renderHeapSnapshotTable(base, head), + '', + ...(headHeapSnapshotUrl != null && headHeapSnapshotUrl !== '' ? [`[Download representative head heap snapshot](${headHeapSnapshotUrl})`, ''] : []), + '
', + '', + ]; + + for (const section of [ + renderHeadHeapSankey(head), + renderLargestRequests(head, 'Largest representative head requests'), + renderFailedRequests(base, 'Failed representative base requests'), + renderFailedRequests(head, 'Failed representative head requests'), + ]) { + if (section == null) continue; + lines.push(section, ''); + } + + return lines.join('\n').trimEnd() + '\n'; +} + +async function main() { + const [baseFile, headFile, outputFile] = process.argv.slice(2); + if (baseFile == null || headFile == null || outputFile == null) { + throw new Error('Usage: node frontend-browser-report.mts '); + } + + const base = JSON.parse(await readFile(baseFile, 'utf8')) as BrowserMetricsReport; + const head = JSON.parse(await readFile(headFile, 'utf8')) as BrowserMetricsReport; + await writeFile(outputFile, renderFrontendBrowserReport(base, head, { + headHeapSnapshotUrl: process.env.FRONTEND_BROWSER_HEAD_HEAP_SNAPSHOT_ARTIFACT_URL, + })); +} + +if (process.argv[1] != null && import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); +} diff --git a/.github/scripts/measure-frontend-browser-comparison.mts b/.github/scripts/measure-frontend-browser-comparison.mts new file mode 100644 index 0000000000..2b107ae2f5 --- /dev/null +++ b/.github/scripts/measure-frontend-browser-comparison.mts @@ -0,0 +1,1061 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { copyFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import * as util from './utility.mts'; +import { heapSnapshotCategory, type HeapSnapshotData } from './heap-snapshot-util.mts'; + +const [baseDirArg, headDirArg, baseOutputArg, headOutputArg, headHeapSnapshotOutputArg] = process.argv.slice(2); + +const baseUrl = process.env.FRONTEND_BROWSER_METRICS_URL ?? 'http://127.0.0.1:61812'; +const serverReadyTimeoutMs = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SERVER_READY_TIMEOUT_MS', 120_000, 1); +const scenarioTimeoutMs = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SCENARIO_TIMEOUT_MS', 90_000, 1); +const settleMs = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SETTLE_MS', 1_000, 0); +const sampleCount = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SAMPLE_COUNT', 5, 1); +const heapSnapshotBreakdownTopN = util.readIntegerEnv('FRONTEND_BROWSER_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 8, 1); +const headHeapSnapshotWorkDir = resolve('frontend-browser-head-heap-snapshots'); + +type ChromeHandle = { + process: ChildProcessWithoutNullStreams; + port: number; + userDataDir: string; +}; + +type CdpResponse = { + id?: number; + method?: string; + params?: any; + result?: T; + error?: { + code: number; + message: string; + }; +}; + +type NetworkRequest = { + requestId: string; + url: string; + method: string; + resourceType: string; + startedAt: number; + status?: number; + mimeType?: string; + encodedDataLength: number; + decodedBodyLength: number; + fromDiskCache: boolean; + fromServiceWorker: boolean; + finished: boolean; + failed: boolean; + errorText?: string; +}; + +type NetworkSummary = { + requestCount: number; + finishedRequestCount: number; + failedRequestCount: number; + cachedRequestCount: number; + serviceWorkerRequestCount: number; + totalEncodedBytes: number; + totalDecodedBodyBytes: number; + sameOriginEncodedBytes: number; + thirdPartyEncodedBytes: number; + byResourceType: Record; + largestRequests: { + url: string; + method: string; + resourceType: string; + status?: number; + encodedBytes: number; + decodedBodyBytes: number; + }[]; + failedRequests: { + url: string; + method: string; + resourceType: string; + errorText?: string; + status?: number; + }[]; +}; + +type BrowserMeasurement = { + label: string; + timestamp: string; + url: string; + scenario: string; + durationMs: number; + network: NetworkSummary; + performance: { + cdpMetrics: Record; + runtimeHeap?: { + usedSize: number; + totalSize: number; + }; + webVitals: { + firstPaintMs?: number; + firstContentfulPaintMs?: number; + domContentLoadedEventEndMs?: number; + loadEventEndMs?: number; + longTaskCount: number; + longTaskDurationMs: number; + maxLongTaskDurationMs: number; + resourceEntryCount: number; + domElements: number; + }; + }; + heapSnapshot: HeapSnapshotData; +}; + +type BrowserMeasurementSample = BrowserMeasurement & { + round: number; +}; + +type BrowserMetricsReport = { + label: string; + timestamp: string; + url: string; + scenario: string; + sampleCount: number; + aggregation: 'median'; + summary: BrowserMeasurement; + samples: BrowserMeasurementSample[]; +}; + +class CdpClient { + private nextId = 1; + private callbacks = new Map void; + reject: (error: Error) => void; + }>(); + private eventHandlers = new Map void>>(); + private ws: WebSocket; + + private constructor(ws: WebSocket) { + this.ws = ws; + ws.addEventListener('message', event => { + const message = JSON.parse(String(event.data)) as CdpResponse; + if (message.id != null) { + const callback = this.callbacks.get(message.id); + if (callback == null) return; + this.callbacks.delete(message.id); + if (message.error != null) { + callback.reject(new Error(`${message.error.message} (${message.error.code})`)); + } else { + callback.resolve(message.result); + } + return; + } + + if (message.method != null) { + for (const handler of this.eventHandlers.get(message.method) ?? []) { + handler(message.params); + } + } + }); + + ws.addEventListener('close', () => { + for (const callback of this.callbacks.values()) { + callback.reject(new Error('CDP websocket closed')); + } + this.callbacks.clear(); + }); + } + + static async connect(wsUrl: string) { + const ws = new WebSocket(wsUrl); + await new Promise((resolvePromise, reject) => { + ws.addEventListener('open', () => resolvePromise(), { once: true }); + ws.addEventListener('error', () => reject(new Error(`Failed to connect to ${wsUrl}`)), { once: true }); + }); + return new CdpClient(ws); + } + + on(method: string, handler: (params: any) => void) { + const handlers = this.eventHandlers.get(method) ?? new Set(); + handlers.add(handler); + this.eventHandlers.set(method, handlers); + } + + send(method: string, params: Record = {}): Promise { + const id = this.nextId++; + this.ws.send(JSON.stringify({ id, method, params })); + + return new Promise((resolvePromise, reject) => { + this.callbacks.set(id, { + resolve: resolvePromise, + reject, + }); + }); + } + + close() { + this.ws.close(); + } +} + +function sleep(ms: number) { + return new Promise(resolvePromise => setTimeout(resolvePromise, ms)); +} + +async function fetchJson(url: string, options?: RequestInit) { + const response = await fetch(url, options); + if (!response.ok) { + throw new Error(`${url} returned ${response.status}: ${await response.text()}`); + } + return await response.json() as T; +} + +function findChrome() { + const envChrome = process.env.CHROME_BIN ?? process.env.GOOGLE_CHROME_BIN; + if (envChrome != null && envChrome !== '') return envChrome; + + const candidates = process.platform === 'win32' + ? [ + 'chrome.exe', + 'msedge.exe', + ] + : [ + 'google-chrome', + 'google-chrome-stable', + 'chromium', + 'chromium-browser', + ]; + + for (const candidate of candidates) { + const result = spawnSync(candidate, ['--version'], { + stdio: 'ignore', + shell: process.platform === 'win32', + }); + if (result.status === 0) return candidate; + } + + throw new Error('Could not find Chrome or Chromium. Set CHROME_BIN to the browser executable.'); +} + +async function launchChrome(label: string): Promise { + const chrome = findChrome(); + const port = label === 'base' ? 9222 : 9223; + const userDataDir = await mkdtemp(join(tmpdir(), `misskey-browser-metrics-${label}-`)); + const child = spawn(chrome, [ + '--headless=new', + '--disable-gpu', + '--disable-dev-shm-usage', + '--disable-background-networking', + '--disable-default-apps', + '--disable-extensions', + '--disable-sync', + '--metrics-recording-only', + '--no-first-run', + '--no-default-browser-check', + '--no-sandbox', + `--remote-debugging-port=${port}`, + `--user-data-dir=${userDataDir}`, + 'about:blank', + ], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + + child.stdout.on('data', data => process.stderr.write(`[chrome:${label}] ${data}`)); + child.stderr.on('data', data => process.stderr.write(`[chrome:${label}] ${data}`)); + + const startedAt = Date.now(); + while (Date.now() - startedAt < 30_000) { + if (child.exitCode != null) throw new Error(`Chrome exited early with code ${child.exitCode}`); + try { + await fetchJson(`http://127.0.0.1:${port}/json/version`); + return { + process: child, + port, + userDataDir, + }; + } catch { + await sleep(250); + } + } + + throw new Error('Timed out waiting for Chrome DevTools Protocol'); +} + +async function closeChrome(handle: ChromeHandle) { + handle.process.kill(); + await new Promise(resolvePromise => { + if (handle.process.exitCode != null) { + resolvePromise(); + return; + } + handle.process.once('exit', () => resolvePromise()); + setTimeout(() => { + handle.process.kill('SIGKILL'); + resolvePromise(); + }, 5_000).unref(); + }); + await rm(handle.userDataDir, { recursive: true, force: true }); +} + +async function connectPage(port: number) { + const page = await fetchJson<{ webSocketDebuggerUrl: string }>( + `http://127.0.0.1:${port}/json/new?${encodeURIComponent('about:blank')}`, + { method: 'PUT' }, + ).catch(async () => await fetchJson<{ webSocketDebuggerUrl: string }>( + `http://127.0.0.1:${port}/json/new?${encodeURIComponent('about:blank')}`, + )); + return await CdpClient.connect(page.webSocketDebuggerUrl); +} + +function startServer(label: string, repoDir: string) { + process.stderr.write(`[${label}] Starting Misskey test server\n`); + const child = spawn(util.commandName('pnpm'), ['start:test'], { + cwd: repoDir, + env: process.env, + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + }); + child.stdout.on('data', data => process.stderr.write(`[server:${label}] ${data}`)); + child.stderr.on('data', data => process.stderr.write(`[server:${label}] ${data}`)); + return child; +} + +async function stopServer(child: ChildProcessWithoutNullStreams) { + if (child.exitCode != null) return; + + if (process.platform === 'win32') { + spawnSync('taskkill', ['/pid', String(child.pid), '/t', '/f'], { stdio: 'ignore' }); + } else if (child.pid != null) { + try { + process.kill(-child.pid, 'SIGTERM'); + } catch { + child.kill('SIGTERM'); + } + } + + await new Promise(resolvePromise => { + if (child.exitCode != null) { + resolvePromise(); + return; + } + child.once('exit', () => resolvePromise()); + setTimeout(() => { + if (child.pid != null) { + try { + if (process.platform === 'win32') { + spawnSync('taskkill', ['/pid', String(child.pid), '/t', '/f'], { stdio: 'ignore' }); + } else { + process.kill(-child.pid, 'SIGKILL'); + } + } catch { + child.kill('SIGKILL'); + } + } + resolvePromise(); + }, 10_000).unref(); + }); +} + +async function waitForServer(child: ChildProcessWithoutNullStreams) { + const startedAt = Date.now(); + while (Date.now() - startedAt < serverReadyTimeoutMs) { + if (child.exitCode != null) throw new Error(`Misskey server exited early with code ${child.exitCode}`); + try { + const response = await fetch(`${baseUrl}/`, { redirect: 'manual' }); + if (response.status < 500) return; + } catch { + // retry + } + await sleep(1_000); + } + throw new Error(`Timed out waiting for ${baseUrl}`); +} + +async function api(endpoint: string, body: Record) { + const response = await fetch(`${baseUrl}/api/${endpoint}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`/api/${endpoint} returned ${response.status}: ${await response.text()}`); + } + if (response.status === 204) return null; + return await response.json(); +} + +async function prepareInstance() { + await api('reset-db', {}); + await api('admin/accounts/create', { + username: 'admin', + password: 'admin1234', + setupPassword: 'example_password_please_change_this_or_you_will_get_hacked', + }); +} + +function installNetworkTracker(cdp: CdpClient): NetworkRequest[] { + const requests = new Map(); + const requestRows: NetworkRequest[] = []; + + cdp.on('Network.requestWillBeSent', params => { + if (params.request?.url == null) return; + const row: NetworkRequest = { + requestId: params.requestId, + url: params.request.url, + method: params.request.method ?? 'GET', + resourceType: params.type ?? 'Other', + startedAt: params.timestamp ?? 0, + encodedDataLength: 0, + decodedBodyLength: 0, + fromDiskCache: false, + fromServiceWorker: false, + finished: false, + failed: false, + }; + requests.set(params.requestId, row); + requestRows.push(row); + }); + + cdp.on('Network.responseReceived', params => { + const row = requests.get(params.requestId); + if (row == null) return; + row.status = params.response?.status; + row.mimeType = params.response?.mimeType; + row.fromDiskCache = params.response?.fromDiskCache === true; + row.fromServiceWorker = params.response?.fromServiceWorker === true; + }); + + cdp.on('Network.dataReceived', params => { + const row = requests.get(params.requestId); + if (row == null) return; + row.decodedBodyLength += params.dataLength ?? 0; + row.encodedDataLength += params.encodedDataLength ?? 0; + }); + + cdp.on('Network.loadingFinished', params => { + const row = requests.get(params.requestId); + if (row == null) return; + row.finished = true; + row.encodedDataLength = Math.max(row.encodedDataLength, params.encodedDataLength ?? 0); + }); + + cdp.on('Network.loadingFailed', params => { + const row = requests.get(params.requestId); + if (row == null) return; + row.failed = true; + row.finished = true; + row.errorText = params.errorText; + }); + + return requestRows; +} + +function isMeasurableRequest(row: NetworkRequest) { + return !row.url.startsWith('data:') && !row.url.startsWith('blob:') && !row.url.startsWith('devtools:'); +} + +function summarizeNetwork(requestRows: NetworkRequest[]): NetworkSummary { + const origin = new URL(baseUrl).origin; + const rows = requestRows.filter(isMeasurableRequest); + const byResourceType = {} as NetworkSummary['byResourceType']; + + for (const row of rows) { + const summary = byResourceType[row.resourceType] ?? { + requests: 0, + encodedBytes: 0, + decodedBodyBytes: 0, + }; + summary.requests += 1; + summary.encodedBytes += row.encodedDataLength; + summary.decodedBodyBytes += row.decodedBodyLength; + byResourceType[row.resourceType] = summary; + } + + function isSameOrigin(url: string) { + try { + return new URL(url).origin === origin; + } catch { + return false; + } + } + + return { + requestCount: rows.length, + finishedRequestCount: rows.filter(row => row.finished).length, + failedRequestCount: rows.filter(row => row.failed).length, + cachedRequestCount: rows.filter(row => row.fromDiskCache).length, + serviceWorkerRequestCount: rows.filter(row => row.fromServiceWorker).length, + totalEncodedBytes: rows.reduce((sum, row) => sum + row.encodedDataLength, 0), + totalDecodedBodyBytes: rows.reduce((sum, row) => sum + row.decodedBodyLength, 0), + sameOriginEncodedBytes: rows + .filter(row => isSameOrigin(row.url)) + .reduce((sum, row) => sum + row.encodedDataLength, 0), + thirdPartyEncodedBytes: rows + .filter(row => !isSameOrigin(row.url)) + .reduce((sum, row) => sum + row.encodedDataLength, 0), + byResourceType, + largestRequests: rows + .toSorted((a, b) => b.encodedDataLength - a.encodedDataLength) + .slice(0, 15) + .map(row => ({ + url: row.url, + method: row.method, + resourceType: row.resourceType, + status: row.status, + encodedBytes: row.encodedDataLength, + decodedBodyBytes: row.decodedBodyLength, + })), + failedRequests: rows + .filter(row => row.failed) + .map(row => ({ + url: row.url, + method: row.method, + resourceType: row.resourceType, + errorText: row.errorText, + status: row.status, + })), + }; +} + +async function evaluate(cdp: CdpClient, expression: string, timeoutMs = 30_000): Promise { + const result = await cdp.send<{ + result: { value: T }; + exceptionDetails?: unknown; + }>('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + timeout: timeoutMs, + }); + + if (result.exceptionDetails != null) { + throw new Error(`Runtime.evaluate failed: ${JSON.stringify(result.exceptionDetails)}`); + } + + return result.result.value; +} + +function selectorReadyExpression(selector: string, options: { visible?: boolean; enabled?: boolean } = {}) { + return `(() => { + const el = document.querySelector(${JSON.stringify(selector)}); + if (el == null) return false; + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + if (${options.visible === true ? 'true' : 'false'} && (style.visibility === 'hidden' || style.display === 'none' || rect.width === 0 || rect.height === 0)) return false; + if (${options.enabled === true ? 'true' : 'false'} && (el.disabled || el.getAttribute('aria-disabled') === 'true')) return false; + return true; + })()`; +} + +async function waitForSelector(cdp: CdpClient, selector: string, options: { timeoutMs?: number; visible?: boolean; enabled?: boolean } = {}) { + const startedAt = Date.now(); + const timeoutMs = options.timeoutMs ?? scenarioTimeoutMs; + while (Date.now() - startedAt < timeoutMs) { + const ready = await evaluate(cdp, selectorReadyExpression(selector, options), 5_000); + if (ready) return true; + await sleep(250); + } + return false; +} + +async function waitForAnySelector(cdp: CdpClient, selectors: string[], options: { timeoutMs?: number; visible?: boolean; enabled?: boolean } = {}) { + const startedAt = Date.now(); + const timeoutMs = options.timeoutMs ?? scenarioTimeoutMs; + while (Date.now() - startedAt < timeoutMs) { + for (const selector of selectors) { + const ready = await evaluate(cdp, selectorReadyExpression(selector, options), 5_000); + if (ready) return selector; + } + await sleep(250); + } + return null; +} + +async function click(cdp: CdpClient, selector: string) { + const found = await waitForSelector(cdp, selector, { visible: true, enabled: true }); + if (!found) throw new Error(`Selector was not clickable: ${selector}`); + await evaluate(cdp, `(() => { + const el = document.querySelector(${JSON.stringify(selector)}); + if (el == null) throw new Error('Element not found'); + el.scrollIntoView({ block: 'center', inline: 'center' }); + el.click(); + })()`); +} + +async function maybeClick(cdp: CdpClient, selector: string, timeoutMs = 3_000) { + if (await waitForSelector(cdp, selector, { visible: true, enabled: true, timeoutMs })) { + await click(cdp, selector); + return true; + } + return false; +} + +async function setValue(cdp: CdpClient, selector: string, value: string) { + const found = await waitForSelector(cdp, selector, { visible: true, enabled: true }); + if (!found) throw new Error(`Selector was not editable: ${selector}`); + await evaluate(cdp, `(() => { + const el = document.querySelector(${JSON.stringify(selector)}); + if (el == null) throw new Error('Element not found'); + el.scrollIntoView({ block: 'center', inline: 'center' }); + el.focus(); + const proto = Object.getPrototypeOf(el); + const descriptor = Object.getOwnPropertyDescriptor(proto, 'value'); + if (descriptor?.set != null) { + descriptor.set.call(el, ${JSON.stringify(value)}); + } else { + el.value = ${JSON.stringify(value)}; + } + el.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: ${JSON.stringify(value)} })); + el.dispatchEvent(new Event('change', { bubbles: true })); + })()`); +} + +async function waitForText(cdp: CdpClient, text: string, timeoutMs = scenarioTimeoutMs) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const found = await evaluate(cdp, `document.body?.innerText?.includes(${JSON.stringify(text)}) === true`, 5_000); + if (found) return true; + await sleep(250); + } + return false; +} + +async function runSignupAndPostScenario(cdp: CdpClient) { + const noteText = `Frontend browser metrics ${Date.now()}`; + + await cdp.send('Page.navigate', { url: `${baseUrl}/` }); + const initialSelector = await waitForAnySelector(cdp, ['[data-cy-signup]', '[data-cy-open-post-form]'], { visible: true, timeoutMs: scenarioTimeoutMs }); + if (initialSelector == null) throw new Error('Timed out waiting for the signup or timeline entry point'); + + if (await waitForSelector(cdp, '[data-cy-signup]', { visible: true, enabled: true, timeoutMs: 5_000 })) { + await click(cdp, '[data-cy-signup]'); + + if (await waitForSelector(cdp, '[data-cy-signup-rules-continue]', { visible: true, timeoutMs: 5_000 })) { + await click(cdp, '[data-cy-signup-rules-notes-agree] [data-cy-switch-toggle]'); + await maybeClick(cdp, '[data-cy-modal-dialog-ok]', 5_000); + await click(cdp, '[data-cy-signup-rules-continue]'); + } + + await setValue(cdp, '[data-cy-signup-username] input', 'alice'); + await setValue(cdp, '[data-cy-signup-password] input', 'alice1234'); + await setValue(cdp, '[data-cy-signup-password-retype] input', 'alice1234'); + if (await waitForSelector(cdp, '[data-cy-signup-invitation-code] input', { visible: true, enabled: true, timeoutMs: 2_000 })) { + await setValue(cdp, '[data-cy-signup-invitation-code] input', 'test-invitation-code'); + } + await click(cdp, '[data-cy-signup-submit]'); + } + + const firstReadySelector = await waitForAnySelector(cdp, [ + '[data-cy-user-setup] [data-cy-modal-window-close]', + '[data-cy-open-post-form]', + ], { visible: true, enabled: true, timeoutMs: scenarioTimeoutMs }); + if (firstReadySelector == null) throw new Error('Timed out waiting for signed-in home timeline'); + + if (firstReadySelector === '[data-cy-user-setup] [data-cy-modal-window-close]') { + await click(cdp, '[data-cy-user-setup] [data-cy-modal-window-close]'); + await maybeClick(cdp, '[data-cy-modal-dialog-ok]', 5_000); + } + + await click(cdp, '[data-cy-open-post-form]'); + await setValue(cdp, '[data-cy-post-form-text]', noteText); + await click(cdp, '[data-cy-open-post-form-submit]'); + + if (!await waitForText(cdp, noteText, scenarioTimeoutMs)) { + throw new Error('The first timeline note did not appear'); + } + + await sleep(settleMs); +} + +async function collectPerformance(cdp: CdpClient): Promise { + const cdpMetricsResult = await cdp.send<{ metrics: { name: string; value: number }[] }>('Performance.getMetrics'); + const cdpMetrics = Object.fromEntries(cdpMetricsResult.metrics.map(metric => [metric.name, metric.value])); + const runtimeHeap = await cdp.send<{ usedSize: number; totalSize: number }>('Runtime.getHeapUsage').catch(() => undefined); + const webVitals = await evaluate(cdp, `(() => { + const navigation = performance.getEntriesByType('navigation')[0]; + const paintEntries = Object.fromEntries(performance.getEntriesByType('paint').map(entry => [entry.name, entry.startTime])); + const longTasks = performance.getEntriesByType('longtask'); + const resourceEntries = performance.getEntriesByType('resource'); + return { + firstPaintMs: paintEntries['first-paint'], + firstContentfulPaintMs: paintEntries['first-contentful-paint'], + domContentLoadedEventEndMs: navigation?.domContentLoadedEventEnd, + loadEventEndMs: navigation?.loadEventEnd, + longTaskCount: longTasks.length, + longTaskDurationMs: longTasks.reduce((sum, entry) => sum + entry.duration, 0), + maxLongTaskDurationMs: longTasks.reduce((max, entry) => Math.max(max, entry.duration), 0), + resourceEntryCount: resourceEntries.length, + domElements: document.getElementsByTagName('*').length, + }; + })()`); + + return { + cdpMetrics, + runtimeHeap, + webVitals, + }; +} + +function emptyHeapSnapshotData(): HeapSnapshotData { + const categories = {} as HeapSnapshotData['categories']; + const nodeCounts = {} as HeapSnapshotData['nodeCounts']; + for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) { + categories[category] = 0; + nodeCounts[category] = 0; + } + return { + categories, + nodeCounts, + breakdowns: {} as HeapSnapshotData['breakdowns'], + }; +} + +function categorizeHeapNode(type: string, name: string): keyof typeof heapSnapshotCategory { + if (/^(ArrayBuffer|SharedArrayBuffer|DataView|(?:Big)?(?:Int|Uint|Float)(?:8|16|32|64)?(?:Clamped)?Array)$/u.test(name)) return 'typedArrays'; + if (type === 'code' || type === 'closure') return 'code'; + if (type === 'string' || type === 'concatenated string' || type === 'sliced string' || type === 'symbol') return 'strings'; + if (type === 'array') return 'jsArrays'; + if (type === 'hidden' || type === 'synthetic' || type === 'object shape') return 'systemObjects'; + if (type === 'native') return 'otherNonJsObjects'; + if (type === 'object' || type === 'regexp' || type === 'number' || type === 'bigint') return 'otherJsObjects'; + return 'otherNonJsObjects'; +} + +function collapseBreakdown(entries: Map) { + const sorted = [...entries] + .filter(([, value]) => value > 0) + .toSorted((a, b) => b[1] - a[1]); + const topEntries = sorted.slice(0, heapSnapshotBreakdownTopN); + const otherValue = sorted + .slice(heapSnapshotBreakdownTopN) + .reduce((sum, [, value]) => sum + value, 0); + const collapsed = Object.fromEntries(topEntries); + if (otherValue > 0) collapsed.Other = otherValue; + return collapsed; +} + +function summarizeHeapSnapshot(snapshot: any): HeapSnapshotData { + const result = emptyHeapSnapshotData(); + const nodeFields = snapshot.snapshot.meta.node_fields as string[]; + const nodeTypes = snapshot.snapshot.meta.node_types as unknown[][]; + const nodes = snapshot.nodes as number[]; + const strings = snapshot.strings as string[]; + const fieldCount = nodeFields.length; + const typeOffset = nodeFields.indexOf('type'); + const nameOffset = nodeFields.indexOf('name'); + const selfSizeOffset = nodeFields.indexOf('self_size'); + const typeNames = nodeTypes[typeOffset] as string[]; + const breakdownMaps = {} as Record>; + + for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) { + if (category !== 'total') breakdownMaps[category] = new Map(); + } + + for (let offset = 0; offset < nodes.length; offset += fieldCount) { + const type = typeNames[nodes[offset + typeOffset]] ?? 'unknown'; + const name = strings[nodes[offset + nameOffset]] ?? ''; + const selfSize = nodes[offset + selfSizeOffset] ?? 0; + const category = categorizeHeapNode(type, name); + + result.categories.total += selfSize; + result.nodeCounts.total += 1; + result.categories[category] += selfSize; + result.nodeCounts[category] += 1; + + const label = `${type}: ${name || '(anonymous)'}`; + breakdownMaps[category].set(label, (breakdownMaps[category].get(label) ?? 0) + selfSize); + } + + result.breakdowns = {} as HeapSnapshotData['breakdowns']; + for (const [category, entries] of Object.entries(breakdownMaps) as [keyof typeof heapSnapshotCategory, Map][]) { + const collapsed = collapseBreakdown(entries); + if (Object.keys(collapsed).length > 0) { + result.breakdowns[category] = collapsed; + } + } + + return result; +} + +function finiteMedian(values: (number | null | undefined)[], defaultValue = 0) { + const finiteValues = values.filter(value => Number.isFinite(value)) as number[]; + if (finiteValues.length === 0) return defaultValue; + return util.median(finiteValues); +} + +function selectRepresentativeSample(samples: BrowserMeasurementSample[], getValue: (sample: BrowserMeasurementSample) => number) { + const medianValue = finiteMedian(samples.map(getValue)); + let selected: { sample: BrowserMeasurementSample; distance: number } | null = null; + + for (const sample of samples) { + const value = getValue(sample); + if (!Number.isFinite(value)) continue; + const distance = Math.abs(value - medianValue); + if (selected == null || distance < selected.distance || (distance === selected.distance && sample.round < selected.sample.round)) { + selected = { + sample, + distance, + }; + } + } + + return selected?.sample ?? samples[0]; +} + +function summarizeResourceType(samples: BrowserMeasurementSample[], resourceType: string) { + return { + requests: finiteMedian(samples.map(sample => sample.network.byResourceType[resourceType]?.requests)), + encodedBytes: finiteMedian(samples.map(sample => sample.network.byResourceType[resourceType]?.encodedBytes)), + decodedBodyBytes: finiteMedian(samples.map(sample => sample.network.byResourceType[resourceType]?.decodedBodyBytes)), + }; +} + +function summarizeNetworkSamples(samples: BrowserMeasurementSample[]): NetworkSummary { + const resourceTypes = new Set(); + for (const sample of samples) { + for (const resourceType of Object.keys(sample.network.byResourceType)) { + resourceTypes.add(resourceType); + } + } + + const representative = selectRepresentativeSample(samples, sample => sample.network.totalEncodedBytes); + const byResourceType = {} as NetworkSummary['byResourceType']; + for (const resourceType of resourceTypes) { + byResourceType[resourceType] = summarizeResourceType(samples, resourceType); + } + + return { + requestCount: finiteMedian(samples.map(sample => sample.network.requestCount)), + finishedRequestCount: finiteMedian(samples.map(sample => sample.network.finishedRequestCount)), + failedRequestCount: finiteMedian(samples.map(sample => sample.network.failedRequestCount)), + cachedRequestCount: finiteMedian(samples.map(sample => sample.network.cachedRequestCount)), + serviceWorkerRequestCount: finiteMedian(samples.map(sample => sample.network.serviceWorkerRequestCount)), + totalEncodedBytes: finiteMedian(samples.map(sample => sample.network.totalEncodedBytes)), + totalDecodedBodyBytes: finiteMedian(samples.map(sample => sample.network.totalDecodedBodyBytes)), + sameOriginEncodedBytes: finiteMedian(samples.map(sample => sample.network.sameOriginEncodedBytes)), + thirdPartyEncodedBytes: finiteMedian(samples.map(sample => sample.network.thirdPartyEncodedBytes)), + byResourceType, + largestRequests: representative.network.largestRequests, + failedRequests: representative.network.failedRequests, + }; +} + +function summarizePerformanceSamples(samples: BrowserMeasurementSample[]): BrowserMeasurement['performance'] { + const cdpMetricKeys = new Set(); + for (const sample of samples) { + for (const key of Object.keys(sample.performance.cdpMetrics)) { + cdpMetricKeys.add(key); + } + } + + const cdpMetrics = {} as Record; + for (const key of cdpMetricKeys) { + cdpMetrics[key] = finiteMedian(samples.map(sample => sample.performance.cdpMetrics[key])); + } + + const webVitalKeys = [ + 'firstPaintMs', + 'firstContentfulPaintMs', + 'domContentLoadedEventEndMs', + 'loadEventEndMs', + 'longTaskCount', + 'longTaskDurationMs', + 'maxLongTaskDurationMs', + 'resourceEntryCount', + 'domElements', + ] as const satisfies (keyof BrowserMeasurement['performance']['webVitals'])[]; + + const webVitals = {} as BrowserMeasurement['performance']['webVitals']; + for (const key of webVitalKeys) { + webVitals[key] = finiteMedian(samples.map(sample => sample.performance.webVitals[key])); + } + + return { + cdpMetrics, + runtimeHeap: { + usedSize: finiteMedian(samples.map(sample => sample.performance.runtimeHeap?.usedSize)), + totalSize: finiteMedian(samples.map(sample => sample.performance.runtimeHeap?.totalSize)), + }, + webVitals, + }; +} + +function summarizeHeapSnapshotSamples(samples: BrowserMeasurementSample[]) { + const summary = emptyHeapSnapshotData(); + + for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) { + summary.categories[category] = finiteMedian(samples.map(sample => sample.heapSnapshot.categories[category])); + summary.nodeCounts[category] = finiteMedian(samples.map(sample => sample.heapSnapshot.nodeCounts[category])); + } + + summary.breakdowns = {} as HeapSnapshotData['breakdowns']; + for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) { + if (category === 'total') continue; + + const childKeys = new Set(); + for (const sample of samples) { + for (const childKey of Object.keys(sample.heapSnapshot.breakdowns?.[category] ?? {})) { + childKeys.add(childKey); + } + } + + const childValues = new Map(); + for (const childKey of childKeys) { + childValues.set(childKey, finiteMedian(samples.map(sample => sample.heapSnapshot.breakdowns?.[category]?.[childKey]))); + } + + const collapsed = collapseBreakdown(childValues); + if (Object.keys(collapsed).length > 0) { + summary.breakdowns[category] = collapsed; + } + } + + return summary; +} + +function summarizeSamples(label: 'base' | 'head', samples: BrowserMeasurementSample[]): BrowserMetricsReport { + if (samples.length === 0) throw new Error(`No browser metric samples for ${label}`); + const representative = selectRepresentativeSample(samples, sample => sample.network.totalEncodedBytes); + const summary: BrowserMeasurement = { + label, + timestamp: new Date().toISOString(), + url: baseUrl, + scenario: representative.scenario, + durationMs: finiteMedian(samples.map(sample => sample.durationMs)), + network: summarizeNetworkSamples(samples), + performance: summarizePerformanceSamples(samples), + heapSnapshot: summarizeHeapSnapshotSamples(samples), + }; + + return { + label, + timestamp: new Date().toISOString(), + url: baseUrl, + scenario: representative.scenario, + sampleCount: samples.length, + aggregation: 'median', + summary, + samples, + }; +} + +async function takeHeapSnapshot(cdp: CdpClient, savePath?: string) { + const chunks: string[] = []; + cdp.on('HeapProfiler.addHeapSnapshotChunk', params => { + chunks.push(params.chunk); + }); + + await cdp.send('HeapProfiler.enable'); + await cdp.send('HeapProfiler.collectGarbage'); + await cdp.send('HeapProfiler.takeHeapSnapshot', { reportProgress: false }); + + const content = chunks.join(''); + if (savePath != null) { + await writeFile(savePath, content); + } + + return summarizeHeapSnapshot(JSON.parse(content)); +} + +async function measureSample(label: 'base' | 'head', round: number, heapSnapshotSavePath?: string) { + let chrome: ChromeHandle | null = null; + let cdp: CdpClient | null = null; + + try { + await prepareInstance(); + + chrome = await launchChrome(label); + cdp = await connectPage(chrome.port); + + const networkRequests = installNetworkTracker(cdp); + await cdp.send('Network.enable'); + await cdp.send('Network.setCacheDisabled', { cacheDisabled: true }); + await cdp.send('Network.setBypassServiceWorker', { bypass: true }); + await cdp.send('Page.enable'); + await cdp.send('Runtime.enable'); + await cdp.send('Performance.enable'); + + const startedAt = Date.now(); + await runSignupAndPostScenario(cdp); + const durationMs = Date.now() - startedAt; + const performance = await collectPerformance(cdp); + const heapSnapshot = await takeHeapSnapshot(cdp, heapSnapshotSavePath); + const measurement: BrowserMeasurementSample = { + label, + round, + timestamp: new Date().toISOString(), + url: baseUrl, + scenario: 'fresh browser signup, first timeline note, after the note becomes visible', + durationMs, + network: summarizeNetwork(networkRequests), + performance, + heapSnapshot, + }; + + return measurement; + } finally { + cdp?.close(); + if (chrome != null) await closeChrome(chrome); + } +} + +function headHeapSnapshotPath(round: number) { + return join(headHeapSnapshotWorkDir, `round-${round}.heapsnapshot`); +} + +async function saveRepresentativeHeadHeapSnapshot(report: BrowserMetricsReport, outputPath: string) { + const representative = selectRepresentativeSample(report.samples, sample => sample.heapSnapshot.categories.total); + await copyFile(headHeapSnapshotPath(representative.round), outputPath); + process.stderr.write(`[head] Selected round ${representative.round} heap snapshot for artifact\n`); + await rm(headHeapSnapshotWorkDir, { recursive: true, force: true }); +} + +async function measureRepo(label: 'base' | 'head', repoDir: string, outputPath: string, heapSnapshotSavePath?: string) { + let server: ChildProcessWithoutNullStreams | null = null; + + try { + server = startServer(label, repoDir); + await waitForServer(server); + + if (label === 'head' && heapSnapshotSavePath != null) { + await rm(headHeapSnapshotWorkDir, { recursive: true, force: true }); + await mkdir(headHeapSnapshotWorkDir, { recursive: true }); + } + + const samples: BrowserMeasurementSample[] = []; + for (let round = 1; round <= sampleCount; round++) { + process.stderr.write(`[${label}] Measuring browser metrics sample ${round}/${sampleCount}\n`); + samples.push(await measureSample( + label, + round, + label === 'head' && heapSnapshotSavePath != null ? headHeapSnapshotPath(round) : undefined, + )); + } + + const report = summarizeSamples(label, samples); + await writeFile(outputPath, JSON.stringify(report, null, '\t')); + process.stderr.write(`[${label}] Wrote browser metrics report to ${outputPath}\n`); + + if (label === 'head' && heapSnapshotSavePath != null) { + await saveRepresentativeHeadHeapSnapshot(report, heapSnapshotSavePath); + } + } finally { + if (server != null) await stopServer(server); + } +} + +async function main() { + if (baseDirArg == null || headDirArg == null || baseOutputArg == null || headOutputArg == null) { + throw new Error('Usage: node measure-frontend-browser-comparison.mts [head-heap-snapshot.heapsnapshot]'); + } + + await measureRepo('base', resolve(baseDirArg), resolve(baseOutputArg)); + await measureRepo('head', resolve(headDirArg), resolve(headOutputArg), headHeapSnapshotOutputArg == null ? undefined : resolve(headHeapSnapshotOutputArg)); +} + +await main(); diff --git a/.github/workflows/frontend-browser-metrics-report-comment.yml b/.github/workflows/frontend-browser-metrics-report-comment.yml new file mode 100644 index 0000000000..0312092a5b --- /dev/null +++ b/.github/workflows/frontend-browser-metrics-report-comment.yml @@ -0,0 +1,44 @@ +name: frontend-browser-metrics-report-comment + +on: + workflow_run: + workflows: + - frontend-browser-metrics-report + types: + - completed + +permissions: + actions: read + contents: read + issues: write + pull-requests: write + +jobs: + comment: + name: Comment frontend browser metrics report + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + concurrency: + group: frontend-browser-metrics-report-comment-${{ github.event.workflow_run.id }} + cancel-in-progress: true + steps: + - name: Download browser metrics report + uses: actions/download-artifact@v8 + with: + name: frontend-browser-metrics-report + path: ${{ runner.temp }}/frontend-browser-metrics-report + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Load PR number + id: load-pr-number + shell: bash + run: echo "pr-number=$(cat "$RUNNER_TEMP/frontend-browser-metrics-report/pr-number.txt")" >> "$GITHUB_OUTPUT" + + - name: Comment on pull request + uses: thollander/actions-comment-pull-request@v3 + with: + pr-number: ${{ steps.load-pr-number.outputs.pr-number }} + comment-tag: frontend_browser_metrics_report + file-path: ${{ runner.temp }}/frontend-browser-metrics-report/frontend-browser-metrics-report.md diff --git a/.github/workflows/frontend-browser-metrics-report.yml b/.github/workflows/frontend-browser-metrics-report.yml new file mode 100644 index 0000000000..ab032d0256 --- /dev/null +++ b/.github/workflows/frontend-browser-metrics-report.yml @@ -0,0 +1,173 @@ +name: frontend-browser-metrics-report + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + paths: + - packages/frontend/** + - packages/frontend-shared/** + - packages/frontend-builder/** + - packages/backend/** + - packages/i18n/** + - packages/icons-subsetter/** + - packages/misskey-js/** + - packages/misskey-reversi/** + - packages/misskey-bubble-game/** + - package.json + - pnpm-lock.yaml + - pnpm-workspace.yaml + - .node-version + - .github/misskey/test.yml + - .github/scripts/utility.mts + - .github/scripts/frontend-browser-report.mts + - .github/scripts/heap-snapshot-util.mts + - .github/scripts/measure-frontend-browser-comparison.mts + - .github/workflows/frontend-browser-metrics-report.yml + - .github/workflows/frontend-browser-metrics-report-comment.yml + +permissions: + contents: read + pull-requests: read + +concurrency: + group: frontend-browser-metrics-report-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + report: + name: Measure frontend browser metrics + runs-on: ubuntu-latest + timeout-minutes: 90 + + services: + postgres: + image: postgres:18 + ports: + - 54312:5432 + env: + POSTGRES_DB: test-misskey + POSTGRES_HOST_AUTH_METHOD: trust + redis: + image: redis:8 + ports: + - 56312:6379 + + steps: + - name: Checkout base + uses: actions/checkout@v6.0.2 + with: + repository: ${{ github.event.pull_request.base.repo.full_name }} + ref: ${{ github.event.pull_request.base.sha }} + path: before + submodules: true + + - name: Checkout pull request + uses: actions/checkout@v6.0.2 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: after + submodules: true + + - name: Setup pnpm + uses: pnpm/action-setup@v6.0.3 + with: + package_json_file: after/package.json + + - name: Setup Node.js + uses: actions/setup-node@v6.4.0 + with: + node-version-file: after/.node-version + cache: pnpm + cache-dependency-path: | + before/pnpm-lock.yaml + after/pnpm-lock.yaml + + - name: Install dependencies for base + working-directory: before + run: pnpm i --frozen-lockfile + + - name: Configure base + working-directory: before + run: cp .github/misskey/test.yml .config + + - name: Build base + working-directory: before + run: pnpm build + + - name: Install dependencies for pull request + working-directory: after + run: pnpm i --frozen-lockfile + + - name: Configure pull request + working-directory: after + run: cp .github/misskey/test.yml .config + + - name: Build pull request + working-directory: after + run: pnpm build + + - name: Measure frontend browser metrics + shell: bash + env: + FRONTEND_BROWSER_METRICS_SAMPLE_COUNT: 5 + FRONTEND_BROWSER_METRICS_SCENARIO_TIMEOUT_MS: 120000 + FRONTEND_BROWSER_METRICS_SETTLE_MS: 1000 + run: | + REPORT_DIR="$RUNNER_TEMP/frontend-browser-metrics-report" + mkdir -p "$REPORT_DIR" + node after/.github/scripts/measure-frontend-browser-comparison.mts before after "$REPORT_DIR/before-browser.json" "$REPORT_DIR/after-browser.json" "$REPORT_DIR/head-heap-snapshot.heapsnapshot" + + - name: Upload browser head heap snapshot + id: upload-browser-head-heap-snapshot + uses: actions/upload-artifact@v7 + with: + name: frontend-browser-metrics-head-heap-snapshot + path: ${{ runner.temp }}/frontend-browser-metrics-report/head-heap-snapshot.heapsnapshot + if-no-files-found: error + retention-days: 7 + + - name: Generate browser metrics report + shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + FRONTEND_BROWSER_HEAD_HEAP_SNAPSHOT_ARTIFACT_URL: ${{ steps.upload-browser-head-heap-snapshot.outputs.artifact-url }} + run: | + REPORT_DIR="$RUNNER_TEMP/frontend-browser-metrics-report" + test -s "$REPORT_DIR/before-browser.json" + test -s "$REPORT_DIR/after-browser.json" + node after/.github/scripts/frontend-browser-report.mts "$REPORT_DIR/before-browser.json" "$REPORT_DIR/after-browser.json" "$REPORT_DIR/frontend-browser-metrics-report.md" + printf '%s\n' "$PR_NUMBER" > "$REPORT_DIR/pr-number.txt" + printf '%s\n' "$BASE_SHA" > "$REPORT_DIR/base-sha.txt" + printf '%s\n' "$HEAD_SHA" > "$REPORT_DIR/head-sha.txt" + printf '%s\n' "${{ github.event.pull_request.html_url }}" > "$REPORT_DIR/pr-url.txt" + + - name: Check browser metrics report + shell: bash + run: | + REPORT_DIR="$RUNNER_TEMP/frontend-browser-metrics-report" + test -s "$REPORT_DIR/frontend-browser-metrics-report.md" + test -s "$REPORT_DIR/pr-number.txt" + test -s "$REPORT_DIR/head-sha.txt" + cat "$REPORT_DIR/frontend-browser-metrics-report.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload browser metrics report + uses: actions/upload-artifact@v7 + with: + name: frontend-browser-metrics-report + path: | + ${{ runner.temp }}/frontend-browser-metrics-report/before-browser.json + ${{ runner.temp }}/frontend-browser-metrics-report/after-browser.json + ${{ runner.temp }}/frontend-browser-metrics-report/frontend-browser-metrics-report.md + ${{ runner.temp }}/frontend-browser-metrics-report/pr-number.txt + ${{ runner.temp }}/frontend-browser-metrics-report/base-sha.txt + ${{ runner.temp }}/frontend-browser-metrics-report/head-sha.txt + ${{ runner.temp }}/frontend-browser-metrics-report/pr-url.txt + if-no-files-found: error + retention-days: 7