diff --git a/.github/scripts/chrome.mts b/.github/scripts/chrome.mts index 5d07ac6efe..2be10bd261 100644 --- a/.github/scripts/chrome.mts +++ b/.github/scripts/chrome.mts @@ -7,6 +7,7 @@ import { createRequire } from 'node:module'; import { writeFile } from 'node:fs/promises'; import type { Browser, BrowserContext, CDPSession, Page } from 'playwright'; import type { HeapSnapshotData } from './heap-snapshot-util.mts'; +import * as util from './utility.mts'; export type NetworkRequest = { requestId: string; @@ -89,11 +90,65 @@ export type TabMemory = { totalBytes: number; }; +export type BrowserConsoleMetrics = { + log: number; + warn: number; + error: number; + info: number; +}; + +export type BrowserDiagnostics = { + pageErrorCount: number; + console: BrowserConsoleMetrics; +}; + +export function createBrowserDiagnostics(): BrowserDiagnostics { + return { + pageErrorCount: 0, + console: { + log: 0, + warn: 0, + error: 0, + info: 0, + }, + }; +} + +export function recordPageError(diagnostics: BrowserDiagnostics) { + diagnostics.pageErrorCount += 1; +} + +function isBrowserConsoleMetricKey(value: string): value is keyof BrowserConsoleMetrics { + return value === 'log' || value === 'warn' || value === 'error' || value === 'info'; +} + +export function recordConsoleMessageType(diagnostics: BrowserDiagnostics, messageType: string) { + const metricKey = messageType === 'warning' ? 'warn' : messageType; + if (!isBrowserConsoleMetricKey(metricKey)) return; + diagnostics.console[metricKey] += 1; +} + +export function summarizeBrowserDiagnostics(samples: BrowserDiagnostics[]): BrowserDiagnostics { + if (samples.length === 0) throw new Error('No browser diagnostic samples'); + const median = (select: (sample: BrowserDiagnostics) => number) => util.median(samples.map(select)); + + return { + pageErrorCount: median(sample => sample.pageErrorCount), + console: { + log: median(sample => sample.console.log), + warn: median(sample => sample.console.warn), + error: median(sample => sample.console.error), + info: median(sample => sample.console.info), + }, + }; +} + export type BrowserMeasurement = { label: string; timestamp: string; url: string; scenario: string; + diagnostics: BrowserDiagnostics; durationMs: number; network: NetworkSummary; performance: { @@ -149,6 +204,7 @@ type PlaywrightBrowserOptions = { export class HeadlessChromeController { public networkRequests: NetworkRequest[] = []; public webSocketConnections: WebSocketConnection[] = []; + private readonly diagnostics = createBrowserDiagnostics(); private readonly browser: Browser; private readonly context: BrowserContext; public readonly page: Page; @@ -168,6 +224,12 @@ export class HeadlessChromeController { this.cdp = cdp; this.page.setDefaultTimeout(options.scenarioTimeoutMs); this.page.setDefaultNavigationTimeout(options.scenarioTimeoutMs); + this.page.on('pageerror', () => { + recordPageError(this.diagnostics); + }); + this.page.on('console', message => { + recordConsoleMessageType(this.diagnostics, message.type()); + }); } static async create(label: string, options: PlaywrightBrowserOptions): Promise { @@ -376,6 +438,13 @@ export class HeadlessChromeController { ]) as T; } + public collectDiagnostics(): BrowserDiagnostics { + return { + pageErrorCount: this.diagnostics.pageErrorCount, + console: { ...this.diagnostics.console }, + }; + } + public async collectPerformance(): Promise { const cdpMetricsResult = await this.cdp.send<{ metrics: { name: string; value: number }[] }>('Performance.getMetrics'); const cdpMetrics = Object.fromEntries(cdpMetricsResult.metrics.map(metric => [metric.name, metric.value])); diff --git a/.github/scripts/frontend-browser-report.mts b/.github/scripts/frontend-browser-report.mts index 755b29bc88..829041f1d0 100644 --- a/.github/scripts/frontend-browser-report.mts +++ b/.github/scripts/frontend-browser-report.mts @@ -8,13 +8,14 @@ import { pathToFileURL } from 'node:url'; import * as util from './utility.mts'; import * as heapSnapshotUtil from './heap-snapshot-util.mts'; import type { HeapSnapshotData, HeapSnapshotReport } from './heap-snapshot-util.mts'; -import type { NetworkRequest } from './chrome.mts'; +import type { BrowserDiagnostics, NetworkRequest } from './chrome.mts'; export type BrowserMeasurement = { label: string; timestamp: string; url: string; scenario: string; + diagnostics: BrowserDiagnostics; durationMs: number; network: { requestCount: number; @@ -171,8 +172,19 @@ function getMetric(report: BrowserMeasurement, key: string) { return report.performance.cdpMetrics[key]; } +export function renderBrowserDiagnosticsRows(base: BrowserMetricsReport, head: BrowserMetricsReport, all = false) { + return [ + metricRow('Page errors', base, head, summary => summary.diagnostics.pageErrorCount, sample => sample.diagnostics.pageErrorCount, util.formatNumber, 1, !all), + metricRow('Console log', base, head, summary => summary.diagnostics.console.log, sample => sample.diagnostics.console.log, util.formatNumber, 1, !all), + metricRow('Console warnings', base, head, summary => summary.diagnostics.console.warn, sample => sample.diagnostics.console.warn, util.formatNumber, 1, !all), + metricRow('Console errors', base, head, summary => summary.diagnostics.console.error, sample => sample.diagnostics.console.error, util.formatNumber, 1, !all), + metricRow('Console info', base, head, summary => summary.diagnostics.console.info, sample => sample.diagnostics.console.info, util.formatNumber, 1, !all), + ].filter(row => row != null); +} + function renderSummaryTable(base: BrowserMetricsReport, head: BrowserMetricsReport, all = false) { const rows = [ + ...renderBrowserDiagnosticsRows(base, head, all), //metricRow('Scenario duration', base, head, summary => summary.durationMs, sample => sample.durationMs, formatMs), metricRow('Requests', base, head, summary => summary.network.requestCount, sample => sample.network.requestCount, util.formatNumber, 1, !all), //metricRow('Failed requests', base, head, summary => summary.network.failedRequestCount, sample => sample.network.failedRequestCount, util.formatNumber), diff --git a/.github/scripts/frontend-browser-report.test.mts b/.github/scripts/frontend-browser-report.test.mts new file mode 100644 index 0000000000..4d4f130504 --- /dev/null +++ b/.github/scripts/frontend-browser-report.test.mts @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + createBrowserDiagnostics, + recordConsoleMessageType, + recordPageError, + summarizeBrowserDiagnostics, + type BrowserDiagnostics, +} from './chrome.mts'; +import { + renderBrowserDiagnosticsRows, + type BrowserMeasurement, + type BrowserMeasurementSample, + type BrowserMetricsReport, +} from './frontend-browser-report.mts'; + +function diagnostics(pageErrorCount: number, log: number, warn: number, error: number, info: number): BrowserDiagnostics { + return { + pageErrorCount, + console: { log, warn, error, info }, + }; +} + +function browserReport(label: 'base' | 'head', values: BrowserDiagnostics[]): BrowserMetricsReport { + const samples = values.map((value, index) => ({ + round: index + 1, + diagnostics: value, + } as BrowserMeasurementSample)); + + return { + label, + timestamp: '2026-07-16T00:00:00.000Z', + url: 'http://127.0.0.1:61812', + scenario: 'test', + sampleCount: samples.length, + aggregation: 'median', + summary: { + diagnostics: summarizeBrowserDiagnostics(values), + } as BrowserMeasurement, + samples, + }; +} + +test('records page errors and only the requested console message types', () => { + const result = createBrowserDiagnostics(); + recordPageError(result); + recordPageError(result); + for (const type of ['log', 'warning', 'error', 'info', 'debug', 'trace']) { + recordConsoleMessageType(result, type); + } + + assert.deepEqual(result, diagnostics(2, 1, 1, 1, 1)); +}); + +test('summarizes browser diagnostics with a median for every counter', () => { + const result = summarizeBrowserDiagnostics([ + diagnostics(0, 10, 20, 30, 40), + diagnostics(4, 14, 24, 34, 44), + diagnostics(2, 12, 22, 32, 42), + diagnostics(3, 13, 23, 33, 43), + diagnostics(1, 11, 21, 31, 41), + ]); + + assert.deepEqual(result, diagnostics(2, 12, 22, 32, 42)); +}); + +test('renders each changed browser diagnostics metric as a separate row', () => { + const base = browserReport('base', Array.from({ length: 5 }, () => diagnostics(0, 0, 0, 0, 0))); + const head = browserReport('head', Array.from({ length: 5 }, () => diagnostics(1, 1, 1, 1, 1))); + const rows = renderBrowserDiagnosticsRows(base, head).join('\n'); + + for (const label of ['Page errors', 'Console log', 'Console warnings', 'Console errors', 'Console info']) { + assert.match(rows, new RegExp(`\\*\\*${label}\\*\\*`)); + } +}); + +test('renders browser diagnostics deltas from paired rounds instead of summary medians', () => { + const base = browserReport('base', [0, 9, 10, 100, 101].map(value => diagnostics(value, 0, 0, 0, 0))); + const head = browserReport('head', [10, 100, 101, 0, 9].map(value => diagnostics(value, 0, 0, 0, 0))); + const rows = renderBrowserDiagnosticsRows(base, head); + + assert.equal(rows.length, 1); + assert.match(rows[0], /\*\*Page errors\*\*/); + assert.match(rows[0], /\\text\{\+10\}/); +}); + +test('omits browser diagnostics rows whose paired median does not change', () => { + const values = Array.from({ length: 5 }, () => diagnostics(2, 3, 4, 5, 6)); + const base = browserReport('base', values); + const head = browserReport('head', values); + + assert.deepEqual(renderBrowserDiagnosticsRows(base, head), []); +}); diff --git a/.github/scripts/measure-frontend-browser-comparison.mts b/.github/scripts/measure-frontend-browser-comparison.mts index 0825133b1d..3e357c3396 100644 --- a/.github/scripts/measure-frontend-browser-comparison.mts +++ b/.github/scripts/measure-frontend-browser-comparison.mts @@ -7,7 +7,7 @@ import { copyFile, mkdir, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import * as util from './utility.mts'; import * as heapSnapshotUtil from './heap-snapshot-util.mts'; -import { HeadlessChromeController, summarizeNetwork } from './chrome.mts'; +import { HeadlessChromeController, summarizeBrowserDiagnostics, summarizeNetwork } from './chrome.mts'; import type { BrowserMeasurement, NetworkRequest, NetworkSummary } from './chrome.mts'; import { closeUserSetupDialog, postNote, signupThroughUi, visitHome } from '../../packages/frontend/test/e2e/shared.ts'; @@ -173,6 +173,7 @@ function summarizeSamples(label: 'base' | 'head', samples: BrowserMeasurementSam timestamp: new Date().toISOString(), url: baseUrl, scenario: representative.scenario, + diagnostics: summarizeBrowserDiagnostics(samples.map(sample => sample.diagnostics)), durationMs: finiteMedian(samples.map(sample => sample.durationMs)), network: summarizeNetworkSamples(samples), performance: summarizePerformanceSamples(samples), @@ -210,6 +211,7 @@ async function measureSample(label: 'base' | 'head', round: number, heapSnapshot timestamp: new Date().toISOString(), url: baseUrl, scenario: 'fresh browser signup, first timeline note, after the note becomes visible', + diagnostics: chrome.collectDiagnostics(), durationMs, network: summarizeNetwork(chrome.networkRequests, baseUrl, chrome.webSocketConnections), networkRequests: chrome.networkRequests, diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63f8684c33..c1a3aeda6e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -152,3 +152,4 @@ jobs: node-version-file: '.node-version' - run: node --test .github/scripts/frontend-js-size.test.mts - run: node --test .github/scripts/memory-stability-util.test.mts + - run: node --test .github/scripts/frontend-browser-report.test.mts