diff --git a/.github/workflows/backend-diagnostics.inspect.yml b/.github/workflows/backend-diagnostics.inspect.yml index 74469ded63..8040b13307 100644 --- a/.github/workflows/backend-diagnostics.inspect.yml +++ b/.github/workflows/backend-diagnostics.inspect.yml @@ -89,9 +89,10 @@ jobs: - name: Measure backend memory usage working-directory: head env: - MK_MEMORY_COMPARE_ROUNDS: 10 + MK_MEMORY_COMPARE_ROUNDS: 15 MK_MEMORY_COMPARE_WARMUP_ROUNDS: 1 MK_MEMORY_HEAP_SNAPSHOT: 1 + MK_MEMORY_HEAP_SNAPSHOT_ROUNDS: 3 # 計測ハーネスはhead側のものを両方に使うが、成果物はrunnerのworkspace直下に出す MK_MEMORY_HEAP_SNAPSHOT_OUTPUT_DIR: ${{ github.workspace }} run: >- diff --git a/packages-private/diagnostics-backend/src/compare.ts b/packages-private/diagnostics-backend/src/compare.ts index 8847b44d4a..2ae1694f82 100644 --- a/packages-private/diagnostics-backend/src/compare.ts +++ b/packages-private/diagnostics-backend/src/compare.ts @@ -6,10 +6,15 @@ import { copyFile, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { execa } from 'execa'; -import { readIntegerEnv, readOptionalEnv } from 'diagnostics-shared/env'; +import { readBooleanEnv, readIntegerEnv, readOptionalEnv } from 'diagnostics-shared/env'; import { median } from 'diagnostics-shared/stats'; import { summarizeHeapSnapshotDataSamples, defaultHeapSnapshotBreakdownTopN } from 'diagnostics-shared/heap-snapshot'; import { resetState } from './db'; +import { + clampHeapSnapshotRounds, + selectRepresentativeHeapSnapshotRound, + shouldCollectHeapSnapshot, +} from './heap-snapshot-sampling'; import { measureBackendMemory } from './measure'; import { memoryPhases, type MemoryReport } from './types'; @@ -67,7 +72,12 @@ export function summarizeSamples(samples: MemoryReport['samples']) { return summary; } -async function genSample(label: string, repoDir: string, round: number, options: { heapSnapshotSavePath?: string } = {}) { +type GenSampleOptions = { + collectHeapSnapshot?: boolean; + heapSnapshotSavePath?: string; +}; + +async function genSample(label: string, repoDir: string, round: number, options: GenSampleOptions = {}) { process.stderr.write(`[${label}] Resetting database and Redis\n`); await resetState(); @@ -82,7 +92,7 @@ async function genSample(label: string, repoDir: string, round: number, options: process.stderr.write(`[${label}] Measuring memory\n`); return await measureBackendMemory(resolve(repoDir, 'packages/backend'), { // warmupラウンド (round <= 0) は捨てるので、重いheap snapshotは取らない - ...(round <= 0 ? { heapSnapshot: false } : {}), + ...(round <= 0 || options.collectHeapSnapshot === false ? { heapSnapshot: false } : {}), heapSnapshotSavePath: options.heapSnapshotSavePath ?? null, }); } @@ -91,32 +101,12 @@ function heapSnapshotPath(label: HeapSnapshotLabel, round: number) { return join(HEAP_SNAPSHOT_WORK_DIRS[label], `round-${round}.heapsnapshot`); } -/** - * 中央値に最も近いラウンドを代表として選ぶ。外れ値のスナップショットを成果物にしないため。 - */ -function selectRepresentativeHeapSnapshotRound(samples: MemoryReport['samples'], summary: MemoryReport['summary']) { - const medianTotal = summary.afterGc.heapSnapshot?.categories.total; - if (medianTotal == null || !Number.isFinite(medianTotal)) return null; - - let selected: { round: number; distance: number } | null = null; - for (const sample of samples) { - const total = sample.phases.afterGc.heapSnapshot?.categories.total; - if (total == null || !Number.isFinite(total)) continue; - - const distance = Math.abs(total - medianTotal); - if (selected == null || distance < selected.distance || (distance === selected.distance && sample.round < selected.round)) { - selected = { - round: sample.round, - distance, - }; - } - } - - return selected?.round ?? null; -} - async function saveRepresentativeHeapSnapshot(label: HeapSnapshotLabel, samples: MemoryReport['samples'], summary: MemoryReport['summary']) { - const round = selectRepresentativeHeapSnapshotRound(samples, summary); + const round = selectRepresentativeHeapSnapshotRound( + samples, + summary.afterGc.heapSnapshot?.categories.total, + sample => sample.phases.afterGc.heapSnapshot?.categories.total, + ); if (round == null) return; await copyFile(heapSnapshotPath(label, round), HEAP_SNAPSHOT_OUTPUT_PATHS[label]); @@ -131,6 +121,11 @@ async function saveRepresentativeHeapSnapshot(label: HeapSnapshotLabel, samples: export async function compareBackendMemory(options: CompareOptions) { const rounds = readIntegerEnv('MK_MEMORY_COMPARE_ROUNDS', 5, 1); const warmupRounds = readIntegerEnv('MK_MEMORY_COMPARE_WARMUP_ROUNDS', 1, 0); + const heapSnapshotsEnabled = readBooleanEnv('MK_MEMORY_HEAP_SNAPSHOT', false); + const requestedHeapSnapshotRounds = heapSnapshotsEnabled + ? readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_ROUNDS', rounds, 1) + : 0; + const heapSnapshotRounds = clampHeapSnapshotRounds(rounds, requestedHeapSnapshotRounds); const startedAt = new Date().toISOString(); for (const label of heapSnapshotLabels) { @@ -161,7 +156,11 @@ export async function compareBackendMemory(options: CompareOptions) { process.stderr.write(`Starting measurement round ${round}/${rounds}: ${order.join(' -> ')}\n`); for (const label of order) { - const sample = await genSample(label, reports[label].dir, round, { heapSnapshotSavePath: heapSnapshotPath(label, round) }); + const collectHeapSnapshot = shouldCollectHeapSnapshot(round, rounds, heapSnapshotRounds); + const sample = await genSample(label, reports[label].dir, round, { + collectHeapSnapshot, + ...(collectHeapSnapshot ? { heapSnapshotSavePath: heapSnapshotPath(label, round) } : {}), + }); reports[label].samples.push({ ...sample, round, @@ -186,6 +185,7 @@ export async function compareBackendMemory(options: CompareOptions) { strategy: 'interleaved-pairs', rounds, warmupRounds, + heapSnapshotRounds, startedAt, }, summary: summaries[label], diff --git a/packages-private/diagnostics-backend/src/heap-snapshot-sampling.ts b/packages-private/diagnostics-backend/src/heap-snapshot-sampling.ts new file mode 100644 index 0000000000..791840f9bd --- /dev/null +++ b/packages-private/diagnostics-backend/src/heap-snapshot-sampling.ts @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export function clampHeapSnapshotRounds(totalRounds: number, requestedRounds: number) { + return Math.min(totalRounds, requestedRounds); +} + +export function shouldCollectHeapSnapshot(round: number, totalRounds: number, requestedRounds: number) { + const snapshotRounds = clampHeapSnapshotRounds(totalRounds, requestedRounds); + return round > totalRounds - snapshotRounds; +} + +/** + * 中央値に最も近いラウンドを代表として選ぶ。外れ値のスナップショットを成果物にしないため。 + */ +export function selectRepresentativeHeapSnapshotRound( + samples: T[], + medianTotal: number | null | undefined, + getTotal: (sample: T) => number | null | undefined, +) { + if (medianTotal == null || !Number.isFinite(medianTotal)) return null; + + let selected: { round: number; distance: number } | null = null; + for (const sample of samples) { + const total = getTotal(sample); + if (total == null || !Number.isFinite(total)) continue; + + const distance = Math.abs(total - medianTotal); + if (selected == null || distance < selected.distance || (distance === selected.distance && sample.round < selected.round)) { + selected = { round: sample.round, distance }; + } + } + + return selected?.round ?? null; +} diff --git a/packages-private/diagnostics-backend/src/report/markdown.ts b/packages-private/diagnostics-backend/src/report/markdown.ts index bd4655dbe2..0fe16bf62f 100644 --- a/packages-private/diagnostics-backend/src/report/markdown.ts +++ b/packages-private/diagnostics-backend/src/report/markdown.ts @@ -3,9 +3,14 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { formatColoredDelta, formatDeltaPercentInMdTable, formatKiBAsMb } from 'diagnostics-shared/format'; -import { median, pairedDeltaSummary, sampleSpread } from 'diagnostics-shared/stats'; -import { renderHeapSnapshotTable } from 'diagnostics-shared/heap-snapshot'; +import { formatKiBAsMb } from 'diagnostics-shared/format'; +import { renderHeapSnapshotTable, type HeapSnapshotReport } from 'diagnostics-shared/heap-snapshot'; +import { renderMetricComparisonTable } from 'diagnostics-shared/metric-table'; +import { + independentDeltaSummary, + isOutsideObservedNoise, + type IndependentDeltaSummary, +} from 'diagnostics-shared/stats'; import type { MemoryPhase, MemoryReport } from '../types'; export type RenderMemoryReportOptions = { @@ -29,6 +34,8 @@ const memoryMetrics = [ type MemoryMetric = typeof memoryMetrics[number]; +const memoryColorThresholdKiB = 100; + function formatMemoryMetricName(metric: MemoryMetric) { return metric === 'Pss' ? 'PSS' : metric; } @@ -40,64 +47,51 @@ function getMemoryValueFromSample(sample: MemoryReport['samples'][number], phase return memoryUsage.Private_Clean + memoryUsage.Private_Dirty; } -function getSampleValues(report: MemoryReport, phase: MemoryPhase, metric: MemoryMetric) { - return report.samples.map(sample => getMemoryValueFromSample(sample, phase, metric)); +function summarizeMemoryMetric(base: MemoryReport, head: MemoryReport, phase: MemoryPhase, metric: MemoryMetric) { + return independentDeltaSummary( + base.samples, + head.samples, + sample => getMemoryValueFromSample(sample, phase, metric), + ); } -function getMemoryValue(report: MemoryReport, phase: MemoryPhase, metric: MemoryMetric) { - if (metric !== 'USS') return report.summary[phase].memoryUsage[metric]; - return median(getSampleValues(report, phase, metric)); -} - -function getSampleSpread(report: MemoryReport, phase: MemoryPhase, metric: MemoryMetric) { - return sampleSpread(getSampleValues(report, phase, metric)); +function getDeltaPercent(summary: IndependentDeltaSummary) { + if (summary.baseMedian === 0) return null; + return summary.delta * 100 / summary.baseMedian; } function renderMainTableForPhase(base: MemoryReport, head: MemoryReport, phase: MemoryPhase) { - const lines = [ - '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', - ]; + return renderMetricComparisonTable( + base.samples, + head.samples, + memoryMetrics.map(metric => ({ + label: `**${formatMemoryMetricName(metric)}**`, + getValue: sample => getMemoryValueFromSample(sample, phase, metric), + formatValue: formatKiBAsMb, + absoluteThreshold: memoryColorThresholdKiB, + })), + ); +} - function formatDeltaMemory(deltaKiB: number) { - return formatColoredDelta(deltaKiB, v => formatKiBAsMb(v), 100); // 0.1 MB threshold - } +function toHeapSnapshotReport(report: MemoryReport): HeapSnapshotReport | null { + const summary = report.summary.afterGc.heapSnapshot; + if (summary == null) return null; - for (const metric of memoryMetrics) { - const baseValue = getMemoryValue(base, phase, metric); - const headValue = getMemoryValue(head, phase, metric); - - const baseSpread = getSampleSpread(base, phase, metric); - const headSpread = getSampleSpread(head, phase, metric); - const summary = pairedDeltaSummary(base.samples, head.samples, (sample) => getMemoryValueFromSample(sample, phase, metric)); - const percent = summary.median * 100 / baseValue; - const deltaMedian = `${formatDeltaMemory(summary.median)}
${formatDeltaPercentInMdTable(percent, 0.1)}`; - - lines.push(`| **${formatMemoryMetricName(metric)}** | ${formatKiBAsMb(baseValue)}
± ${formatKiBAsMb(baseSpread)} | ${formatKiBAsMb(headValue)}
± ${formatKiBAsMb(headSpread)} | ${deltaMedian} | ${formatKiBAsMb(summary.mad)} | ${formatDeltaMemory(summary.min)} | ${formatDeltaMemory(summary.max)} |`); - } - - return lines.join('\n'); + return { + summary, + samples: report.samples.flatMap(sample => { + const data = sample.phases.afterGc.heapSnapshot; + return data == null ? [] : [{ round: sample.round, data }]; + }), + }; } function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) { - const baseHeapSnapshotReport = { - summary: base.summary.afterGc.heapSnapshot!, - samples: base.samples.map(sample => ({ - round: sample.round, - data: sample.phases.afterGc.heapSnapshot!, - })), - }; - - const headHeapSnapshotReport = { - summary: head.summary.afterGc.heapSnapshot!, - samples: head.samples.map(sample => ({ - round: sample.round, - data: sample.phases.afterGc.heapSnapshot!, - })), - }; + const baseHeapSnapshotReport = toHeapSnapshotReport(base); + const headHeapSnapshotReport = toHeapSnapshotReport(head); + if (baseHeapSnapshotReport == null || headHeapSnapshotReport == null) return null; const table = renderHeapSnapshotTable(baseHeapSnapshotReport, headHeapSnapshotReport); - if (table == null) return null; const lines = [ '### V8 Heap Snapshot Statistics', @@ -119,29 +113,11 @@ function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) { return lines.join('\n'); } -function getDiffPercent(base: MemoryReport, head: MemoryReport, phase: MemoryPhase, metric: MemoryMetric) { - const baseValue = getMemoryValue(base, phase, metric); - const headValue = getMemoryValue(head, phase, metric); - return ((headValue - baseValue) * 100) / baseValue; -} - -/** - * 増加分がサンプルのばらつきを明確に超えているかを見る。 - * 測定ノイズで警告が出続けるのを避けるため、合成ばらつきの3倍を閾値にする。 - */ -function isBeyondSampleNoise(base: MemoryReport, head: MemoryReport, phase: MemoryPhase, metric: MemoryMetric) { - const baseValue = getMemoryValue(base, phase, metric); - const headValue = getMemoryValue(head, phase, metric); - - const delta = headValue - baseValue; - if (delta <= 0) return false; - - const baseSpread = getSampleSpread(base, phase, metric); - const headSpread = getSampleSpread(head, phase, metric); - if (baseSpread == null || headSpread == null) return true; - - const combinedSpread = Math.hypot(baseSpread, headSpread); - return delta > combinedSpread * 3; +function countNonConvergedMemorySamples(base: MemoryReport, head: MemoryReport) { + return [base, head] + .flatMap(report => report.samples) + .filter(sample => memoryReportPhases.some(phase => !sample.phases[phase.key].memoryStability.converged)) + .length; } export function renderMemoryReportMarkdown(base: MemoryReport, head: MemoryReport, options: RenderMemoryReportOptions) { @@ -162,6 +138,16 @@ export function renderMemoryReportMarkdown(base: MemoryReport, head: MemoryRepor lines.push(''); } + lines.push(`_Values are median ± MAD (${base.samples.length} base / ${head.samples.length} head samples). Delta is Head - Base. Deltas are highlighted when their absolute value reaches the metric threshold and exceeds 3 × MAD._`); + lines.push(''); + + const nonConvergedSamples = countNonConvergedMemorySamples(base, head); + if (nonConvergedSamples > 0) { + const noun = nonConvergedSamples === 1 ? 'sample' : 'samples'; + lines.push(`⚠️ **Measurement warning**: ${nonConvergedSamples} memory ${noun} did not converge.`); + lines.push(''); + } + const heapSnapshotSection = renderHeapSnapshotSection(base, head); if (heapSnapshotSection != null) { lines.push(heapSnapshotSection); @@ -172,8 +158,14 @@ export function renderMemoryReportMarkdown(base: MemoryReport, head: MemoryRepor lines.push(''); const warningMetric = 'Pss'; - const warningDiffPercent = getDiffPercent(base, head, 'afterGc', warningMetric); - if (warningDiffPercent > 5 && isBeyondSampleNoise(base, head, 'afterGc', warningMetric)) { + const warningSummary = summarizeMemoryMetric(base, head, 'afterGc', warningMetric); + const warningDiffPercent = getDeltaPercent(warningSummary); + if ( + warningSummary.delta > 0 && + warningDiffPercent != null && + warningDiffPercent > 5 && + isOutsideObservedNoise(warningSummary) + ) { lines.push(`⚠️ **Warning**: Memory usage (${formatMemoryMetricName(warningMetric)}) has increased by more than 5% and exceeds the observed sample noise. Please verify this is not an unintended change.`); lines.push(''); } diff --git a/packages-private/diagnostics-backend/src/types.ts b/packages-private/diagnostics-backend/src/types.ts index 2dff718a5c..e095572f29 100644 --- a/packages-private/diagnostics-backend/src/types.ts +++ b/packages-private/diagnostics-backend/src/types.ts @@ -37,6 +37,7 @@ export type MemoryReport = { strategy: string; rounds: number; warmupRounds: number; + heapSnapshotRounds?: number; startedAt: string; }; summary: Record ± 1 MB | 168 MB
± 1 MB | $\color{orange}{\text{+16 MB}}$
$\color{orange}{\text{+10.5\\%}}$ | 0 MB | $\color{orange}{\text{+16 MB}}$ | $\color{orange}{\text{+16 MB}}$ | -| **PSS** | 202 MB
± 1 MB | 218 MB
± 1 MB | $\color{orange}{\text{+16 MB}}$
$\color{orange}{\text{+7.9\\%}}$ | 0 MB | $\color{orange}{\text{+16 MB}}$ | $\color{orange}{\text{+16 MB}}$ | -| **USS** | 184 MB
± 1 MB | 200 MB
± 1 MB | $\color{orange}{\text{+16 MB}}$
$\color{orange}{\text{+8.7\\%}}$ | 0 MB | $\color{orange}{\text{+16 MB}}$ | $\color{orange}{\text{+16 MB}}$ | -| **External** | 8.2 MB
± 0.1 MB | 8.2 MB
± 0.1 MB | 0 MB
0% | 0 MB | 0 MB | 0 MB | +| Metric | @ Base | @ Head | Δ | MAD | +| --- | ---: | ---: | ---: | ---: | +| **HeapUsed** | 152 MB
± 1 MB | 168 MB
± 1 MB | $\color{orange}{\text{+16 MB}}$
$\color{orange}{\text{+10.5\\%}}$ | 1.4 MB | +| **PSS** | 202 MB
± 1 MB | 218 MB
± 1 MB | $\color{orange}{\text{+16 MB}}$
$\color{orange}{\text{+7.9\\%}}$ | 1.4 MB | +| **USS** | 184 MB
± 1 MB | 200 MB
± 1 MB | $\color{orange}{\text{+16 MB}}$
$\color{orange}{\text{+8.7\\%}}$ | 1.4 MB | +| **External** | 8.2 MB
± 0.1 MB | 8.2 MB
± 0.1 MB | 0 MB
0% | 0.1 MB | + +_Values are median ± MAD (3 base / 3 head samples). Delta is Head - Base. Deltas are highlighted when their absolute value reaches the metric threshold and exceeds 3 × MAD._ ### V8 Heap Snapshot Statistics -| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| $\color{gray}{\rule{8pt}{8pt}}$ **Total** | 40 MB
± 200 KB | 44 MB
± 200 KB | $\color{orange}{\text{+3.2 MB}}$
$\color{orange}{\text{+7.9\\%}}$ | 0 B | $\color{orange}{\text{+3.2 MB}}$ | $\color{orange}{\text{+3.2 MB}}$ | -| | | | | | | | -|
$\color{orange}{\rule{8pt}{8pt}}$ **Code**11.8% → 11.8%
| 4.7 MB | 5.1 MB | $\color{orange}{\text{+376 KB}}$ | 0 B | $\color{orange}{\text{+376 KB}}$ | $\color{orange}{\text{+376 KB}}$ | -|
$\color{red}{\rule{8pt}{8pt}}$ **Strings**11% → 11%
| 4.4 MB | 4.8 MB | $\color{orange}{\text{+352 KB}}$ | 0 B | $\color{orange}{\text{+352 KB}}$ | $\color{orange}{\text{+352 KB}}$ | -|
$\color{cyan}{\rule{8pt}{8pt}}$ **JS arrays**10.3% → 10.3%
| 4.1 MB | 4.5 MB | $\color{orange}{\text{+328 KB}}$ | 0 B | $\color{orange}{\text{+328 KB}}$ | $\color{orange}{\text{+328 KB}}$ | -|
$\color{green}{\rule{8pt}{8pt}}$ **Typed arrays**9.5% → 9.5%
| 3.8 MB | 4.1 MB | $\color{orange}{\text{+304 KB}}$ | 0 B | $\color{orange}{\text{+304 KB}}$ | $\color{orange}{\text{+304 KB}}$ | -|
$\color{yellow}{\rule{8pt}{8pt}}$ **System objects**8.8% → 8.8%
| 3.5 MB | 3.8 MB | $\color{orange}{\text{+280 KB}}$ | 0 B | $\color{orange}{\text{+280 KB}}$ | $\color{orange}{\text{+280 KB}}$ | -|
$\color{violet}{\rule{8pt}{8pt}}$ **Other JS objs**8% → 8%
| 3.2 MB | 3.5 MB | $\color{orange}{\text{+256 KB}}$ | 0 B | $\color{orange}{\text{+256 KB}}$ | $\color{orange}{\text{+256 KB}}$ | -|
$\color{pink}{\rule{8pt}{8pt}}$ **Other non-JS objs**7.3% → 7.3%
| 2.9 MB | 3.2 MB | $\color{orange}{\text{+232 KB}}$ | 0 B | $\color{orange}{\text{+232 KB}}$ | $\color{orange}{\text{+232 KB}}$ | +| Metric | @ Base | @ Head | Δ | MAD | +| --- | ---: | ---: | ---: | ---: | +| $\color{gray}{\rule{8pt}{8pt}}$ **Total** | 40 MB
± 200 KB | 44 MB
± 200 KB | $\color{orange}{\text{+3.2 MB}}$
$\color{orange}{\text{+7.9\\%}}$ | 283 KB | +| | | | | | +| $\color{orange}{\rule{8pt}{8pt}}$ **Code** | 4.7 MB | 5.1 MB | $\color{orange}{\text{+376 KB}}$ | 33 KB | +| $\color{red}{\rule{8pt}{8pt}}$ **Strings** | 4.4 MB | 4.8 MB | $\color{orange}{\text{+352 KB}}$ | 31 KB | +| $\color{cyan}{\rule{8pt}{8pt}}$ **JS arrays** | 4.1 MB | 4.5 MB | $\color{orange}{\text{+328 KB}}$ | 29 KB | +| $\color{green}{\rule{8pt}{8pt}}$ **Typed arrays** | 3.8 MB | 4.1 MB | $\color{orange}{\text{+304 KB}}$ | 27 KB | +| $\color{yellow}{\rule{8pt}{8pt}}$ **System objects** | 3.5 MB | 3.8 MB | $\color{orange}{\text{+280 KB}}$ | 25 KB | +| $\color{violet}{\rule{8pt}{8pt}}$ **Other JS objs** | 3.2 MB | 3.5 MB | $\color{orange}{\text{+256 KB}}$ | 23 KB | +| $\color{pink}{\rule{8pt}{8pt}}$ **Other non-JS objs** | 2.9 MB | 3.2 MB | $\color{orange}{\text{+232 KB}}$ | 21 KB | Download representative heap snapshot: [base](https://example.invalid/base) / [head](https://example.invalid/head) diff --git a/packages-private/diagnostics-backend/test/heap-snapshot-sampling.test.ts b/packages-private/diagnostics-backend/test/heap-snapshot-sampling.test.ts new file mode 100644 index 0000000000..8409fe166b --- /dev/null +++ b/packages-private/diagnostics-backend/test/heap-snapshot-sampling.test.ts @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { describe, expect, test } from 'vitest'; +import { + clampHeapSnapshotRounds, + selectRepresentativeHeapSnapshotRound, + shouldCollectHeapSnapshot, +} from '../src/heap-snapshot-sampling'; + +describe('heap snapshot sampling', () => { + test('collects only the final requested rounds', () => { + const selected = Array.from({ length: 15 }, (_, index) => index + 1) + .filter(round => shouldCollectHeapSnapshot(round, 15, 3)); + expect(selected).toStrictEqual([13, 14, 15]); + }); + + test('clamps the requested count to the measured round count', () => { + expect(clampHeapSnapshotRounds(5, 20)).toBe(5); + expect(Array.from({ length: 5 }, (_, index) => index + 1) + .filter(round => shouldCollectHeapSnapshot(round, 5, 20))) + .toStrictEqual([1, 2, 3, 4, 5]); + }); + + test('collects no snapshots when the effective count is zero', () => { + expect(Array.from({ length: 5 }, (_, index) => index + 1) + .filter(round => shouldCollectHeapSnapshot(round, 5, 0))) + .toStrictEqual([]); + }); + + test('selects the snapshot nearest the median and ignores missing rounds', () => { + const samples = [ + { round: 12, total: null }, + { round: 13, total: 100 }, + { round: 14, total: 110 }, + { round: 15, total: 130 }, + ]; + expect(selectRepresentativeHeapSnapshotRound(samples, 110, sample => sample.total)).toBe(14); + }); +}); diff --git a/packages-private/diagnostics-backend/test/render-md.test.ts b/packages-private/diagnostics-backend/test/render-md.test.ts index 468822014f..6dcf1979ca 100644 --- a/packages-private/diagnostics-backend/test/render-md.test.ts +++ b/packages-private/diagnostics-backend/test/render-md.test.ts @@ -6,6 +6,7 @@ import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import { expect, test } from 'vitest'; +import { pairedDeltaSummary } from 'diagnostics-shared/stats'; import { renderMemoryReportMarkdown } from '../src/report/markdown'; import type { MemoryReport } from '../src/types'; @@ -15,6 +16,26 @@ async function loadFixture(name: string) { return JSON.parse(await readFile(join(fixturesDir, `${name}.json`), 'utf8')) as MemoryReport; } +function replacePssSamples(report: MemoryReport, values: number[]) { + const templates = structuredClone(report.samples); + report.samples = values.map((Pss, index) => { + const sample = structuredClone(templates[index % templates.length]); + sample.round = index + 1; + sample.phases.afterGc.memoryUsage.Pss = Pss; + const privateClean = sample.phases.afterGc.memoryUsage.Private_Clean; + sample.phases.afterGc.memoryUsage.Private_Dirty = Pss - privateClean; + return sample; + }); + report.sampleCount = report.samples.length; + return report; +} + +function findMetricRow(markdown: string, metric: string) { + const row = markdown.split('\n').find(line => line.startsWith(`| **${metric}**`)); + if (row === undefined) throw new Error(`expected memory report to contain a ${metric} row`); + return row; +} + /** * 出力をゴールデンファイルで固定する。 * 意図的に変更したときは `vitest -u` で更新し、__snapshots__ の差分もレビューすること。 @@ -27,3 +48,89 @@ test('renders the backend memory report', async () => { await expect(markdown).toMatchFileSnapshot('./__snapshots__/render-md.md'); }); + +test('throws when filtering leaves fewer than two heap snapshots per side', async () => { + const base = await loadFixture('base'); + const head = await loadFixture('head'); + for (const report of [base, head]) { + for (const sample of report.samples.slice(0, -1)) { + sample.phases.afterGc.heapSnapshot = null; + } + } + + expect(() => renderMemoryReportMarkdown(base, head, { + baseHeapSnapshotUrl: 'https://example.invalid/base', + headHeapSnapshotUrl: 'https://example.invalid/head', + })).toThrow('At least two samples per side are required'); +}); + +test('reports the difference of medians and leaves a paired-looking PSS delta uncoloured', async () => { + const base = replacePssSamples(await loadFixture('base'), [290_000, 292_900, 295_800, 298_700, 301_600]); + const head = replacePssSamples(await loadFixture('head'), [292_900, 296_300, 298_700, 301_600, 290_000]); + + expect(pairedDeltaSummary( + base.samples, + head.samples, + sample => sample.phases.afterGc.memoryUsage.Pss, + ).median).toBe(2_900); + + const markdown = renderMemoryReportMarkdown(base, head, { + baseHeapSnapshotUrl: 'https://example.invalid/base', + headHeapSnapshotUrl: 'https://example.invalid/head', + }); + const row = findMetricRow(markdown, 'PSS'); + + expect(row).toContain('295.8 MB
± 2.9 MB'); + expect(row).toContain('296.3 MB
± 3.4 MB'); + expect(row).toContain('$\\text{+0.5 MB}$'); + expect(row).toContain('4.5 MB'); + expect(row.split('|')).toHaveLength(7); + expect(row).not.toMatch(/within noise|increase|decrease|inconclusive/); + expect(row).not.toContain('\\color{orange}'); + expect(findMetricRow(markdown, 'USS')).not.toContain('\\color{orange}'); +}); + +test('keeps the convergence warning without suppressing table colour or the PSS warning', async () => { + const base = await loadFixture('base'); + const head = await loadFixture('head'); + base.samples[0].phases.afterGc.memoryStability.converged = false; + + const markdown = renderMemoryReportMarkdown(base, head, { + baseHeapSnapshotUrl: 'https://example.invalid/base', + headHeapSnapshotUrl: 'https://example.invalid/head', + }); + const memorySection = markdown.slice(0, markdown.indexOf('### V8 Heap Snapshot Statistics')); + + expect(memorySection).not.toContain('inconclusive'); + expect(findMetricRow(markdown, 'PSS')).toContain('\\color{orange}'); + expect(markdown).toContain('⚠️ **Measurement warning**: 1 memory sample did not converge.'); + expect(markdown).not.toContain('results are marked inconclusive'); + expect(markdown).toContain('⚠️ **Warning**: Memory usage (PSS)'); +}); + +test('throws for an undersampled memory comparison', async () => { + const base = await loadFixture('base'); + const head = await loadFixture('head'); + base.samples = base.samples.slice(0, 1); + head.samples = head.samples.slice(0, 1); + + expect(() => renderMemoryReportMarkdown(base, head, { + baseHeapSnapshotUrl: 'https://example.invalid/base', + headHeapSnapshotUrl: 'https://example.invalid/head', + })).toThrow('At least two samples per side are required'); +}); + +test('renders an unavailable percentage when the base median is zero', async () => { + const base = await loadFixture('base'); + const head = await loadFixture('head'); + for (const sample of base.samples) sample.phases.afterGc.memoryUsage.External = 0; + + const markdown = renderMemoryReportMarkdown(base, head, { + baseHeapSnapshotUrl: 'https://example.invalid/base', + headHeapSnapshotUrl: 'https://example.invalid/head', + }); + + expect(findMetricRow(markdown, 'External')).toContain('
-'); + expect(markdown).toContain('| Metric | @ Base | @ Head | Δ | MAD |'); + expect(markdown).not.toContain('| Metric | @ Base | @ Head | Δ | MAD | Result |'); +}); diff --git a/packages-private/diagnostics-frontend/src/report.ts b/packages-private/diagnostics-frontend/src/report.ts index e605b0f931..ab85c625f8 100644 --- a/packages-private/diagnostics-frontend/src/report.ts +++ b/packages-private/diagnostics-frontend/src/report.ts @@ -5,11 +5,11 @@ import { formatBytes, formatColoredDelta, formatNumber } from 'diagnostics-shared/format'; import { renderHeapSnapshotTable, type HeapSnapshotReport } from 'diagnostics-shared/heap-snapshot'; -import { pairedDeltaSummary } from 'diagnostics-shared/stats'; +import { renderMetricComparisonTable } from 'diagnostics-shared/metric-table'; import { renderFrontendChunkReport } from './bundle/chunk-report'; import { collectVisualizerReport, renderVisualizerSummaryTable, type VisualizerReport } from './bundle/visualizer'; import type { CollectedBundleReport } from './bundle/manifest'; -import type { BrowserMeasurement, BrowserMeasurementSample, BrowserMetricsReport } from './browser/types'; +import type { BrowserMeasurementSample, BrowserMetricsReport } from './browser/types'; export type FrontendDiagnosticsMarkdownInput = { bundle: { @@ -29,86 +29,132 @@ export type FrontendDiagnosticsMarkdownInput = { }; }; -function renderMetricRow( - label: string, - base: BrowserMetricsReport, - head: BrowserMetricsReport, - getSummaryValue: (summary: BrowserMeasurement) => number, - getSampleValue: (sample: BrowserMeasurementSample) => number, - formatter: (value: number) => string, - significantThreshold = 0, - skipIfNotSignificant = true, -) { - const baseValue = getSummaryValue(base.summary); - const headValue = getSummaryValue(head.summary); - if (baseValue == null || headValue == null || !Number.isFinite(baseValue) || !Number.isFinite(headValue)) return null; - - const summary = pairedDeltaSummary(base.samples, head.samples, sample => getSampleValue(sample)); - // 有意な閾値に満たない場合はそもそもrowとして出力しない - if (skipIfNotSignificant && (Math.abs(summary.median) < significantThreshold)) return null; - - const deltaMedian = formatColoredDelta(summary.median, formatter, significantThreshold); - - return `| **${label}** | ${formatter(baseValue)} | ${formatter(headValue)} | ${deltaMedian} | ${formatter(summary.mad)} | ${formatColoredDelta(summary.min, formatter, significantThreshold)} | ${formatColoredDelta(summary.max, formatter, significantThreshold)} |`; -} - -function resourceTypeBytes(report: BrowserMeasurement, resourceTypes: string[]) { - return resourceTypes.reduce((sum, resourceType) => sum + (report.network.byResourceType[resourceType]?.encodedBytes ?? 0), 0); -} - function resourceTypeSampleBytes(sample: BrowserMeasurementSample, resourceTypes: string[]) { - return resourceTypeBytes(sample, resourceTypes); + return resourceTypes.reduce((sum, resourceType) => sum + (sample.network.byResourceType[resourceType]?.encodedBytes ?? 0), 0); } -function renderBrowserSummaryTable(base: BrowserMetricsReport, head: BrowserMetricsReport, all = false) { - //function getMetric(report: BrowserMeasurement, key: string) { - // return report.performance.cdpMetrics[key]; - //} - - const rows = [ - //renderMetricRow('Scenario duration', base, head, summary => summary.durationMs, sample => sample.durationMs, formatMs), - renderMetricRow('Requests', base, head, summary => summary.network.requestCount, sample => sample.network.requestCount, formatNumber, 1, !all), - //renderMetricRow('Failed requests', base, head, summary => summary.network.failedRequestCount, sample => sample.network.failedRequestCount, formatNumber), - renderMetricRow('Encoded network', base, head, summary => summary.network.totalEncodedBytes, sample => sample.network.totalEncodedBytes, formatBytes, 10000, !all), - renderMetricRow('Decoded body', base, head, summary => summary.network.totalDecodedBodyBytes, sample => sample.network.totalDecodedBodyBytes, formatBytes, 10000, !all), - renderMetricRow('Same-origin encoded', base, head, summary => summary.network.sameOriginEncodedBytes, sample => sample.network.sameOriginEncodedBytes, formatBytes, 10000, !all), - renderMetricRow('Third-party encoded', base, head, summary => summary.network.thirdPartyEncodedBytes, sample => sample.network.thirdPartyEncodedBytes, formatBytes, 10000, !all), - renderMetricRow('Script encoded', base, head, summary => resourceTypeBytes(summary, ['Script']), sample => resourceTypeSampleBytes(sample, ['Script']), formatBytes, 10000, !all), - renderMetricRow('Stylesheet encoded', base, head, summary => resourceTypeBytes(summary, ['Stylesheet']), sample => resourceTypeSampleBytes(sample, ['Stylesheet']), formatBytes, 10000, !all), - renderMetricRow('Fetch/XHR encoded', base, head, summary => resourceTypeBytes(summary, ['Fetch', 'XHR']), sample => resourceTypeSampleBytes(sample, ['Fetch', 'XHR']), formatBytes, 10000, !all), - renderMetricRow('Image encoded', base, head, summary => resourceTypeBytes(summary, ['Image']), sample => resourceTypeSampleBytes(sample, ['Image']), formatBytes, 10000, !all), - renderMetricRow('Font encoded', base, head, summary => resourceTypeBytes(summary, ['Font']), sample => resourceTypeSampleBytes(sample, ['Font']), formatBytes, 10000, !all), - //renderMetricRow('First contentful paint', base, head, summary => summary.performance.webVitals.firstContentfulPaintMs, sample => sample.performance.webVitals.firstContentfulPaintMs, formatMs), - //renderMetricRow('Load event end', base, head, summary => summary.performance.webVitals.loadEventEndMs, sample => sample.performance.webVitals.loadEventEndMs, formatMs), - //renderMetricRow('Long tasks', base, head, summary => summary.performance.webVitals.longTaskCount, sample => sample.performance.webVitals.longTaskCount, formatNumber), - //renderMetricRow('Long task duration', base, head, summary => summary.performance.webVitals.longTaskDurationMs, sample => sample.performance.webVitals.longTaskDurationMs, formatMs), - //renderMetricRow('Max long task', base, head, summary => summary.performance.webVitals.maxLongTaskDurationMs, sample => sample.performance.webVitals.maxLongTaskDurationMs, formatMs), - //renderMetricRow('JS heap used', base, head, summary => summary.performance.runtimeHeap?.usedSize ?? getMetric(summary, 'JSHeapUsedSize'), sample => sample.performance.runtimeHeap?.usedSize ?? getMetric(sample, 'JSHeapUsedSize'), formatBytes), - //renderMetricRow('JS heap total', base, head, summary => summary.performance.runtimeHeap?.totalSize ?? getMetric(summary, 'JSHeapTotalSize'), sample => sample.performance.runtimeHeap?.totalSize ?? getMetric(sample, 'JSHeapTotalSize'), formatBytes), - //renderMetricRow('V8 heap snapshot total', base, head, summary => summary.heapSnapshot.categories.total, sample => sample.heapSnapshot.categories.total, formatBytes, 10000), - //renderMetricRow('DOM elements', base, head, summary => summary.performance.webVitals.domElements, sample => sample.performance.webVitals.domElements, formatNumber), - //renderMetricRow('CDP nodes', base, head, summary => getMetric(summary, 'Nodes'), sample => getMetric(sample, 'Nodes'), formatNumber), - //renderMetricRow('JS event listeners', base, head, summary => getMetric(summary, 'JSEventListeners'), sample => getMetric(sample, 'JSEventListeners'), formatNumber), - //renderMetricRow('Layout count', base, head, summary => getMetric(summary, 'LayoutCount'), sample => getMetric(sample, 'LayoutCount'), formatNumber), - //renderMetricRow('Recalc style count', base, head, summary => getMetric(summary, 'RecalcStyleCount'), sample => getMetric(sample, 'RecalcStyleCount'), formatNumber), - //renderMetricRow('Script duration', base, head, summary => getMetric(summary, 'ScriptDuration'), sample => getMetric(sample, 'ScriptDuration'), formatSecondsAsMs), - //renderMetricRow('Task duration', base, head, summary => getMetric(summary, 'TaskDuration'), sample => getMetric(sample, 'TaskDuration'), formatSecondsAsMs), - renderMetricRow('WebSocket connections', base, head, summary => summary.network.webSocketConnectionCount, sample => sample.network.webSocketConnectionCount, formatNumber, 1, !all), - renderMetricRow('WebSocket sent', base, head, summary => summary.network.webSocketSentBytes, sample => sample.network.webSocketSentBytes, formatBytes, 10000, !all), - renderMetricRow('WebSocket received', base, head, summary => summary.network.webSocketReceivedBytes, sample => sample.network.webSocketReceivedBytes, formatBytes, 10000, !all), - renderMetricRow('Page errors', base, head, summary => summary.diagnostics.pageErrorCount, sample => sample.diagnostics.pageErrorCount, formatNumber, 1, !all), - renderMetricRow('Console log', base, head, summary => summary.diagnostics.console.log, sample => sample.diagnostics.console.log, formatNumber, 1, !all), - renderMetricRow('Console warnings', base, head, summary => summary.diagnostics.console.warning, sample => sample.diagnostics.console.warning, formatNumber, 1, !all), - renderMetricRow('Console errors', base, head, summary => summary.diagnostics.console.error, sample => sample.diagnostics.console.error, formatNumber, 1, !all), - renderMetricRow('Console info', base, head, summary => summary.diagnostics.console.info, sample => sample.diagnostics.console.info, formatNumber, 1, !all), - renderMetricRow('Page-attributed memory', base, head, summary => summary.performance.tabMemory.totalBytes, sample => sample.performance.tabMemory.totalBytes, formatBytes, 10000, !all), - ].filter(row => row != null); - - return [ - '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows, - ].join('\n'); +function renderBrowserSummaryTable(base: BrowserMetricsReport, head: BrowserMetricsReport) { + return renderMetricComparisonTable( + base.samples, + head.samples, + [ + { + label: '**Requests**', + getValue: sample => sample.network.requestCount, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**Encoded network**', + getValue: sample => sample.network.totalEncodedBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Decoded body**', + getValue: sample => sample.network.totalDecodedBodyBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Same-origin encoded**', + getValue: sample => sample.network.sameOriginEncodedBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Third-party encoded**', + getValue: sample => sample.network.thirdPartyEncodedBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Script encoded**', + getValue: sample => resourceTypeSampleBytes(sample, ['Script']), + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Stylesheet encoded**', + getValue: sample => resourceTypeSampleBytes(sample, ['Stylesheet']), + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Fetch/XHR encoded**', + getValue: sample => resourceTypeSampleBytes(sample, ['Fetch', 'XHR']), + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Image encoded**', + getValue: sample => resourceTypeSampleBytes(sample, ['Image']), + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Font encoded**', + getValue: sample => resourceTypeSampleBytes(sample, ['Font']), + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**WebSocket connections**', + getValue: sample => sample.network.webSocketConnectionCount, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**WebSocket sent**', + getValue: sample => sample.network.webSocketSentBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**WebSocket received**', + getValue: sample => sample.network.webSocketReceivedBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + { + label: '**Page errors**', + getValue: sample => sample.diagnostics.pageErrorCount, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**Console log**', + getValue: sample => sample.diagnostics.console.log, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**Console warnings**', + getValue: sample => sample.diagnostics.console.warning, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**Console errors**', + getValue: sample => sample.diagnostics.console.error, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**Console info**', + getValue: sample => sample.diagnostics.console.info, + formatValue: formatNumber, + absoluteThreshold: 1, + }, + { + label: '**Page-attributed memory**', + getValue: sample => sample.performance.tabMemory.totalBytes, + formatValue: formatBytes, + absoluteThreshold: 10_000, + }, + ], + { onlySignificantChanges: true }, + ); } function renderResourceTypeTable(base: BrowserMetricsReport, head: BrowserMetricsReport) { @@ -199,7 +245,7 @@ export function renderFrontendDiagnosticsMarkdown(input: FrontendDiagnosticsMark `Download representative heap snapshot: [base](${browser.baseHeapSnapshotUrl}) / [head](${browser.headHeapSnapshotUrl})`, '', '', - '### 📦 Bundle Stats', + '## 📦 Bundle Stats', '', renderFrontendChunkReport(bundle.base, bundle.head), '', diff --git a/packages-private/diagnostics-frontend/test/__snapshots__/report.md b/packages-private/diagnostics-frontend/test/__snapshots__/report.md index 41d3aae727..144ff18305 100644 --- a/packages-private/diagnostics-frontend/test/__snapshots__/report.md +++ b/packages-private/diagnostics-frontend/test/__snapshots__/report.md @@ -1,12 +1,12 @@ ## 🖥 Frontend Diagnostics Report -| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| **Encoded network** | 1 MB | 1.1 MB | $\color{orange}{\text{+83 KB}}$ | 816 B | $\color{orange}{\text{+82 KB}}$ | $\color{orange}{\text{+84 KB}}$ | -| **Decoded body** | 3.5 MB | 3.7 MB | $\color{orange}{\text{+277 KB}}$ | 2.7 KB | $\color{orange}{\text{+274 KB}}$ | $\color{orange}{\text{+279 KB}}$ | -| **Same-origin encoded** | 1 MB | 1.1 MB | $\color{orange}{\text{+82 KB}}$ | 800 B | $\color{orange}{\text{+81 KB}}$ | $\color{orange}{\text{+82 KB}}$ | -| **Script encoded** | 918 KB | 991 KB | $\color{orange}{\text{+73 KB}}$ | 720 B | $\color{orange}{\text{+73 KB}}$ | $\color{orange}{\text{+74 KB}}$ | -| **Page-attributed memory** | 92 MB | 99 MB | $\color{orange}{\text{+7.3 MB}}$ | 72 KB | $\color{orange}{\text{+7.3 MB}}$ | $\color{orange}{\text{+7.4 MB}}$ | +| Metric | @ Base | @ Head | Δ | MAD | +| --- | ---: | ---: | ---: | ---: | +| **Encoded network** | 1 MB
± 10 KB | 1.1 MB
± 11 KB | $\color{orange}{\text{+83 KB}}$
$\color{orange}{\text{+8\\%}}$ | 15 KB | +| **Decoded body** | 3.5 MB
± 34 KB | 3.7 MB
± 37 KB | $\color{orange}{\text{+277 KB}}$
$\color{orange}{\text{+8\\%}}$ | 50 KB | +| **Same-origin encoded** | 1 MB
± 10 KB | 1.1 MB
± 11 KB | $\color{orange}{\text{+82 KB}}$
$\color{orange}{\text{+8\\%}}$ | 15 KB | +| **Script encoded** | 918 KB
± 9 KB | 991 KB
± 9.7 KB | $\color{orange}{\text{+73 KB}}$
$\color{orange}{\text{+8\\%}}$ | 13 KB | +| **Page-attributed memory** | 92 MB
± 900 KB | 99 MB
± 972 KB | $\color{orange}{\text{+7.3 MB}}$
$\color{orange}{\text{+8\\%}}$ | 1.3 MB | Only metrics showing significant changes are displayed. @@ -67,22 +67,22 @@
V8 heap snapshot statistics -| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| $\color{gray}{\rule{8pt}{8pt}}$ **Total** | 1 MB
± 10 KB | 1.1 MB
± 11 KB | $\text{+82 KB}$
$\color{orange}{\text{+8\\%}}$ | 800 B | $\text{+81 KB}$ | $\text{+82 KB}$ | -| | | | | | | | -|
$\color{orange}{\rule{8pt}{8pt}}$ **Code**200% → 200%
| 2 MB | 2.2 MB | $\color{orange}{\text{+163 KB}}$ | 1.6 KB | $\color{orange}{\text{+162 KB}}$ | $\color{orange}{\text{+165 KB}}$ | -|
$\color{red}{\rule{8pt}{8pt}}$ **Strings**300% → 300%
| 3.1 MB | 3.3 MB | $\color{orange}{\text{+245 KB}}$ | 2.4 KB | $\color{orange}{\text{+242 KB}}$ | $\color{orange}{\text{+247 KB}}$ | -|
$\color{cyan}{\rule{8pt}{8pt}}$ **JS arrays**400% → 400%
| 4.1 MB | 4.4 MB | $\color{orange}{\text{+326 KB}}$ | 3.2 KB | $\color{orange}{\text{+323 KB}}$ | $\color{orange}{\text{+330 KB}}$ | -|
$\color{green}{\rule{8pt}{8pt}}$ **Typed arrays**500% → 500%
| 5.1 MB | 5.5 MB | $\color{orange}{\text{+408 KB}}$ | 4 KB | $\color{orange}{\text{+404 KB}}$ | $\color{orange}{\text{+412 KB}}$ | -|
$\color{yellow}{\rule{8pt}{8pt}}$ **System objects**600% → 600%
| 6.1 MB | 6.6 MB | $\color{orange}{\text{+490 KB}}$ | 4.8 KB | $\color{orange}{\text{+485 KB}}$ | $\color{orange}{\text{+494 KB}}$ | -|
$\color{violet}{\rule{8pt}{8pt}}$ **Other JS objs**700% → 700%
| 7.1 MB | 7.7 MB | $\color{orange}{\text{+571 KB}}$ | 5.6 KB | $\color{orange}{\text{+566 KB}}$ | $\color{orange}{\text{+577 KB}}$ | -|
$\color{pink}{\rule{8pt}{8pt}}$ **Other non-JS objs**800% → 800%
| 8.2 MB | 8.8 MB | $\color{orange}{\text{+653 KB}}$ | 6.4 KB | $\color{orange}{\text{+646 KB}}$ | $\color{orange}{\text{+659 KB}}$ | +| Metric | @ Base | @ Head | Δ | MAD | +| --- | ---: | ---: | ---: | ---: | +| $\color{gray}{\rule{8pt}{8pt}}$ **Total** | 1 MB
± 10 KB | 1.1 MB
± 11 KB | $\text{+82 KB}$
$\text{+8\\%}$ | 15 KB | +| | | | | | +| $\color{orange}{\rule{8pt}{8pt}}$ **Code** | 2 MB | 2.2 MB | $\color{orange}{\text{+163 KB}}$ | 29 KB | +| $\color{red}{\rule{8pt}{8pt}}$ **Strings** | 3.1 MB | 3.3 MB | $\color{orange}{\text{+245 KB}}$ | 44 KB | +| $\color{cyan}{\rule{8pt}{8pt}}$ **JS arrays** | 4.1 MB | 4.4 MB | $\color{orange}{\text{+326 KB}}$ | 59 KB | +| $\color{green}{\rule{8pt}{8pt}}$ **Typed arrays** | 5.1 MB | 5.5 MB | $\color{orange}{\text{+408 KB}}$ | 74 KB | +| $\color{yellow}{\rule{8pt}{8pt}}$ **System objects** | 6.1 MB | 6.6 MB | $\color{orange}{\text{+490 KB}}$ | 88 KB | +| $\color{violet}{\rule{8pt}{8pt}}$ **Other JS objs** | 7.1 MB | 7.7 MB | $\color{orange}{\text{+571 KB}}$ | 103 KB | +| $\color{pink}{\rule{8pt}{8pt}}$ **Other non-JS objs** | 8.2 MB | 8.8 MB | $\color{orange}{\text{+653 KB}}$ | 118 KB | Download representative heap snapshot: [base](https://example.invalid/base) / [head](https://example.invalid/head)
-### 📦 Bundle Stats +## 📦 Bundle Stats
Chunk size diff (2 updated, 0 added, 0 removed) diff --git a/packages-private/diagnostics-frontend/test/report.test.ts b/packages-private/diagnostics-frontend/test/report.test.ts index c45961ccb9..463374aeec 100644 --- a/packages-private/diagnostics-frontend/test/report.test.ts +++ b/packages-private/diagnostics-frontend/test/report.test.ts @@ -9,7 +9,7 @@ import { dirname, join } from 'node:path'; import { afterAll, beforeAll, expect, test } from 'vitest'; import { collectBundleReport } from '../src/bundle/manifest'; import { renderFrontendDiagnosticsMarkdown } from '../src/report'; -import type { BrowserMetricsReport } from '../src/browser/types'; +import type { BrowserMeasurementSample, BrowserMetricsReport } from '../src/browser/types'; import type { VisualizerReport } from '../src/bundle/visualizer'; const bundleFixturesDir = join(import.meta.dirname, 'bundle/fixtures'); @@ -90,7 +90,15 @@ async function loadBrowserReport(name: 'base' | 'head') { return JSON.parse(await readFile(join(browserFixturesDir, `${name}.json`), 'utf8')) as BrowserMetricsReport; } -async function renderReport(detailedHtmlUrl: string | null) { +type BrowserReports = { + base: BrowserMetricsReport; + head: BrowserMetricsReport; +}; + +async function renderReport(detailedHtmlUrl: string | null, reports?: BrowserReports) { + const base = reports?.base ?? await loadBrowserReport('base'); + const head = reports?.head ?? await loadBrowserReport('head'); + return renderFrontendDiagnosticsMarkdown({ bundle: { base: await collectBundleReport(repoDirs.base), @@ -100,8 +108,8 @@ async function renderReport(detailedHtmlUrl: string | null) { visualizerArtifactUrl: 'https://example.invalid/treemap', }, browser: { - base: await loadBrowserReport('base'), - head: await loadBrowserReport('head'), + base, + head, baseHeapSnapshotUrl: 'https://example.invalid/base', headHeapSnapshotUrl: 'https://example.invalid/head', detailedHtmlUrl, @@ -109,6 +117,32 @@ async function renderReport(detailedHtmlUrl: string | null) { }); } +function withMetricSamples( + report: BrowserMetricsReport, + values: number[], + setValue: (sample: BrowserMeasurementSample, value: number) => void, +): BrowserMetricsReport { + const cloned = structuredClone(report); + const samples = values.map((value, index) => { + const sample = structuredClone(cloned.samples[index % cloned.samples.length]); + sample.round = index + 1; + setValue(sample, value); + return sample; + }); + + return { + ...cloned, + sampleCount: samples.length, + samples, + }; +} + +function requireMetricRow(markdown: string, label: string) { + const row = markdown.split('\n').find(line => line.startsWith(`| **${label}** |`)); + if (row == null) throw new Error(`Metric row not found: ${label}`); + return row; +} + /** * 出力をゴールデンファイルで固定する。 * 意図的に変更したときは `vitest -u` で更新し、__snapshots__ の差分もレビューすること。 @@ -117,6 +151,10 @@ test('renders one frontend diagnostics markdown report from bundle and browser d const markdown = await renderReport('https://example.invalid/html'); await expect(markdown).toMatchFileSnapshot('./__snapshots__/report.md'); + expect(markdown).toContain('| Metric | @ Base | @ Head | Δ | MAD |'); + expect(markdown).not.toContain('| Metric | @ Base | @ Head | Δ | MAD | Result |'); + expect(markdown).toContain('Requests by resource type'); + expect(markdown).toContain('## 📦 Bundle Stats'); }); test('omits the browser details link when no detailed html artifact was uploaded', async () => { @@ -124,3 +162,148 @@ test('omits the browser details link when no detailed html artifact was uploaded expect(markdown).not.toContain('View details'); }); + +test('renders the difference of independent medians instead of the paired median delta', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [100, 100, 100, 1_000, 1_000], + (sample, value) => { sample.network.requestCount = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [0, 0, 200, 200, 200], + (sample, value) => { sample.network.requestCount = value; }, + ); + + const row = requireMetricRow(await renderReport(null, { base, head }), 'Requests'); + + expect(row).toContain('100
± 0'); + expect(row).toContain('200
± 0'); + expect(row).toContain('$\\color{orange}{\\text{+100}}$
$\\color{orange}{\\text{+100\\\\%}}$'); + expect(row).toContain('| 0 |'); + expect(row.split('|')).toHaveLength(7); + expect(row).not.toMatch(/increase|decrease|within noise|inconclusive/); + expect(row).not.toContain('\\color{green}'); +}); + +test('hides a threshold-sized change that remains within observed noise', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [100, 110, 120], + (sample, value) => { sample.network.requestCount = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [110, 120, 130], + (sample, value) => { sample.network.requestCount = value; }, + ); + + const markdown = await renderReport(null, { base, head }); + + expect(markdown).not.toContain('| **Requests** |'); +}); + +test('renders a directional request count delta at the absolute threshold', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [100, 100, 100], + (sample, value) => { sample.network.requestCount = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [101, 101, 101], + (sample, value) => { sample.network.requestCount = value; }, + ); + + const row = requireMetricRow(await renderReport(null, { base, head }), 'Requests'); + + expect(row).toContain('$\\color{orange}{\\text{+1}}$'); +}); + +test('hides a directional byte change below the existing absolute threshold', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [1_000_000, 1_000_000, 1_000_000], + (sample, value) => { sample.network.totalEncodedBytes = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [1_005_000, 1_005_000, 1_005_000], + (sample, value) => { sample.network.totalEncodedBytes = value; }, + ); + + const markdown = await renderReport(null, { base, head }); + + expect(markdown).not.toContain('| **Encoded network** |'); +}); + +test('renders a directional encoded byte delta at the absolute threshold', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [1_000_000, 1_000_000, 1_000_000], + (sample, value) => { sample.network.totalEncodedBytes = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [1_010_000, 1_010_000, 1_010_000], + (sample, value) => { sample.network.totalEncodedBytes = value; }, + ); + + const row = requireMetricRow(await renderReport(null, { base, head }), 'Encoded network'); + + expect(row).toContain('$\\color{orange}{\\text{+10 KB}}$'); +}); + +test('colours absolute and relative deltas together when the row is significant', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [100_000_000, 100_000_000, 100_000_000], + (sample, value) => { sample.network.totalEncodedBytes = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [100_020_000, 100_020_000, 100_020_000], + (sample, value) => { sample.network.totalEncodedBytes = value; }, + ); + + const row = requireMetricRow(await renderReport(null, { base, head }), 'Encoded network'); + + expect(row).toContain('100 MB
± 0 B'); + expect(row).toContain('$\\color{orange}{\\text{+20 KB}}$
$\\color{orange}{\\text{+0\\\\%}}$'); + expect(row).toContain('| 0 B |'); + expect(row.split('|')).toHaveLength(7); +}); + +test('renders an unavailable relative delta when the base median is zero', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [0, 0, 0], + (sample, value) => { sample.network.requestCount = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [2, 2, 2], + (sample, value) => { sample.network.requestCount = value; }, + ); + + const row = requireMetricRow(await renderReport(null, { base, head }), 'Requests'); + + expect(row).toContain('$\\color{orange}{\\text{+2}}$
-'); + expect(row).not.toContain('increase'); +}); + +test('throws for a metric with fewer than two samples on one side', async () => { + const base = withMetricSamples( + await loadBrowserReport('base'), + [100], + (sample, value) => { sample.network.requestCount = value; }, + ); + const head = withMetricSamples( + await loadBrowserReport('head'), + [102, 102], + (sample, value) => { sample.network.requestCount = value; }, + ); + + await expect(renderReport(null, { base, head })) + .rejects.toThrow('At least two samples per side are required'); +}); diff --git a/packages-private/diagnostics-shared/package.json b/packages-private/diagnostics-shared/package.json index 1fd0a14682..7a0268b2df 100644 --- a/packages-private/diagnostics-shared/package.json +++ b/packages-private/diagnostics-shared/package.json @@ -7,6 +7,7 @@ "./format": "./src/format.ts", "./html": "./src/html.ts", "./stats": "./src/stats.ts", + "./metric-table": "./src/metric-table.ts", "./heap-snapshot": "./src/heap-snapshot/index.ts" }, "scripts": { diff --git a/packages-private/diagnostics-shared/src/heap-snapshot/render.ts b/packages-private/diagnostics-shared/src/heap-snapshot/render.ts index 7690c834c5..1f8631ff8b 100644 --- a/packages-private/diagnostics-shared/src/heap-snapshot/render.ts +++ b/packages-private/diagnostics-shared/src/heap-snapshot/render.ts @@ -3,13 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { - formatBytes, - formatDeltaBytes, - formatDeltaPercentInMdTable, - formatPercent, -} from '../format'; -import { mad, pairedDeltaSummary } from '../stats'; +import { formatBytes } from '../format'; +import { renderMetricComparisonTable } from '../metric-table'; import { heapSnapshotCategories, heapSnapshotCategory, @@ -24,15 +19,6 @@ function categoryValue(report: HeapSnapshotReport, category: HeapSnapshotCategor return report.summary.categories[category]; } -function categorySampleSpread(report: HeapSnapshotReport, category: HeapSnapshotCategory) { - const values = report.samples - .map(sample => sample.data.categories[category]) - .filter(value => Number.isFinite(value)); - if (values.length < 2) throw new Error(`Not enough samples for category ${category}`); - - return mad(values); -} - function swatch(category: HeapSnapshotCategory) { return `$\\color{${heapSnapshotCategory[category].color}}{\\rule{8pt}{8pt}}$ **${heapSnapshotCategory[category].label}**`; } @@ -41,38 +27,19 @@ function swatch(category: HeapSnapshotCategory) { * base / head のheap snapshotをカテゴリ別に比較するMarkdownテーブルを描画する。 */ export function renderHeapSnapshotTable(base: HeapSnapshotReport, head: HeapSnapshotReport) { - const lines = [ - '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', - ]; - const baseTotal = categoryValue(base, 'total'); - const headTotal = categoryValue(head, 'total'); - - for (const category of heapSnapshotCategories) { - const baseValue = categoryValue(base, category); - const headValue = categoryValue(head, category); - const summary = pairedDeltaSummary(base.samples, head.samples, sample => sample.data.categories[category]); - const deltaColumns = `${formatBytes(summary.mad)} | ${formatDeltaBytes(summary.min, byteColorThreshold)} | ${formatDeltaBytes(summary.max, byteColorThreshold)}`; - - if (category === 'total') { - // Totalだけはばらつきと変化率も併記する - const percent = summary.median * 100 / baseValue; - const deltaMedian = `${formatDeltaBytes(summary.median, byteColorThreshold)}
${formatDeltaPercentInMdTable(percent, 0.1)}`; - const baseText = `${formatBytes(baseValue)}
± ${formatBytes(categorySampleSpread(base, category))}`; - const headText = `${formatBytes(headValue)}
± ${formatBytes(categorySampleSpread(head, category))}`; - lines.push(`| ${swatch(category)} | ${baseText} | ${headText} | ${deltaMedian} | ${deltaColumns} |`); - lines.push('| | | | | | | |'); - } else { - // 各カテゴリはTotalに占める割合の推移をdetailsで見せる - const basePercent = formatPercent((baseValue * 100) / baseTotal); - const headPercent = formatPercent((headValue * 100) / headTotal); - const metricText = `
${swatch(category)}${basePercent} → ${headPercent}
`; - lines.push(`| ${metricText} | ${formatBytes(baseValue)} | ${formatBytes(headValue)} | ${formatDeltaBytes(summary.median, byteColorThreshold)} | ${deltaColumns} |`); - } - } - - if (lines.length === 2) return null; - return lines.join('\n'); + return renderMetricComparisonTable( + base.samples, + head.samples, + heapSnapshotCategories.map(category => ({ + label: swatch(category), + getValue: sample => sample.data.categories[category], + formatValue: formatBytes, + absoluteThreshold: byteColorThreshold, + showMedianMad: category === 'total', + showDeltaPercentage: category === 'total', + separatorAfter: category === 'total', + })), + ); } const sankeyChildMinRatio = 0.3; diff --git a/packages-private/diagnostics-shared/src/metric-table.ts b/packages-private/diagnostics-shared/src/metric-table.ts new file mode 100644 index 0000000000..62f57aa483 --- /dev/null +++ b/packages-private/diagnostics-shared/src/metric-table.ts @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { formatColoredDelta, formatDeltaPercentInMdTable } from './format'; +import { + independentDeltaSummary, + isOutsideObservedNoise, + type IndependentDeltaSummary, +} from './stats'; + +export type MetricComparisonRow = { + label: string; + getValue: (sample: T) => number; + formatValue: (value: number) => string; + absoluteThreshold: number; + showMedianMad?: boolean; + showDeltaPercentage?: boolean; + separatorAfter?: boolean; +}; + +export type MetricComparisonTableOptions = { + onlySignificantChanges?: boolean; +}; + +function isSignificant(summary: IndependentDeltaSummary, absoluteThreshold: number) { + return isOutsideObservedNoise(summary) && Math.abs(summary.delta) >= absoluteThreshold; +} + +function formatMedian( + value: number, + spread: number, + row: MetricComparisonRow, +) { + const formatted = row.formatValue(value); + if (row.showMedianMad === false) return formatted; + return `${formatted}
± ${row.formatValue(spread)}`; +} + +function formatDelta( + summary: IndependentDeltaSummary, + row: MetricComparisonRow, + significant: boolean, +) { + const colorThreshold = significant ? 0 : Number.POSITIVE_INFINITY; + const absolute = formatColoredDelta(summary.delta, row.formatValue, colorThreshold); + if (row.showDeltaPercentage === false) return absolute; + + const percentage = summary.baseMedian === 0 + ? '-' + : formatDeltaPercentInMdTable(summary.delta * 100 / summary.baseMedian, colorThreshold); + return `${absolute}
${percentage}`; +} + +export function renderMetricComparisonTable( + baseSamples: T[], + headSamples: T[], + rows: MetricComparisonRow[], + options: MetricComparisonTableOptions = {}, +): string { + const lines: string[] = []; + + for (const row of rows) { + const summary = independentDeltaSummary(baseSamples, headSamples, row.getValue); + const significant = isSignificant(summary, row.absoluteThreshold); + if (options.onlySignificantChanges === true && !significant) continue; + + lines.push(`| ${row.label} | ${formatMedian(summary.baseMedian, summary.baseMad, row)} | ${formatMedian(summary.headMedian, summary.headMad, row)} | ${formatDelta(summary, row, significant)} | ${row.formatValue(summary.combinedMad)} |`); + if (row.separatorAfter === true) lines.push('| | | | | |'); + } + + if (lines.length === 0) return '**(No data)**'; + + return [ + '| Metric | @ Base | @ Head | Δ | MAD |', + '| --- | ---: | ---: | ---: | ---: |', + ...lines, + ].join('\n'); +} diff --git a/packages-private/diagnostics-shared/src/stats.ts b/packages-private/diagnostics-shared/src/stats.ts index 7516242c32..20fd7b1347 100644 --- a/packages-private/diagnostics-shared/src/stats.ts +++ b/packages-private/diagnostics-shared/src/stats.ts @@ -37,6 +37,49 @@ export function sampleSpread(values: (number | null | undefined)[]) { return mad(finiteValues); } +export type IndependentDeltaSummary = { + baseMedian: number; + headMedian: number; + delta: number; + baseMad: number; + headMad: number; + combinedMad: number; + baseSamples: number; + headSamples: number; +}; + +export function independentDeltaSummary( + baseSamples: T[], + headSamples: T[], + getValue: (sample: T) => number, +): IndependentDeltaSummary { + const baseValues = baseSamples.map(getValue); + const headValues = headSamples.map(getValue); + if (baseValues.length < 2 || headValues.length < 2) { + throw new Error('At least two samples per side are required'); + } + + const baseMedian = median(baseValues); + const headMedian = median(headValues); + const baseMad = mad(baseValues); + const headMad = mad(headValues); + + return { + baseMedian, + headMedian, + delta: headMedian - baseMedian, + baseMad, + headMad, + combinedMad: Math.hypot(baseMad, headMad), + baseSamples: baseValues.length, + headSamples: headValues.length, + }; +} + +export function isOutsideObservedNoise(summary: IndependentDeltaSummary) { + return Math.abs(summary.delta) > summary.combinedMad * 3; +} + type RoundedSample = { round: number }; function indexByRound(samples: T[]) { diff --git a/packages-private/diagnostics-shared/test/heap-snapshot-render.test.ts b/packages-private/diagnostics-shared/test/heap-snapshot-render.test.ts new file mode 100644 index 0000000000..ba56dbb659 --- /dev/null +++ b/packages-private/diagnostics-shared/test/heap-snapshot-render.test.ts @@ -0,0 +1,103 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { describe, expect, test } from 'vitest'; +import { + createEmptyHeapSnapshotData, + renderHeapSnapshotTable, + summarizeHeapSnapshotDataSamples, + type HeapSnapshotData, + type HeapSnapshotReport, +} from '../src/heap-snapshot'; + +function snapshot(total: number): HeapSnapshotData { + const data = createEmptyHeapSnapshotData(); + data.categories.total = total; + return data; +} + +function report(totals: number[]): HeapSnapshotReport { + const samples = totals.map((total, index) => ({ + round: index + 1, + data: snapshot(total), + })); + const summary = summarizeHeapSnapshotDataSamples(samples, sample => sample.data); + if (summary == null) throw new Error('expected heap snapshot samples to produce a summary'); + + return { + summary, + samples, + }; +} + +function totalRow(markdown: string) { + const row = markdown.split('\n').find(line => line.includes('**Total**')); + if (row === undefined) throw new Error('expected heap snapshot table to contain a Total row'); + return row; +} + +describe('renderHeapSnapshotTable', () => { + test('leaves a threshold-sized delta uncoloured when it is within observed noise', () => { + const row = totalRow(renderHeapSnapshotTable( + report([1_000_000, 1_100_000, 1_200_000]), + report([1_100_000, 1_200_000, 1_300_000]), + )); + + expect(row).toContain('$\\text{+100 KB}$'); + expect(row).not.toContain('within noise'); + expect(row).not.toContain('\\color{orange}'); + }); + + test('colours both delta lines for a clear increase at or above the absolute threshold', () => { + const row = totalRow(renderHeapSnapshotTable( + report([1_000_000, 1_000_000, 1_000_000]), + report([1_200_000, 1_200_000, 1_200_000]), + )); + + expect(row).toContain('$\\color{orange}{\\text{+200 KB}}$'); + expect(row).toContain('$\\color{orange}{\\text{+20\\\\%}}$'); + expect(row).not.toContain('increase'); + }); + + test('leaves both delta lines uncoloured below the absolute threshold', () => { + const row = totalRow(renderHeapSnapshotTable( + report([1_000_000, 1_000_000, 1_000_000]), + report([1_050_000, 1_050_000, 1_050_000]), + )); + + expect(row).toContain('$\\text{+50 KB}$
$\\text{+5\\\\%}$'); + expect(row.split('|')[4]).not.toContain('\\color{'); + }); + + test('throws when only one snapshot per side reaches the renderer', () => { + expect(() => renderHeapSnapshotTable( + report([1_000_000]), + report([1_200_000]), + )).toThrow('At least two samples per side are required'); + }); + + test('uses two-line formatting only for Total and puts a five-column separator after it', () => { + const table = renderHeapSnapshotTable( + report([1_000_000, 1_000_000, 1_000_000]), + report([1_200_000, 1_200_000, 1_200_000]), + ); + const lines = table.split('\n'); + const totalIndex = lines.findIndex(line => line.includes('**Total**')); + const categoryRow = lines.find(line => line.includes('**Code**')); + + expect(lines.slice(0, 2)).toStrictEqual([ + '| Metric | @ Base | @ Head | Δ | MAD |', + '| --- | ---: | ---: | ---: | ---: |', + ]); + expect(table).not.toContain('Result'); + expect(totalIndex).toBeGreaterThan(1); + expect(lines[totalIndex].match(/
/g)).toHaveLength(3); + expect(lines[totalIndex + 1]).toBe('| | | | | |'); + expect(categoryRow).toBeDefined(); + expect(categoryRow).not.toContain('
'); + expect(categoryRow).not.toContain('
'); + expect(categoryRow).not.toContain('→'); + }); +}); diff --git a/packages-private/diagnostics-shared/test/metric-table.test.ts b/packages-private/diagnostics-shared/test/metric-table.test.ts new file mode 100644 index 0000000000..fb638acf48 --- /dev/null +++ b/packages-private/diagnostics-shared/test/metric-table.test.ts @@ -0,0 +1,145 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { describe, expect, test } from 'vitest'; +import { + renderMetricComparisonTable, + type MetricComparisonRow, +} from '../src/metric-table'; + +type Sample = { value: number }; + +const defaultRow: MetricComparisonRow = { + label: 'Metric', + getValue: sample => sample.value, + formatValue: value => `${value} units`, + absoluteThreshold: 10, +}; + +function samples(...values: number[]): Sample[] { + return values.map(value => ({ value })); +} + +describe('renderMetricComparisonTable', () => { + test('renders the fixed five-column layout, raw label, MAD, and percentage by default', () => { + const table = renderMetricComparisonTable( + samples(100, 100, 100), + samples(120, 120, 120), + [defaultRow], + ); + + expect(table.split('\n')).toStrictEqual([ + '| Metric | @ Base | @ Head | Δ | MAD |', + '| --- | ---: | ---: | ---: | ---: |', + '| Metric | 100 units
± 0 units | 120 units
± 0 units | $\\color{orange}{\\text{+20 units}}$
$\\color{orange}{\\text{+20\\\\%}}$ | 0 units |', + ]); + }); + + test('can hide second lines and insert a five-column separator row', () => { + const table = renderMetricComparisonTable( + samples(100, 100, 100), + samples(120, 120, 120), + [{ + ...defaultRow, + showMedianMad: false, + showDeltaPercentage: false, + separatorAfter: true, + }], + ); + + expect(table.split('\n')).toStrictEqual([ + '| Metric | @ Base | @ Head | Δ | MAD |', + '| --- | ---: | ---: | ---: | ---: |', + '| Metric | 100 units | 120 units | $\\color{orange}{\\text{+20 units}}$ | 0 units |', + '| | | | | |', + ]); + }); + + test('leaves both delta lines uncolored below the absolute threshold and can filter the row', () => { + const base = samples(100, 100, 100); + const head = samples(109, 109, 109); + const table = renderMetricComparisonTable(base, head, [defaultRow]); + + expect(table).toContain('$\\text{+9 units}$
$\\text{+9\\\\%}$'); + expect(table).not.toContain('\\color{'); + expect(renderMetricComparisonTable(base, head, [defaultRow], { + onlySignificantChanges: true, + })).toBe('**(No data)**'); + }); + + test('renders a no-data marker when no rows are configured', () => { + expect(renderMetricComparisonTable( + samples(100, 100, 100), + samples(120, 120, 120), + [], + )).toBe('**(No data)**'); + }); + + test('treats the absolute threshold itself as significant', () => { + const table = renderMetricComparisonTable( + samples(100, 100, 100), + samples(110, 110, 110), + [defaultRow], + { onlySignificantChanges: true }, + ); + + expect(table).toContain('$\\color{orange}{\\text{+10 units}}$'); + expect(table).toContain('$\\color{orange}{\\text{+10\\\\%}}$'); + }); + + test('requires the delta to be strictly outside three combined MADs', () => { + const inside = renderMetricComparisonTable( + samples(100, 110, 120), + samples(109, 119, 129), + [{ ...defaultRow, absoluteThreshold: 1 }], + ); + const boundary = renderMetricComparisonTable( + samples(100, 100, 100), + samples(120, 130, 140), + [defaultRow], + ); + const outside = renderMetricComparisonTable( + samples(100, 100, 100), + samples(121, 131, 141), + [defaultRow], + ); + + expect(inside).toContain('$\\text{+9 units}$'); + expect(inside).not.toContain('\\color{'); + expect(boundary).toContain('$\\text{+30 units}$'); + expect(boundary).not.toContain('\\color{'); + expect(outside).toContain('$\\color{orange}{\\text{+31 units}}$'); + expect(outside).toContain('$\\color{orange}{\\text{+31\\\\%}}$'); + }); + + test('colours a significant decrease green on both delta lines', () => { + const table = renderMetricComparisonTable( + samples(120, 120, 120), + samples(100, 100, 100), + [defaultRow], + ); + + expect(table).toContain('$\\color{green}{\\text{-20 units}}$'); + expect(table).toContain('$\\color{green}{\\text{-16.7\\\\%}}$'); + }); + + test('shows a dash for percentage when the base median is zero', () => { + const table = renderMetricComparisonTable( + samples(0, 0, 0), + samples(20, 20, 20), + [defaultRow], + ); + + expect(table).toContain('$\\color{orange}{\\text{+20 units}}$
-'); + }); + + test('propagates the minimum sample contract', () => { + expect(() => renderMetricComparisonTable( + samples(100), + samples(120, 120), + [defaultRow], + )).toThrow('At least two samples per side are required'); + }); +}); diff --git a/packages-private/diagnostics-shared/test/stats.test.ts b/packages-private/diagnostics-shared/test/stats.test.ts index 1d57f0d67d..22da6a6531 100644 --- a/packages-private/diagnostics-shared/test/stats.test.ts +++ b/packages-private/diagnostics-shared/test/stats.test.ts @@ -4,7 +4,16 @@ */ import { describe, expect, test } from 'vitest'; -import { finiteMedian, mad, median, pairedDeltaSummary, sampleSpread } from '../src/stats'; +import { + finiteMedian, + independentDeltaSummary, + isOutsideObservedNoise, + mad, + median, + pairedDeltaSummary, + sampleSpread, + type IndependentDeltaSummary, +} from '../src/stats'; describe('median', () => { test('takes the middle value of an odd-length sample', () => { @@ -107,3 +116,79 @@ describe('pairedDeltaSummary', () => { expect(pairedDeltaSummary(warmupBase, warmupHead, sample => sample.value).samples).toBe(3); }); }); + +describe('independentDeltaSummary', () => { + const base = [290_000, 292_900, 295_800, 298_700, 301_600] + .map((value, index) => ({ round: index + 1, value })); + const head = [292_900, 296_300, 298_700, 301_600, 290_000] + .map((value, index) => ({ round: index + 1, value })); + + test('uses the difference of independent medians instead of the paired median', () => { + expect(pairedDeltaSummary(base, head, sample => sample.value).median).toBe(2_900); + + const summary = independentDeltaSummary(base, head, sample => sample.value); + expect(summary).toMatchObject({ + baseMedian: 295_800, + headMedian: 296_300, + delta: 500, + baseMad: 2_900, + headMad: 3_400, + baseSamples: 5, + headSamples: 5, + }); + expect(summary.combinedMad).toBeCloseTo(Math.hypot(2_900, 3_400)); + }); + + test('allows unequal sample counts', () => { + const summary = independentDeltaSummary( + [{ value: 10 }, { value: 20 }], + [{ value: 20 }, { value: 30 }, { value: 40 }], + sample => sample.value, + ); + + expect(summary).toStrictEqual({ + baseMedian: 15, + headMedian: 30, + delta: 15, + baseMad: 5, + headMad: 10, + combinedMad: Math.hypot(5, 10), + baseSamples: 2, + headSamples: 3, + }); + }); + + test('throws when either side has fewer than two samples', () => { + expect(() => independentDeltaSummary( + [{ value: 10 }], + [{ value: 20 }, { value: 20 }], + sample => sample.value, + )).toThrow('At least two samples per side are required'); + + expect(() => independentDeltaSummary( + [{ value: 10 }, { value: 10 }], + [{ value: 20 }], + sample => sample.value, + )).toThrow('At least two samples per side are required'); + }); + + test('treats exactly three combined MADs as noise and only larger deltas as outside noise', () => { + function summary(delta: number, combinedMad: number): IndependentDeltaSummary { + return { + baseMedian: 100, + headMedian: 100 + delta, + delta, + baseMad: 0, + headMad: combinedMad, + combinedMad, + baseSamples: 3, + headSamples: 3, + }; + } + + expect(isOutsideObservedNoise(summary(29, 10))).toBe(false); + expect(isOutsideObservedNoise(summary(30, 10))).toBe(false); + expect(isOutsideObservedNoise(summary(31, 10))).toBe(true); + expect(isOutsideObservedNoise(summary(-31, 10))).toBe(true); + }); +});