mirror of
https://github.com/misskey-dev/misskey.git
synced 2026-07-25 15:35:14 +02:00
refactor
This commit is contained in:
13
.github/scripts/chrome.mts
vendored
13
.github/scripts/chrome.mts
vendored
@@ -4,11 +4,11 @@
|
||||
*/
|
||||
|
||||
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:child_process';
|
||||
import { copyFile, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import * as util from './utility.mts';
|
||||
import { heapSnapshotCategory, HeapSnapshotData } from './heap-snapshot-util.mts';
|
||||
import type { HeapSnapshotData } from './heap-snapshot-util.mts';
|
||||
|
||||
type ChromeHandle = {
|
||||
process: ChildProcessWithoutNullStreams;
|
||||
@@ -339,6 +339,15 @@ export class Chrome {
|
||||
}
|
||||
}
|
||||
|
||||
static async with<T>(label: string, options: ChromeOptions, callback: (chrome: Chrome) => T | Promise<T>): Promise<T> {
|
||||
const chrome = await Chrome.create(label, options);
|
||||
try {
|
||||
return await callback(chrome);
|
||||
} finally {
|
||||
await chrome.close();
|
||||
}
|
||||
}
|
||||
|
||||
public async enableNetworkTracking() {
|
||||
const requests = new Map<string, NetworkRequest>();
|
||||
|
||||
|
||||
320
.github/scripts/heap-snapshot-util.mts
vendored
320
.github/scripts/heap-snapshot-util.mts
vendored
@@ -32,6 +32,326 @@ export type HeapSnapshotReport = {
|
||||
}[];
|
||||
};
|
||||
|
||||
export const defaultHeapSnapshotBreakdownTopN = 6;
|
||||
|
||||
export function createEmptyHeapSnapshotData(): HeapSnapshotData {
|
||||
const categories = {} as HeapSnapshotData['categories'];
|
||||
const nodeCounts = {} as HeapSnapshotData['nodeCounts'];
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
categories[category] = 0;
|
||||
nodeCounts[category] = 0;
|
||||
}
|
||||
return {
|
||||
categories,
|
||||
nodeCounts,
|
||||
breakdowns: {} as HeapSnapshotData['breakdowns'],
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeHeapSnapshotBreakdownLabel(value: unknown, fallback = 'unknown') {
|
||||
const label = String(value ?? '').replace(/\s+/g, ' ').trim();
|
||||
if (label === '') return fallback;
|
||||
if (label.length <= 80) return label;
|
||||
return `${label.slice(0, 77)}...`;
|
||||
}
|
||||
|
||||
function classifyHeapSnapshotBreakdown(category: keyof typeof heapSnapshotCategory, type: string, name: string) {
|
||||
if (category === 'strings') return type;
|
||||
|
||||
if (category === 'jsArrays') {
|
||||
if (type === 'array elements') return 'Array elements';
|
||||
if (type === 'object' && name === 'Array') return 'Array objects';
|
||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`);
|
||||
}
|
||||
|
||||
if (category === 'typedArrays') {
|
||||
if (name === 'system / JSArrayBufferData') return 'ArrayBuffer data';
|
||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`);
|
||||
}
|
||||
|
||||
if (category === 'systemObjects') {
|
||||
if (name.startsWith('system /')) return sanitizeHeapSnapshotBreakdownLabel(name);
|
||||
if (name.startsWith('(system ')) return sanitizeHeapSnapshotBreakdownLabel(name);
|
||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
|
||||
}
|
||||
|
||||
if (category === 'otherJsObjects') {
|
||||
if (type === 'object') return sanitizeHeapSnapshotBreakdownLabel(`object: ${name}`, 'object: unknown');
|
||||
return type;
|
||||
}
|
||||
|
||||
if (category === 'otherNonJsObjects') {
|
||||
if (type === 'extra native bytes') return 'Extra native bytes';
|
||||
if (type === 'native') return sanitizeHeapSnapshotBreakdownLabel(`native: ${name}`, 'native: unknown');
|
||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
|
||||
}
|
||||
|
||||
if (category === 'code') {
|
||||
const lowerName = name.toLowerCase();
|
||||
if (lowerName.includes('bytecode')) return 'bytecode';
|
||||
if (lowerName.includes('builtin')) return 'builtins';
|
||||
if (lowerName.includes('regexp')) return 'regexp code';
|
||||
if (lowerName.includes('stub')) return 'stubs';
|
||||
return sanitizeHeapSnapshotBreakdownLabel(`code: ${name}`, 'code: unknown');
|
||||
}
|
||||
|
||||
return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type);
|
||||
}
|
||||
|
||||
export function collapseHeapSnapshotBreakdown(breakdown: Record<string, number>, topN = defaultHeapSnapshotBreakdownTopN) {
|
||||
const entries = Object.entries(breakdown)
|
||||
.filter(([, value]) => value > 0)
|
||||
.toSorted((a, b) => b[1] - a[1]);
|
||||
|
||||
const topEntries = entries.slice(0, topN);
|
||||
const otherValue = entries
|
||||
.slice(topN)
|
||||
.reduce((sum, [, value]) => sum + value, 0);
|
||||
|
||||
const collapsed = Object.fromEntries(topEntries);
|
||||
if (otherValue > 0) collapsed.Other = otherValue;
|
||||
return collapsed;
|
||||
}
|
||||
|
||||
export function collapseHeapSnapshotBreakdowns(
|
||||
breakdowns: Partial<Record<keyof typeof heapSnapshotCategory, Record<string, number>>>,
|
||||
topN = defaultHeapSnapshotBreakdownTopN,
|
||||
) {
|
||||
const collapsed = {} as NonNullable<HeapSnapshotData['breakdowns']>;
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
if (category === 'total') continue;
|
||||
|
||||
const categoryBreakdown = breakdowns[category];
|
||||
if (categoryBreakdown == null) continue;
|
||||
|
||||
const collapsedCategory = collapseHeapSnapshotBreakdown(categoryBreakdown, topN);
|
||||
if (Object.keys(collapsedCategory).length > 0) {
|
||||
collapsed[category] = collapsedCategory;
|
||||
}
|
||||
}
|
||||
|
||||
return collapsed;
|
||||
}
|
||||
|
||||
// Keep these buckets aligned with Chrome DevTools' heap snapshot Statistics view.
|
||||
export function analyzeHeapSnapshot(snapshot: any, options: { breakdownTopN?: number } = {}): HeapSnapshotData {
|
||||
const meta = snapshot?.snapshot?.meta;
|
||||
const nodes = snapshot?.nodes;
|
||||
const edges = snapshot?.edges;
|
||||
const strings = snapshot?.strings;
|
||||
if (meta == null || !Array.isArray(nodes) || !Array.isArray(edges) || !Array.isArray(strings)) {
|
||||
throw new Error('Invalid heap snapshot format');
|
||||
}
|
||||
|
||||
const nodeFields = meta.node_fields;
|
||||
if (!Array.isArray(nodeFields)) throw new Error('Invalid heap snapshot node fields');
|
||||
const edgeFields = meta.edge_fields;
|
||||
if (!Array.isArray(edgeFields)) throw new Error('Invalid heap snapshot edge fields');
|
||||
|
||||
const typeOffset = nodeFields.indexOf('type');
|
||||
const nameOffset = nodeFields.indexOf('name');
|
||||
const selfSizeOffset = nodeFields.indexOf('self_size');
|
||||
const edgeCountOffset = nodeFields.indexOf('edge_count');
|
||||
if (typeOffset < 0 || nameOffset < 0 || selfSizeOffset < 0 || edgeCountOffset < 0) {
|
||||
throw new Error('Heap snapshot is missing required node fields');
|
||||
}
|
||||
const edgeTypeOffset = edgeFields.indexOf('type');
|
||||
const edgeNameOffset = edgeFields.indexOf('name_or_index');
|
||||
const edgeToNodeOffset = edgeFields.indexOf('to_node');
|
||||
if (edgeTypeOffset < 0 || edgeNameOffset < 0 || edgeToNodeOffset < 0) {
|
||||
throw new Error('Heap snapshot is missing required edge fields');
|
||||
}
|
||||
|
||||
const nodeTypeNames = meta.node_types?.[typeOffset];
|
||||
if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types');
|
||||
const edgeTypeNames = meta.edge_types?.[edgeTypeOffset];
|
||||
if (!Array.isArray(edgeTypeNames)) throw new Error('Invalid heap snapshot edge types');
|
||||
|
||||
const nodeFieldCount = nodeFields.length;
|
||||
const edgeFieldCount = edgeFields.length;
|
||||
const nativeType = nodeTypeNames.indexOf('native');
|
||||
const codeType = nodeTypeNames.indexOf('code');
|
||||
const hiddenType = nodeTypeNames.indexOf('hidden');
|
||||
const stringTypes = new Set([
|
||||
nodeTypeNames.indexOf('string'),
|
||||
nodeTypeNames.indexOf('concatenated string'),
|
||||
nodeTypeNames.indexOf('sliced string'),
|
||||
]);
|
||||
const internalEdgeType = edgeTypeNames.indexOf('internal');
|
||||
const extraNativeBytes = Number.isFinite(snapshot.snapshot.extra_native_bytes) ? snapshot.snapshot.extra_native_bytes : 0;
|
||||
const { categories, nodeCounts } = createEmptyHeapSnapshotData();
|
||||
const breakdowns = {} as Record<keyof typeof heapSnapshotCategory, Record<string, number>>;
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
if (category !== 'total') breakdowns[category] = {};
|
||||
}
|
||||
|
||||
function addValue(map: Record<string, number>, key: string, value: number) {
|
||||
map[key] = (map[key] ?? 0) + value;
|
||||
}
|
||||
|
||||
const edgeStartIndexes = new Map<number, number>();
|
||||
const retainerCounts = new Map<number, number>();
|
||||
let edgeIndex = 0;
|
||||
for (let nodeIndex = 0; nodeIndex < nodes.length; nodeIndex += nodeFieldCount) {
|
||||
edgeStartIndexes.set(nodeIndex, edgeIndex);
|
||||
const edgeCount = nodes[nodeIndex + edgeCountOffset] ?? 0;
|
||||
for (let i = 0; i < edgeCount; i++, edgeIndex += edgeFieldCount) {
|
||||
const toNodeIndex = edges[edgeIndex + edgeToNodeOffset];
|
||||
retainerCounts.set(toNodeIndex, (retainerCounts.get(toNodeIndex) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const jsArrayElementNodeIndexes = new Set<number>();
|
||||
|
||||
function addCategoryValue(category: keyof typeof heapSnapshotCategory, value: number, type: string, name: string, nodeIndex: number | null = null) {
|
||||
if (value <= 0) return;
|
||||
categories[category] += value;
|
||||
addValue(breakdowns[category], classifyHeapSnapshotBreakdown(category, type, name), value);
|
||||
if (nodeIndex != null) nodeCounts[category]++;
|
||||
}
|
||||
|
||||
function addJsArrayElementSize(nodeIndex: number) {
|
||||
const beginEdgeIndex = edgeStartIndexes.get(nodeIndex) ?? 0;
|
||||
const edgeCount = nodes[nodeIndex + edgeCountOffset] ?? 0;
|
||||
for (let i = 0, currentEdgeIndex = beginEdgeIndex; i < edgeCount; i++, currentEdgeIndex += edgeFieldCount) {
|
||||
const edgeType = edges[currentEdgeIndex + edgeTypeOffset];
|
||||
if (edgeType !== internalEdgeType) continue;
|
||||
|
||||
const edgeName = strings[edges[currentEdgeIndex + edgeNameOffset]];
|
||||
if (edgeName !== 'elements') continue;
|
||||
|
||||
const elementsNodeIndex = edges[currentEdgeIndex + edgeToNodeOffset];
|
||||
if ((retainerCounts.get(elementsNodeIndex) ?? 0) === 1) {
|
||||
const elementsSize = nodes[elementsNodeIndex + selfSizeOffset] ?? 0;
|
||||
addCategoryValue('jsArrays', elementsSize, 'array elements', 'Array elements', elementsNodeIndex);
|
||||
jsArrayElementNodeIndexes.add(elementsNodeIndex);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (extraNativeBytes > 0) {
|
||||
addCategoryValue('otherNonJsObjects', extraNativeBytes, 'extra native bytes', 'extra native bytes');
|
||||
}
|
||||
|
||||
for (let nodeIndex = 0; nodeIndex < nodes.length; nodeIndex += nodeFieldCount) {
|
||||
const typeId = nodes[nodeIndex + typeOffset];
|
||||
const type = nodeTypeNames[typeId] ?? 'unknown';
|
||||
const name = strings[nodes[nodeIndex + nameOffset]] ?? '';
|
||||
const selfSize = nodes[nodeIndex + selfSizeOffset] ?? 0;
|
||||
categories.total += selfSize;
|
||||
nodeCounts.total++;
|
||||
|
||||
if (typeId === hiddenType) {
|
||||
addCategoryValue('systemObjects', selfSize, type, name, nodeIndex);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeId === nativeType) {
|
||||
if (name === 'system / JSArrayBufferData') {
|
||||
addCategoryValue('typedArrays', selfSize, type, name, nodeIndex);
|
||||
} else {
|
||||
addCategoryValue('otherNonJsObjects', selfSize, type, name, nodeIndex);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeId === codeType) {
|
||||
addCategoryValue('code', selfSize, type, name, nodeIndex);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (stringTypes.has(typeId)) {
|
||||
addCategoryValue('strings', selfSize, type, name, nodeIndex);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (name === 'Array') {
|
||||
addCategoryValue('jsArrays', selfSize, type, name, nodeIndex);
|
||||
addJsArrayElementSize(nodeIndex);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
categories.total += extraNativeBytes;
|
||||
|
||||
for (let nodeIndex = 0; nodeIndex < nodes.length; nodeIndex += nodeFieldCount) {
|
||||
if (jsArrayElementNodeIndexes.has(nodeIndex)) continue;
|
||||
|
||||
const typeId = nodes[nodeIndex + typeOffset];
|
||||
if (typeId === hiddenType || typeId === nativeType || typeId === codeType || stringTypes.has(typeId)) continue;
|
||||
|
||||
const name = strings[nodes[nodeIndex + nameOffset]] ?? '';
|
||||
if (name === 'Array') continue;
|
||||
|
||||
const type = nodeTypeNames[typeId] ?? 'unknown';
|
||||
const selfSize = nodes[nodeIndex + selfSizeOffset] ?? 0;
|
||||
addCategoryValue('otherJsObjects', selfSize, type, name, nodeIndex);
|
||||
}
|
||||
|
||||
return {
|
||||
categories,
|
||||
nodeCounts,
|
||||
breakdowns: collapseHeapSnapshotBreakdowns(breakdowns, options.breakdownTopN),
|
||||
};
|
||||
}
|
||||
|
||||
function finiteMedian(values: (number | null | undefined)[]) {
|
||||
const finiteValues = values.filter(value => Number.isFinite(value)) as number[];
|
||||
if (finiteValues.length === 0) return null;
|
||||
return util.median(finiteValues);
|
||||
}
|
||||
|
||||
export function summarizeHeapSnapshotDataSamples<T>(
|
||||
samples: T[],
|
||||
getData: (sample: T) => HeapSnapshotData | null | undefined,
|
||||
options: { breakdownTopN?: number } = {},
|
||||
) {
|
||||
const data = samples.map(getData);
|
||||
const categories = {} as HeapSnapshotData['categories'];
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
const value = finiteMedian(data.map(snapshot => snapshot?.categories?.[category]));
|
||||
if (value != null) categories[category] = value;
|
||||
}
|
||||
|
||||
const nodeCounts = {} as HeapSnapshotData['nodeCounts'];
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
const value = finiteMedian(data.map(snapshot => snapshot?.nodeCounts?.[category]));
|
||||
if (value != null) nodeCounts[category] = value;
|
||||
}
|
||||
|
||||
if (Object.keys(categories).length === 0) return null;
|
||||
|
||||
const breakdowns = {} as NonNullable<HeapSnapshotData['breakdowns']>;
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
if (category === 'total') continue;
|
||||
|
||||
const childKeys = new Set<string>();
|
||||
for (const snapshot of data) {
|
||||
for (const childKey of Object.keys(snapshot?.breakdowns?.[category] ?? {})) {
|
||||
childKeys.add(childKey);
|
||||
}
|
||||
}
|
||||
|
||||
const categoryBreakdown = {} as Record<string, number>;
|
||||
for (const childKey of childKeys) {
|
||||
const value = finiteMedian(data.map(snapshot => snapshot?.breakdowns?.[category]?.[childKey]));
|
||||
if (value != null) categoryBreakdown[childKey] = value;
|
||||
}
|
||||
|
||||
const collapsed = collapseHeapSnapshotBreakdown(categoryBreakdown, options.breakdownTopN);
|
||||
if (Object.keys(collapsed).length > 0) {
|
||||
breakdowns[category] = collapsed;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
categories,
|
||||
nodeCounts,
|
||||
...(Object.keys(breakdowns).length > 0 ? { breakdowns } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function getHeapSnapshotCategoryValue(report: HeapSnapshotReport, category: keyof typeof heapSnapshotCategory) {
|
||||
return report.summary.categories[category];
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ export type MemoryReport = {
|
||||
|
||||
const [baseDirArg, headDirArg, baseOutputArg, headOutputArg] = process.argv.slice(2);
|
||||
|
||||
const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = util.readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 6, 1);
|
||||
const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = util.readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', heapSnapshotUtil.defaultHeapSnapshotBreakdownTopN, 1);
|
||||
const HEAD_HEAP_SNAPSHOT_WORK_DIR = resolve('head-heap-snapshots');
|
||||
const HEAD_HEAP_SNAPSHOT_OUTPUT_PATH = resolve('head-heap-snapshot.heapsnapshot');
|
||||
|
||||
@@ -70,51 +70,6 @@ async function resetState(repoDir: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeHeapSnapshotBreakdowns(samples: MemoryReport['samples'], phase: typeof phases[number]) {
|
||||
const breakdowns = {} as Record<keyof typeof heapSnapshotUtil.heapSnapshotCategory, Record<string, number>>;
|
||||
|
||||
for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) {
|
||||
if (category === 'total') continue;
|
||||
|
||||
const childKeys = new Set<string>();
|
||||
for (const sample of samples) {
|
||||
for (const childKey of Object.keys(sample.phases[phase].heapSnapshot?.breakdowns?.[category] ?? {})) {
|
||||
childKeys.add(childKey);
|
||||
}
|
||||
}
|
||||
|
||||
const categoryBreakdown = {} as Record<string, number>;
|
||||
for (const childKey of childKeys) {
|
||||
const values = samples
|
||||
.map(sample => sample.phases[phase].heapSnapshot?.breakdowns?.[category]?.[childKey])
|
||||
.filter(value => Number.isFinite(value)) as number[];
|
||||
|
||||
if (values.length > 0) categoryBreakdown[childKey] = util.median(values);
|
||||
}
|
||||
|
||||
if (Object.keys(categoryBreakdown).length > 0) {
|
||||
breakdowns[category] = collapseHeapSnapshotBreakdown(categoryBreakdown);
|
||||
}
|
||||
}
|
||||
|
||||
return breakdowns;
|
||||
}
|
||||
|
||||
function collapseHeapSnapshotBreakdown(breakdown: Record<string, number>) {
|
||||
const entries = Object.entries(breakdown)
|
||||
.filter(([, value]) => value > 0)
|
||||
.toSorted((a, b) => b[1] - a[1]);
|
||||
|
||||
const topEntries = entries.slice(0, HEAP_SNAPSHOT_BREAKDOWN_TOP_N);
|
||||
const otherValue = entries
|
||||
.slice(HEAP_SNAPSHOT_BREAKDOWN_TOP_N)
|
||||
.reduce((sum, [, value]) => sum + value, 0);
|
||||
|
||||
const collapsed = Object.fromEntries(topEntries);
|
||||
if (otherValue > 0) collapsed.Other = otherValue;
|
||||
return collapsed;
|
||||
}
|
||||
|
||||
function summarizeSamples(samples: MemoryReport['samples']) {
|
||||
const summary = {} as MemoryReport['summary'];
|
||||
|
||||
@@ -135,33 +90,12 @@ function summarizeSamples(samples: MemoryReport['samples']) {
|
||||
summary[phase].memoryUsage[key] = util.median(values);
|
||||
}
|
||||
|
||||
const heapSnapshotCategoryValues = {} as Record<keyof typeof heapSnapshotUtil.heapSnapshotCategory, number>;
|
||||
for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) {
|
||||
const values = samples
|
||||
.map(sample => sample.phases[phase].heapSnapshot?.categories?.[category])
|
||||
.filter(value => Number.isFinite(value)) as number[];
|
||||
|
||||
if (values.length > 0) heapSnapshotCategoryValues[category] = util.median(values);
|
||||
}
|
||||
|
||||
const heapSnapshotNodeCountValues = {} as Record<keyof typeof heapSnapshotUtil.heapSnapshotCategory, number>;
|
||||
for (const category of Object.keys(heapSnapshotUtil.heapSnapshotCategory) as (keyof typeof heapSnapshotUtil.heapSnapshotCategory)[]) {
|
||||
const values = samples
|
||||
.map(sample => sample.phases[phase].heapSnapshot?.nodeCounts?.[category])
|
||||
.filter(value => Number.isFinite(value)) as number[];
|
||||
|
||||
if (values.length > 0) heapSnapshotNodeCountValues[category] = util.median(values);
|
||||
}
|
||||
|
||||
if (Object.keys(heapSnapshotCategoryValues).length > 0) {
|
||||
const heapSnapshotBreakdowns = summarizeHeapSnapshotBreakdowns(samples, phase);
|
||||
|
||||
summary[phase].heapSnapshot = {
|
||||
categories: heapSnapshotCategoryValues,
|
||||
nodeCounts: heapSnapshotNodeCountValues,
|
||||
...(Object.keys(heapSnapshotBreakdowns).length > 0 ? { breakdowns: heapSnapshotBreakdowns } : {}),
|
||||
};
|
||||
}
|
||||
const heapSnapshot = heapSnapshotUtil.summarizeHeapSnapshotDataSamples(
|
||||
samples,
|
||||
sample => sample.phases[phase].heapSnapshot,
|
||||
{ breakdownTopN: HEAP_SNAPSHOT_BREAKDOWN_TOP_N },
|
||||
);
|
||||
if (heapSnapshot != null) summary[phase].heapSnapshot = heapSnapshot;
|
||||
}
|
||||
|
||||
return summary;
|
||||
|
||||
@@ -7,8 +7,9 @@ import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from 'node:chil
|
||||
import { copyFile, mkdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
import * as util from './utility.mts';
|
||||
import { heapSnapshotCategory, type HeapSnapshotData } from './heap-snapshot-util.mts';
|
||||
import { BrowserMeasurement, Chrome, NetworkSummary, summarizeNetwork } from './chrome.mts';
|
||||
import * as heapSnapshotUtil from './heap-snapshot-util.mts';
|
||||
import { Chrome, summarizeNetwork } from './chrome.mts';
|
||||
import type { BrowserMeasurement, NetworkSummary } from './chrome.mts';
|
||||
|
||||
const [baseDirArg, headDirArg, baseOutputArg, headOutputArg, headHeapSnapshotOutputArg] = process.argv.slice(2);
|
||||
|
||||
@@ -17,7 +18,7 @@ const serverReadyTimeoutMs = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SERVE
|
||||
const scenarioTimeoutMs = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SCENARIO_TIMEOUT_MS', 90_000, 1);
|
||||
const settleMs = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SETTLE_MS', 1_000, 0);
|
||||
const sampleCount = util.readIntegerEnv('FRONTEND_BROWSER_METRICS_SAMPLE_COUNT', 5, 1);
|
||||
const heapSnapshotBreakdownTopN = util.readIntegerEnv('FRONTEND_BROWSER_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', 8, 1);
|
||||
const heapSnapshotBreakdownTopN = util.readIntegerEnv('FRONTEND_BROWSER_HEAP_SNAPSHOT_BREAKDOWN_TOP_N', heapSnapshotUtil.defaultHeapSnapshotBreakdownTopN, 1);
|
||||
const headHeapSnapshotWorkDir = resolve('frontend-browser-head-heap-snapshots');
|
||||
|
||||
type BrowserMeasurementSample = BrowserMeasurement & {
|
||||
@@ -170,33 +171,6 @@ async function runSignupAndPostScenario(chrome: Chrome) {
|
||||
await util.sleep(settleMs);
|
||||
}
|
||||
|
||||
function emptyHeapSnapshotData(): HeapSnapshotData {
|
||||
const categories = {} as HeapSnapshotData['categories'];
|
||||
const nodeCounts = {} as HeapSnapshotData['nodeCounts'];
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
categories[category] = 0;
|
||||
nodeCounts[category] = 0;
|
||||
}
|
||||
return {
|
||||
categories,
|
||||
nodeCounts,
|
||||
breakdowns: {} as HeapSnapshotData['breakdowns'],
|
||||
};
|
||||
}
|
||||
|
||||
function collapseBreakdown(entries: Map<string, number>) {
|
||||
const sorted = [...entries]
|
||||
.filter(([, value]) => value > 0)
|
||||
.toSorted((a, b) => b[1] - a[1]);
|
||||
const topEntries = sorted.slice(0, heapSnapshotBreakdownTopN);
|
||||
const otherValue = sorted
|
||||
.slice(heapSnapshotBreakdownTopN)
|
||||
.reduce((sum, [, value]) => sum + value, 0);
|
||||
const collapsed = Object.fromEntries(topEntries);
|
||||
if (otherValue > 0) collapsed.Other = otherValue;
|
||||
return collapsed;
|
||||
}
|
||||
|
||||
function finiteMedian(values: (number | null | undefined)[], defaultValue = 0) {
|
||||
const finiteValues = values.filter(value => Number.isFinite(value)) as number[];
|
||||
if (finiteValues.length === 0) return defaultValue;
|
||||
@@ -300,90 +274,13 @@ function summarizePerformanceSamples(samples: BrowserMeasurementSample[]): Brows
|
||||
};
|
||||
}
|
||||
|
||||
function categorizeHeapNode(type: string, name: string): keyof typeof heapSnapshotCategory {
|
||||
if (/^(ArrayBuffer|SharedArrayBuffer|DataView|(?:Big)?(?:Int|Uint|Float)(?:8|16|32|64)?(?:Clamped)?Array)$/u.test(name)) return 'typedArrays';
|
||||
if (type === 'code' || type === 'closure') return 'code';
|
||||
if (type === 'string' || type === 'concatenated string' || type === 'sliced string' || type === 'symbol') return 'strings';
|
||||
if (type === 'array') return 'jsArrays';
|
||||
if (type === 'hidden' || type === 'synthetic' || type === 'object shape') return 'systemObjects';
|
||||
if (type === 'native') return 'otherNonJsObjects';
|
||||
if (type === 'object' || type === 'regexp' || type === 'number' || type === 'bigint') return 'otherJsObjects';
|
||||
return 'otherNonJsObjects';
|
||||
}
|
||||
|
||||
function summarizeHeapSnapshot(snapshot: any): HeapSnapshotData {
|
||||
const result = emptyHeapSnapshotData();
|
||||
const nodeFields = snapshot.snapshot.meta.node_fields as string[];
|
||||
const nodeTypes = snapshot.snapshot.meta.node_types as unknown[][];
|
||||
const nodes = snapshot.nodes as number[];
|
||||
const strings = snapshot.strings as string[];
|
||||
const fieldCount = nodeFields.length;
|
||||
const typeOffset = nodeFields.indexOf('type');
|
||||
const nameOffset = nodeFields.indexOf('name');
|
||||
const selfSizeOffset = nodeFields.indexOf('self_size');
|
||||
const typeNames = nodeTypes[typeOffset] as string[];
|
||||
const breakdownMaps = {} as Record<keyof typeof heapSnapshotCategory, Map<string, number>>;
|
||||
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
if (category !== 'total') breakdownMaps[category] = new Map();
|
||||
}
|
||||
|
||||
for (let offset = 0; offset < nodes.length; offset += fieldCount) {
|
||||
const type = typeNames[nodes[offset + typeOffset]] ?? 'unknown';
|
||||
const name = strings[nodes[offset + nameOffset]] ?? '';
|
||||
const selfSize = nodes[offset + selfSizeOffset] ?? 0;
|
||||
const category = categorizeHeapNode(type, name);
|
||||
|
||||
result.categories.total += selfSize;
|
||||
result.nodeCounts.total += 1;
|
||||
result.categories[category] += selfSize;
|
||||
result.nodeCounts[category] += 1;
|
||||
|
||||
const label = `${type}: ${name || '(anonymous)'}`;
|
||||
breakdownMaps[category].set(label, (breakdownMaps[category].get(label) ?? 0) + selfSize);
|
||||
}
|
||||
|
||||
result.breakdowns = {} as HeapSnapshotData['breakdowns'];
|
||||
for (const [category, entries] of Object.entries(breakdownMaps) as [keyof typeof heapSnapshotCategory, Map<string, number>][]) {
|
||||
const collapsed = collapseBreakdown(entries);
|
||||
if (Object.keys(collapsed).length > 0) {
|
||||
result.breakdowns[category] = collapsed;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function summarizeHeapSnapshotSamples(samples: BrowserMeasurementSample[]) {
|
||||
const summary = emptyHeapSnapshotData();
|
||||
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
summary.categories[category] = finiteMedian(samples.map(sample => sample.heapSnapshot.categories[category]));
|
||||
summary.nodeCounts[category] = finiteMedian(samples.map(sample => sample.heapSnapshot.nodeCounts[category]));
|
||||
}
|
||||
|
||||
summary.breakdowns = {} as HeapSnapshotData['breakdowns'];
|
||||
for (const category of Object.keys(heapSnapshotCategory) as (keyof typeof heapSnapshotCategory)[]) {
|
||||
if (category === 'total') continue;
|
||||
|
||||
const childKeys = new Set<string>();
|
||||
for (const sample of samples) {
|
||||
for (const childKey of Object.keys(sample.heapSnapshot.breakdowns?.[category] ?? {})) {
|
||||
childKeys.add(childKey);
|
||||
}
|
||||
}
|
||||
|
||||
const childValues = new Map<string, number>();
|
||||
for (const childKey of childKeys) {
|
||||
childValues.set(childKey, finiteMedian(samples.map(sample => sample.heapSnapshot.breakdowns?.[category]?.[childKey])));
|
||||
}
|
||||
|
||||
const collapsed = collapseBreakdown(childValues);
|
||||
if (Object.keys(collapsed).length > 0) {
|
||||
summary.breakdowns[category] = collapsed;
|
||||
}
|
||||
}
|
||||
|
||||
const summary = heapSnapshotUtil.summarizeHeapSnapshotDataSamples(
|
||||
samples,
|
||||
sample => sample.heapSnapshot,
|
||||
{ breakdownTopN: heapSnapshotBreakdownTopN },
|
||||
);
|
||||
if (summary == null) throw new Error('No heap snapshot samples');
|
||||
return summary;
|
||||
}
|
||||
|
||||
@@ -416,9 +313,7 @@ function summarizeSamples(label: 'base' | 'head', samples: BrowserMeasurementSam
|
||||
async function measureSample(label: 'base' | 'head', round: number, heapSnapshotSavePath?: string) {
|
||||
await prepareInstance();
|
||||
|
||||
const chrome = await Chrome.create(label, { scenarioTimeoutMs });
|
||||
|
||||
try {
|
||||
return await Chrome.with(label, { scenarioTimeoutMs }, async chrome => {
|
||||
await chrome.enableNetworkTracking();
|
||||
|
||||
const startedAt = Date.now();
|
||||
@@ -426,7 +321,7 @@ async function measureSample(label: 'base' | 'head', round: number, heapSnapshot
|
||||
const durationMs = Date.now() - startedAt;
|
||||
const performance = await chrome.collectPerformance();
|
||||
const heapSnapshotRaw = await chrome.takeHeapSnapshot(heapSnapshotSavePath);
|
||||
const heapSnapshot = summarizeHeapSnapshot(heapSnapshotRaw);
|
||||
const heapSnapshot = heapSnapshotUtil.analyzeHeapSnapshot(heapSnapshotRaw, { breakdownTopN: heapSnapshotBreakdownTopN });
|
||||
const measurement: BrowserMeasurementSample = {
|
||||
label,
|
||||
round,
|
||||
@@ -440,9 +335,7 @@ async function measureSample(label: 'base' | 'head', round: number, heapSnapshot
|
||||
};
|
||||
|
||||
return measurement;
|
||||
} finally {
|
||||
await chrome.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function headHeapSnapshotPath(round: number) {
|
||||
|
||||
Reference in New Issue
Block a user