diff --git a/.github/scripts/backend-memory-report.mts b/.github/scripts/backend-memory-report.mts index 7d2ab1677c..a758beb979 100644 --- a/.github/scripts/backend-memory-report.mts +++ b/.github/scripts/backend-memory-report.mts @@ -5,6 +5,7 @@ import { readFile, writeFile } from 'node:fs/promises'; import * as util from './utility.mts'; +import * as heapSnapshotUtil from './heap-snapshot-util.mts'; import type { MemoryReport } from './measure-backend-memory-comparison.mts'; const [baseFile, headFile, outputFile, baseJsFootprintFile, headJsFootprintFile] = process.argv.slice(2); @@ -51,28 +52,6 @@ const metrics = [ 'External', ] as const; -const heapSnapshotCategoriesColors = { - 'Total': 'gray', - 'Code': 'orange', - 'Strings': 'red', - 'JS arrays': 'cyan', - 'Typed arrays': 'green', - 'System objects': 'yellow', - 'Other JS objects': 'violet', - 'Other non-JS objects': 'pink', -} as const; - -const heapSnapshotCategoriesColorsHex = { - 'Total': '#888888', - 'Code': '#f28e2c', - 'Strings': '#e15759', - 'JS arrays': '#76b7b2', - 'Typed arrays': '#59a14f', - 'System objects': '#edc949', - 'Other JS objects': '#af7aa1', - 'Other non-JS objects': '#ff9da7', -} as const; - function formatMemoryMb(valueKiB: number | null | undefined) { if (valueKiB == null) return '-'; return `${util.formatNumber(valueKiB / 1024)} MB`; @@ -94,43 +73,6 @@ function getSampleSpread(report: MemoryReport, phase: typeof memoryReportPhases[ return util.median(values.map(value => Math.abs(value - center))); } -function getSamplesByRound(report: MemoryReport) { - const samplesByRound = new Map(); - if (!Array.isArray(report.samples)) return samplesByRound; - - for (const sample of report.samples) { - if (sample.round <= 0) continue; - samplesByRound.set(sample.round, sample); - } - - return samplesByRound; -} - -function pairedDeltaSummary(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], metric: typeof metrics[number]) { - const baseSamplesByRound = getSamplesByRound(base); - const headSamplesByRound = getSamplesByRound(head); - const values = []; - - for (const [round, baseSample] of baseSamplesByRound) { - const headSample = headSamplesByRound.get(round); - if (headSample == null) continue; - - const baseValue = getMemoryValueFromSample(baseSample, phase, metric); - const headValue = getMemoryValueFromSample(headSample, phase, metric); - if (baseValue == null || headValue == null) continue; - - values.push(headValue - baseValue); - } - - return { - median: util.median(values), - mad: util.mad(values), - min: Math.min(...values), - max: Math.max(...values), - samples: values.length, - }; -} - function renderMainTableForPhase(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key']) { const lines = [ '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', @@ -147,7 +89,7 @@ function renderMainTableForPhase(base: MemoryReport, head: MemoryReport, phase: const baseSpread = getSampleSpread(base, phase, metric); const headSpread = getSampleSpread(head, phase, metric); - const summary = pairedDeltaSummary(base, head, phase, metric); + const summary = util.pairedDeltaSummary(base.samples, head.samples, (sample) => getMemoryValueFromSample(sample, phase, metric)); const percent = summary.median * 100 / baseValue; const deltaMedian = summary == null ? '-' : `${formatDeltaMemory(summary.median)}
${util.formatDeltaPercent(percent).replaceAll('\\%', '\\\\%')}`; @@ -174,206 +116,24 @@ function measurementSummary(base, head) { } */ -function getHeapSnapshotCategoryValue(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) { - const value = report.summary[phase]?.heapSnapshot?.categories?.[category]; - return Number.isFinite(value) ? value : null; -} - -function getHeapSnapshotCategoryValueFromSample(sample: MemoryReport['samples'][number], phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) { - const value = sample.phases[phase]?.heapSnapshot?.categories?.[category]; - return Number.isFinite(value) ? value : null; -} - -const heapSnapshotSankeyChildMinRatio = 0.3; -const heapSnapshotSankeyParentMinPercent = 10; - -function escapeCsvValue(value: string) { - return `"${String(value).replaceAll('"', '""')}"`; -} - -function renderHeapSnapshotSankey(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], title: string) { - const total = getHeapSnapshotCategoryValue(report, phase, 'Total'); - if (total == null || total <= 0) return null; - - function getHeapSnapshotBreakdownEntries(category: typeof util.heapSnapshotCategories[number]) { - const breakdown = report.summary[phase].heapSnapshot?.breakdowns?.[category]; - if (breakdown == null || typeof breakdown !== 'object') return []; - - return Object.entries(breakdown) - .filter(([, value]) => Number.isFinite(value) && value > 0) - .toSorted((a, b) => b[1] - a[1]); - } - - function formatHeapSnapshotSankeyChildLabel(label: string) { - return String(label).replace(/^[^:]+:\s*/, ''); - } - - function formatSankeyPercentValue(value: number) { - const rounded = Math.round(value * 100) / 100; - if (rounded === 0 && value > 0) return '0.01'; - if (Number.isInteger(rounded)) return String(rounded); - return rounded.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); - } - - const categories = util.heapSnapshotCategories - .filter(category => category !== 'Total') - .map(category => { - const value = getHeapSnapshotCategoryValue(report, phase, category); - if (value == null || value <= 0) return null; - const breakdownEntries = getHeapSnapshotBreakdownEntries(category); - const breakdownTotal = breakdownEntries.reduce((sum, [, childValue]) => sum + childValue, 0); - const percent = (value * 100) / total; - const childEntries = []; - let otherPercent = 0; - - if (breakdownTotal > 0 && percent > heapSnapshotSankeyParentMinPercent) { - for (const [childName, childValue] of breakdownEntries) { - const childRatio = childValue / breakdownTotal; - const childPercent = percent * childRatio; - if (childRatio >= heapSnapshotSankeyChildMinRatio) { - childEntries.push([formatHeapSnapshotSankeyChildLabel(childName), childPercent]); - } else { - otherPercent += childPercent; - } - } - - if (childEntries.length > 0 && otherPercent > 0) { - childEntries.push(['Other', otherPercent]); - } - } - - return { - category, - percent, - childEntries, - }; - }) - .filter(value => value != null); - - if (categories.length === 0) return null; - - const nodeColors = { - [title]: heapSnapshotCategoriesColorsHex.Total, - } as Record; - for (const { category, childEntries } of categories) { - const categoryColor = heapSnapshotCategoriesColorsHex[category] ?? heapSnapshotCategoriesColorsHex.Total; - nodeColors[category] = categoryColor; - - for (const [childName] of childEntries) { - nodeColors[childName] = categoryColor; - } - } - - const lines = [ - `
${title} heap snapshot composition`, - '', - '```mermaid', - `%%{init: ${JSON.stringify({ - sankey: { - showValues: false, - linkColor: 'target', - labelStyle: 'outlined', - nodeAlignment: 'center', - nodePadding: 10, - nodeColors: { - ...nodeColors, - 'Other': '#888888', - }, - }, - })}}%%`, - 'sankey-beta', - ]; - - for (const { category, percent, childEntries } of categories) { - lines.push(`${escapeCsvValue(title)},${escapeCsvValue(category)},${formatSankeyPercentValue(percent)}`); - - for (const [childName, childPercent] of childEntries) { - lines.push(`${escapeCsvValue(category)},${escapeCsvValue(childName)},${formatSankeyPercentValue(childPercent)}`); - } - } - - lines.push('```'); - lines.push(''); - lines.push('
'); - - return lines.join('\n'); -} - -function pairedHeapSnapshotDeltaSummary(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) { - const baseSamplesByRound = getSamplesByRound(base); - const headSamplesByRound = getSamplesByRound(head); - const values = [] as number[]; - - for (const [round, baseSample] of baseSamplesByRound) { - const headSample = headSamplesByRound.get(round); - if (headSample == null) continue; - - const baseValue = getHeapSnapshotCategoryValueFromSample(baseSample, phase, category); - const headValue = getHeapSnapshotCategoryValueFromSample(headSample, phase, category); - if (baseValue == null || headValue == null) continue; - - values.push(headValue - baseValue); - } - - return { - median: util.median(values), - mad: util.mad(values), - min: Math.min(...values), - max: Math.max(...values), - samples: values.length, - }; -} - -function renderHeapSnapshotTable(base: MemoryReport, head: MemoryReport, phase: typeof memoryReportPhases[number]['key']) { - const lines = [ - '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', - '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', - ]; - const baseTotal = getHeapSnapshotCategoryValue(base, phase, 'Total'); - const headTotal = getHeapSnapshotCategoryValue(head, phase, 'Total'); - - function formatHeapSnapshotCategoryLabel(category: typeof heapSnapshotCategories[number], baseValue: number, headValue: number, baseTotal: number, headTotal: number) { - if (category === 'Total' || baseTotal == null || headTotal == null || baseTotal <= 0 || headTotal <= 0) return `**${category}**`; - - const basePercent = util.formatPercent((baseValue * 100) / baseTotal); - const headPercent = util.formatPercent((headValue * 100) / headTotal); - return `**${category}**
${basePercent} → ${headPercent}`; - } - - function getHeapSnapshotSampleSpread(report: MemoryReport, phase: typeof memoryReportPhases[number]['key'], category: typeof util.heapSnapshotCategories[number]) { - const values = report.samples - .map(sample => getHeapSnapshotCategoryValueFromSample(sample, phase, category)) - .filter(value => Number.isFinite(value)) as number[]; - if (values.length < 2) return null; - - const center = util.median(values); - return util.median(values.map(value => Math.abs(value - center))); - } - - for (const category of util.heapSnapshotCategories) { - const baseValue = getHeapSnapshotCategoryValue(base, phase, category); - const headValue = getHeapSnapshotCategoryValue(head, phase, category); - if (baseValue == null || headValue == null) continue; - - const baseSpread = getHeapSnapshotSampleSpread(base, phase, category); - const headSpread = getHeapSnapshotSampleSpread(head, phase, category); - const summary = pairedHeapSnapshotDeltaSummary(base, head, phase, category); - const percent = summary.median * 100 / baseValue; - const deltaMedian = summary == null ? '-' : `${util.formatDeltaBytes(summary.median)}
${util.formatDeltaPercent(percent).replaceAll('\\%', '\\\\%')}`; - const categoryLabel = formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal); - - lines.push(`| $\\color{${heapSnapshotCategoriesColors[category]}}{\\rule{8pt}{8pt}}$ ${categoryLabel} | ${util.formatBytes(baseValue)}
± ${baseSpread == null ? '-' : util.formatBytes(baseSpread)} | ${util.formatBytes(headValue)}
± ${headSpread == null ? '-' : util.formatBytes(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : util.formatBytes(summary.mad)} | ${summary == null ? '-' : util.formatDeltaBytes(summary.min)} | ${summary == null ? '-' : util.formatDeltaBytes(summary.max)} |`); - if (category === 'Total') { - lines.push('| | | | | | | |'); - } - } - - if (lines.length === 2) return null; - return lines.join('\n'); -} - function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) { - const table = renderHeapSnapshotTable(base, head, 'afterGc'); + const baseHeapSnapshotReport = { + summary: base.summary.afterGc.heapSnapshot!, + samples: base.samples.map(sample => ({ + round: sample.round, + data: sample.phases.afterGc.heapSnapshot!, + })), + }; + + const headHeapSnapshotReport = { + summary: head.summary.afterGc.heapSnapshot!, + samples: head.samples.map(sample => ({ + round: sample.round, + data: sample.phases.afterGc.heapSnapshot!, + })), + }; + + const table = heapSnapshotUtil.renderHeapSnapshotTable(baseHeapSnapshotReport, headHeapSnapshotReport); if (table == null) return null; const lines = [ @@ -384,8 +144,8 @@ function renderHeapSnapshotSection(base: MemoryReport, head: MemoryReport) { ]; for (const graph of [ - renderHeapSnapshotSankey(base, 'afterGc', 'Base'), - renderHeapSnapshotSankey(head, 'afterGc', 'Head'), + heapSnapshotUtil.renderHeapSnapshotSankey(baseHeapSnapshotReport, 'Base'), + heapSnapshotUtil.renderHeapSnapshotSankey(headHeapSnapshotReport, 'Head'), ]) { if (graph == null) continue; lines.push(graph); diff --git a/.github/scripts/heap-snapshot-util.mts b/.github/scripts/heap-snapshot-util.mts new file mode 100644 index 0000000000..d4555348a5 --- /dev/null +++ b/.github/scripts/heap-snapshot-util.mts @@ -0,0 +1,228 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +// NOTE: このファイルはworkflow上でバックエンドからも参照されるため、side effectがあってはならない + +import * as util from './utility.mts'; + +export const heapSnapshotCategories = [ + 'Total', + 'Code', + 'Strings', + 'JS arrays', + 'Typed arrays', + 'System objects', + 'Other JS objects', + 'Other non-JS objects', +] as const; + +const heapSnapshotCategoriesColors = { + 'Total': 'gray', + 'Code': 'orange', + 'Strings': 'red', + 'JS arrays': 'cyan', + 'Typed arrays': 'green', + 'System objects': 'yellow', + 'Other JS objects': 'violet', + 'Other non-JS objects': 'pink', +} as const satisfies Record; + +const heapSnapshotCategoriesColorsHex = { + 'Total': '#888888', + 'Code': '#f28e2c', + 'Strings': '#e15759', + 'JS arrays': '#76b7b2', + 'Typed arrays': '#59a14f', + 'System objects': '#edc949', + 'Other JS objects': '#af7aa1', + 'Other non-JS objects': '#ff9da7', +} as const satisfies Record; + +export type HeapSnapshotData = { + categories: Record; + nodeCounts: Record; + breakdowns?: Record>; +}; + +export type HeapSnapshotReport = { + summary: HeapSnapshotData; + samples: { + round: number; + data: HeapSnapshotData; + }[]; +}; + +function getHeapSnapshotCategoryValue(report: HeapSnapshotReport, category: typeof heapSnapshotCategories[number]) { + const value = report.summary.categories[category]; + return Number.isFinite(value) ? value : null; +} + +function getHeapSnapshotCategoryValueFromSample(sample: HeapSnapshotReport['samples'][number], category: typeof heapSnapshotCategories[number]) { + const value = sample.data.categories[category]; + return Number.isFinite(value) ? value : null; +} + +export function renderHeapSnapshotTable(base: HeapSnapshotReport, head: HeapSnapshotReport) { + const lines = [ + '| Metric | Base | Head | Δ median | Δ MAD | Δ min | Δ max |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + const baseTotal = getHeapSnapshotCategoryValue(base, 'Total'); + const headTotal = getHeapSnapshotCategoryValue(head, 'Total'); + + function formatHeapSnapshotCategoryLabel(category: typeof heapSnapshotCategories[number], baseValue: number, headValue: number, baseTotal: number, headTotal: number) { + if (category === 'Total' || baseTotal == null || headTotal == null || baseTotal <= 0 || headTotal <= 0) return `**${category}**`; + + const basePercent = util.formatPercent((baseValue * 100) / baseTotal); + const headPercent = util.formatPercent((headValue * 100) / headTotal); + return `**${category}**
${basePercent} → ${headPercent}`; + } + + function getHeapSnapshotSampleSpread(report: HeapSnapshotReport, category: typeof heapSnapshotCategories[number]) { + const values = report.samples + .map(sample => getHeapSnapshotCategoryValueFromSample(sample, category)) + .filter(value => Number.isFinite(value)) as number[]; + if (values.length < 2) return null; + + const center = util.median(values); + return util.median(values.map(value => Math.abs(value - center))); + } + + for (const category of heapSnapshotCategories) { + const baseValue = getHeapSnapshotCategoryValue(base, category); + const headValue = getHeapSnapshotCategoryValue(head, category); + if (baseValue == null || headValue == null) continue; + + const baseSpread = getHeapSnapshotSampleSpread(base, category); + const headSpread = getHeapSnapshotSampleSpread(head, category); + const summary = util.pairedDeltaSummary(base.samples, head.samples, (sample) => getHeapSnapshotCategoryValueFromSample(sample, category)); + const percent = summary.median * 100 / baseValue; + const deltaMedian = summary == null ? '-' : `${util.formatDeltaBytes(summary.median)}
${util.formatDeltaPercent(percent).replaceAll('\\%', '\\\\%')}`; + const categoryLabel = formatHeapSnapshotCategoryLabel(category, baseValue, headValue, baseTotal, headTotal); + + lines.push(`| $\\color{${heapSnapshotCategoriesColors[category]}}{\\rule{8pt}{8pt}}$ ${categoryLabel} | ${util.formatBytes(baseValue)}
± ${baseSpread == null ? '-' : util.formatBytes(baseSpread)} | ${util.formatBytes(headValue)}
± ${headSpread == null ? '-' : util.formatBytes(headSpread)} | ${deltaMedian} | ${summary?.mad == null ? '-' : util.formatBytes(summary.mad)} | ${summary == null ? '-' : util.formatDeltaBytes(summary.min)} | ${summary == null ? '-' : util.formatDeltaBytes(summary.max)} |`); + if (category === 'Total') { + lines.push('| | | | | | | |'); + } + } + + if (lines.length === 2) return null; + return lines.join('\n'); +} + +const heapSnapshotSankeyChildMinRatio = 0.3; +const heapSnapshotSankeyParentMinPercent = 10; + +function escapeCsvValue(value: string) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +export function renderHeapSnapshotSankey(report: HeapSnapshotReport, title: string) { + const total = getHeapSnapshotCategoryValue(report, 'Total'); + if (total == null || total <= 0) return null; + + function getHeapSnapshotBreakdownEntries(category: typeof heapSnapshotCategories[number]) { + const breakdown = report.summary.breakdowns?.[category]; + if (breakdown == null || typeof breakdown !== 'object') return []; + + return Object.entries(breakdown) + .filter(([, value]) => Number.isFinite(value) && value > 0) + .toSorted((a, b) => b[1] - a[1]); + } + + function formatHeapSnapshotSankeyChildLabel(label: string) { + return String(label).replace(/^[^:]+:\s*/, ''); + } + + function formatSankeyPercentValue(value: number) { + const rounded = Math.round(value * 100) / 100; + if (rounded === 0 && value > 0) return '0.01'; + if (Number.isInteger(rounded)) return String(rounded); + return rounded.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); + } + + const categories = heapSnapshotCategories + .filter(category => category !== 'Total') + .map(category => { + const value = getHeapSnapshotCategoryValue(report, category); + if (value == null || value <= 0) return null; + const breakdownEntries = getHeapSnapshotBreakdownEntries(category); + const breakdownTotal = breakdownEntries.reduce((sum, [, childValue]) => sum + childValue, 0); + const percent = (value * 100) / total; + const childEntries = []; + let otherPercent = 0; + + if (breakdownTotal > 0 && percent > heapSnapshotSankeyParentMinPercent) { + for (const [childName, childValue] of breakdownEntries) { + const childRatio = childValue / breakdownTotal; + const childPercent = percent * childRatio; + if (childRatio >= heapSnapshotSankeyChildMinRatio) { + childEntries.push([formatHeapSnapshotSankeyChildLabel(childName), childPercent]); + } else { + otherPercent += childPercent; + } + } + + if (childEntries.length > 0 && otherPercent > 0) { + childEntries.push(['Other', otherPercent]); + } + } + + return { + category, + percent, + childEntries, + }; + }) + .filter(value => value != null); + + if (categories.length === 0) return null; + + const nodeColors = { + [title]: heapSnapshotCategoriesColorsHex.Total, + } as Record; + for (const { category, childEntries } of categories) { + const categoryColor = heapSnapshotCategoriesColorsHex[category] ?? heapSnapshotCategoriesColorsHex.Total; + nodeColors[category] = categoryColor; + + for (const [childName] of childEntries) { + nodeColors[childName] = categoryColor; + } + } + + const lines = [ + `
${title} heap snapshot composition`, + '', + '```mermaid', + `%%{init: ${JSON.stringify({ + sankey: { + showValues: false, + linkColor: 'target', + labelStyle: 'outlined', + nodeAlignment: 'center', + nodePadding: 10, + nodeColors: { + ...nodeColors, + 'Other': '#888888', + }, + }, + })}}%%`, + 'sankey-beta', + ]; + + for (const { category, percent, childEntries } of categories) { + lines.push(`${escapeCsvValue(title)},${escapeCsvValue(category)},${formatSankeyPercentValue(percent)}`); + + for (const [childName, childPercent] of childEntries) { + lines.push(`${escapeCsvValue(category)},${escapeCsvValue(childName)},${formatSankeyPercentValue(childPercent)}`); + } + } + + lines.push('```'); + lines.push(''); + lines.push('
'); + + return lines.join('\n'); +} diff --git a/.github/scripts/measure-backend-memory-comparison.mts b/.github/scripts/measure-backend-memory-comparison.mts index be0b7ad75d..23f02c717e 100644 --- a/.github/scripts/measure-backend-memory-comparison.mts +++ b/.github/scripts/measure-backend-memory-comparison.mts @@ -7,6 +7,7 @@ import { createRequire } from 'node:module'; import { writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import * as util from './utility.mts'; +import * as heapSnapshotUtil from './heap-snapshot-util.mts'; import type { MemoryReportRaw } from '../../packages/backend/scripts/measure-memory.mts'; const phases = ['afterGc'] as const; @@ -28,11 +29,7 @@ export type MemoryReport = { }; summary: Record; - heapSnapshot?: { - categories: Record; - nodeCounts: Record; - breakdowns?: Record>; - }; + heapSnapshot?: heapSnapshotUtil.HeapSnapshotData; }>; samples: (MemoryReportRaw['samples'][number] & { round: number; @@ -72,9 +69,9 @@ async function resetState(repoDir: string) { } function summarizeHeapSnapshotBreakdowns(samples: MemoryReport['samples'], phase: typeof phases[number]) { - const breakdowns = {} as Record>; + const breakdowns = {} as Record>; - for (const category of util.heapSnapshotCategories) { + for (const category of heapSnapshotUtil.heapSnapshotCategories) { if (category === 'Total') continue; const childKeys = new Set(); @@ -88,7 +85,7 @@ function summarizeHeapSnapshotBreakdowns(samples: MemoryReport['samples'], phase for (const childKey of childKeys) { const values = samples .map(sample => sample.phases[phase].heapSnapshot?.breakdowns?.[category]?.[childKey]) - .filter(value => Number.isFinite(value)); + .filter(value => Number.isFinite(value)) as number[]; if (values.length > 0) categoryBreakdown[childKey] = util.median(values); } @@ -136,8 +133,8 @@ function summarizeSamples(samples: MemoryReport['samples']) { summary[phase].memoryUsage[key] = util.median(values); } - const heapSnapshotCategoryValues = {} as Record; - for (const category of util.heapSnapshotCategories) { + const heapSnapshotCategoryValues = {} as Record; + for (const category of heapSnapshotUtil.heapSnapshotCategories) { const values = samples .map(sample => sample.phases[phase].heapSnapshot?.categories?.[category]) .filter(value => Number.isFinite(value)) as number[]; @@ -145,8 +142,8 @@ function summarizeSamples(samples: MemoryReport['samples']) { if (values.length > 0) heapSnapshotCategoryValues[category] = util.median(values); } - const heapSnapshotNodeCountValues = {} as Record; - for (const category of util.heapSnapshotCategories) { + const heapSnapshotNodeCountValues = {} as Record; + for (const category of heapSnapshotUtil.heapSnapshotCategories) { const values = samples .map(sample => sample.phases[phase].heapSnapshot?.nodeCounts?.[category]) .filter(value => Number.isFinite(value)) as number[]; diff --git a/.github/scripts/utility.mts b/.github/scripts/utility.mts index a0c2f35e1c..a65da25a6d 100644 --- a/.github/scripts/utility.mts +++ b/.github/scripts/utility.mts @@ -3,21 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +// NOTE: このファイルはworkflow上でバックエンドからも参照されるため、side effectがあってはならない + import { spawn } from 'node:child_process'; import { promises as fs } from 'node:fs'; import path from 'node:path'; -export const heapSnapshotCategories = [ - 'Total', - 'Code', - 'Strings', - 'JS arrays', - 'Typed arrays', - 'System objects', - 'Other JS objects', - 'Other non-JS objects', -] as const; - export function median(values: number[]) { const sorted = values.toSorted((a, b) => a - b); const center = Math.floor(sorted.length / 2); @@ -32,6 +23,40 @@ export function mad(values: number[]) { return median(values.map(value => Math.abs(value - center))); } +function getSamplesByRound(samples: T) { + const samplesByRound = new Map(); + for (const sample of samples) { + if (sample.round <= 0) continue; + samplesByRound.set(sample.round, sample); + } + return samplesByRound; +} + +export function pairedDeltaSummary(baseSamples: T, headSamples: T, getValue: (sample: T[number]) => number | null) { + const baseSamplesByRound = getSamplesByRound(baseSamples); + const headSamplesByRound = getSamplesByRound(headSamples); + const values = []; + + for (const [round, baseSample] of baseSamplesByRound) { + const headSample = headSamplesByRound.get(round); + if (headSample == null) continue; + + const baseValue = getValue(baseSample); + const headValue = getValue(headSample); + if (baseValue == null || headValue == null) continue; + + values.push(headValue - baseValue); + } + + return { + median: median(values), + mad: mad(values), + min: Math.min(...values), + max: Math.max(...values), + samples: values.length, + }; +} + export function normalizePath(filePath: string) { return filePath.split(path.sep).join('/'); } diff --git a/packages/backend/scripts/measure-memory.mts b/packages/backend/scripts/measure-memory.mts index f814ac27ac..51abc20ece 100644 --- a/packages/backend/scripts/measure-memory.mts +++ b/packages/backend/scripts/measure-memory.mts @@ -10,6 +10,7 @@ import { dirname, join } from 'node:path'; import { tmpdir } from 'node:os'; //import * as http from 'node:http'; import * as fs from 'node:fs/promises'; +import { heapSnapshotCategories, type HeapSnapshotData } from '../../../.github/scripts/heap-snapshot-util.mts'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -44,17 +45,6 @@ const HEAP_SNAPSHOT_BREAKDOWN_TOP_N = readIntegerEnv('MK_MEMORY_HEAP_SNAPSHOT_BR const procStatusKeys = ['VmPeak', 'VmSize', 'VmHWM', 'VmRSS', 'VmData', 'VmStk', 'VmExe', 'VmLib', 'VmPTE', 'VmSwap'] as const; const smapsRollupKeys = ['Pss', 'Shared_Clean', 'Shared_Dirty', 'Private_Clean', 'Private_Dirty', 'Swap', 'SwapPss'] as const; -const heapSnapshotCategories = [ - 'Code', - 'Strings', - 'JS arrays', - 'Typed arrays', - 'System objects', - 'Other JS objects', - 'Other non-JS objects', - 'Total', -]; - const typedArrayNames = new Set([ 'ArrayBuffer', 'SharedArrayBuffer', @@ -101,10 +91,6 @@ function bytesToKiB(value: number) { return Math.round(value / 1024); } -function createEmptyHeapSnapshotCategoryMap() { - return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0])); -} - function isTypedArrayNode(type, name) { return typedArrayNames.has(name) || (type === 'native' && (name.includes('ArrayBuffer') || name.includes('TypedArray'))); @@ -181,8 +167,8 @@ function classifyHeapSnapshotBreakdown(category, type, name) { return sanitizeHeapSnapshotBreakdownLabel(`${type}: ${name}`, type); } -function collapseHeapSnapshotBreakdown(breakdowns) { - const collapsed = {}; +function collapseHeapSnapshotBreakdown(breakdowns: Record>) { + const collapsed = {} as Record>; for (const [category, children] of Object.entries(breakdowns)) { const entries = Object.entries(children) @@ -223,6 +209,10 @@ function analyzeHeapSnapshot(snapshot) { const nodeTypeNames = meta.node_types?.[typeOffset]; if (!Array.isArray(nodeTypeNames)) throw new Error('Invalid heap snapshot node types'); + function createEmptyHeapSnapshotCategoryMap() { + return Object.fromEntries(heapSnapshotCategories.map(category => [category, 0])) as Record; + } + const fieldCount = nodeFields.length; const categories = createEmptyHeapSnapshotCategoryMap(); const nodeCounts = createEmptyHeapSnapshotCategoryMap(); @@ -306,7 +296,7 @@ async function getRuntimeMemoryUsage(serverProcess: ChildProcess) { }; } -async function getHeapSnapshotStatistics(serverProcess: ChildProcess) { +async function getHeapSnapshotStatistics(serverProcess: ChildProcess): Promise { if (!HEAP_SNAPSHOT) return null; const snapshotPath = join(tmpdir(), `misskey-backend-heap-${process.pid}-${serverProcess.pid}-${Date.now()}.heapsnapshot`);