mirrors/misskey
1
0
Fork 1
mirror of https://github.com/misskey-dev/misskey.git synced 2026-07-25 17:55:06 +02:00
Code Releases Activity

enhance(dev): improve backend memory usage comparison workflow (#17591)

Browse Source 
* wip

* Update get-backend-memory.yml

* [ci skip] tweak table
This commit is contained in:
syuilo
2026-06-23 11:01:27 +09:00
committed by GitHub
parent d54b948085
commit 6d11f572b3
5 changed files with 304 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
.github/workflows/get-backend-memory.yml vendored
View File

@@ -10,6 +10,7 @@ on:
- packages/backend/**
- packages/misskey-js/**
- .github/workflows/get-backend-memory.yml
- .github/workflows/report-backend-memory.yml
jobs:
get-memory-usage:
@@ -17,15 +18,6 @@ jobs:
permissions:
contents: read
strategy:
matrix:
memory-json-name: [memory-base.json, memory-head.json]
include:
- memory-json-name: memory-base.json
ref: ${{ github.base_ref }}
- memory-json-name: memory-head.json
ref: refs/pull/${{ github.event.number }}/merge
services:
postgres:
image: postgres:18
@@ -40,37 +32,121 @@ jobs:
- 56312:6379
steps:
- uses: actions/checkout@v6.0.2
- name: Checkout base
uses: actions/checkout@v6.0.2
with:
ref: ${{ matrix.ref }}
ref: ${{ github.base_ref }}
path: base
submodules: true
- name: Checkout head
uses: actions/checkout@v6.0.2
with:
ref: refs/pull/${{ github.event.number }}/merge
path: head
submodules: true
- name: Setup pnpm
uses: pnpm/action-setup@v6.0.3
with:
package_json_file: head/package.json
- name: Use Node.js
uses: actions/setup-node@v6.4.0
with:
node-version-file: '.node-version'
node-version-file: 'head/.node-version'
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- name: Check pnpm-lock.yaml
cache-dependency-path: |
base/pnpm-lock.yaml
head/pnpm-lock.yaml
- name: Install base dependencies
working-directory: base
run: pnpm i --frozen-lockfile
- name: Check base pnpm-lock.yaml
working-directory: base
run: git diff --exit-code pnpm-lock.yaml
- name: Copy Configure
run: cp .github/misskey/test.yml .config/default.yml
- name: Compile Configure
run: pnpm compile-config
- name: Build
run: pnpm build
- name: Run migrations
run: pnpm --filter backend migrate
- name: Measure memory usage
- name: Configure base
working-directory: base
run: |
# Start the server and measure memory usage
node packages/backend/scripts/measure-memory.mjs > ${{ matrix.memory-json-name }}
cp .github/misskey/test.yml .config/default.yml
pnpm compile-config
- name: Build base
working-directory: base
run: pnpm build
- name: Install head dependencies
working-directory: head
run: pnpm i --frozen-lockfile
- name: Check head pnpm-lock.yaml
working-directory: head
run: git diff --exit-code pnpm-lock.yaml
- name: Configure head
working-directory: head
run: |
cp .github/misskey/test.yml .config/default.yml
pnpm compile-config
- name: Build head
working-directory: head
run: pnpm build
- name: Measure base memory usage
working-directory: base
run: |
node --input-type=module <<'EOF'
import { createRequire } from 'node:module';
import { join } from 'node:path';
const require = createRequire(join(process.cwd(), 'packages/backend/package.json'));
const pg = require('pg');
const Redis = require('ioredis');
const postgres = new pg.Client({
host: '127.0.0.1',
port: 54312,
database: 'postgres',
user: 'postgres',
});
await postgres.connect();
await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
await postgres.query('CREATE DATABASE "test-misskey"');
await postgres.end();
const redis = new Redis({ host: '127.0.0.1', port: 56312 });
await redis.flushall();
redis.disconnect();
EOF
pnpm --filter backend migrate
node packages/backend/scripts/measure-memory.mjs > ../memory-base.json
- name: Measure head memory usage
working-directory: head
run: |
node --input-type=module <<'EOF'
import { createRequire } from 'node:module';
import { join } from 'node:path';
const require = createRequire(join(process.cwd(), 'packages/backend/package.json'));
const pg = require('pg');
const Redis = require('ioredis');
const postgres = new pg.Client({
host: '127.0.0.1',
port: 54312,
database: 'postgres',
user: 'postgres',
});
await postgres.connect();
await postgres.query('DROP DATABASE IF EXISTS "test-misskey" WITH (FORCE)');
await postgres.query('CREATE DATABASE "test-misskey"');
await postgres.end();
const redis = new Redis({ host: '127.0.0.1', port: 56312 });
await redis.flushall();
redis.disconnect();
EOF
pnpm --filter backend migrate
node packages/backend/scripts/measure-memory.mjs > ../memory-head.json
- name: Upload Artifact
uses: actions/upload-artifact@v7
with:
name: memory-artifact-${{ matrix.memory-json-name }}
path: ${{ matrix.memory-json-name }}
name: memory-artifact-results
path: |
memory-base.json
memory-head.json
save-pr-number:
runs-on: ubuntu-latest

52
.github/workflows/report-backend-memory.yml vendored
View File

@@ -56,20 +56,25 @@ jobs:
variation() {
calc() {
BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // 0")
HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // 0")
BASE=$(echo "$BASE_MEMORY" | jq -r ".${1}.${2} // empty")
HEAD=$(echo "$HEAD_MEMORY" | jq -r ".${1}.${2} // empty")
if [ -z "$BASE" ] || [ -z "$HEAD" ]; then
echo "null"
return
fi
DIFF=$((HEAD - BASE))
if [ "$BASE" -gt 0 ]; then
DIFF_PERCENT=$(echo "scale=2; ($DIFF * 100) / $BASE" | bc)
DIFF_PERCENT=$(awk -v diff="$DIFF" -v base="$BASE" 'BEGIN { printf "%.2f", (diff * 100) / base }')
else
DIFF_PERCENT=0
DIFF_PERCENT=0.00
fi
# Convert KB to MB for readability
BASE_MB=$(echo "scale=2; $BASE / 1024" | bc)
HEAD_MB=$(echo "scale=2; $HEAD / 1024" | bc)
DIFF_MB=$(echo "scale=2; $DIFF / 1024" | bc)
BASE_MB=$(awk -v value="$BASE" 'BEGIN { printf "%.2f", value / 1024 }')
HEAD_MB=$(awk -v value="$HEAD" 'BEGIN { printf "%.2f", value / 1024 }')
DIFF_MB=$(awk -v value="$DIFF" 'BEGIN { printf "%.2f", value / 1024 }')
JSON=$(jq -c -n \
--argjson base "$BASE_MB" \
@@ -82,11 +87,20 @@ jobs:
}
JSON=$(jq -c -n \
--argjson HeapUsed "$(calc $1 HeapUsed)" \
--argjson HeapTotal "$(calc $1 HeapTotal)" \
--argjson External "$(calc $1 External)" \
--argjson ArrayBuffers "$(calc $1 ArrayBuffers)" \
--argjson Pss "$(calc $1 Pss)" \
--argjson Private_Dirty "$(calc $1 Private_Dirty)" \
--argjson Private_Clean "$(calc $1 Private_Clean)" \
--argjson Shared_Dirty "$(calc $1 Shared_Dirty)" \
--argjson Shared_Clean "$(calc $1 Shared_Clean)" \
--argjson VmRSS "$(calc $1 VmRSS)" \
--argjson VmHWM "$(calc $1 VmHWM)" \
--argjson VmSize "$(calc $1 VmSize)" \
--argjson VmData "$(calc $1 VmData)" \
'{VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
'{HeapUsed: $HeapUsed, HeapTotal: $HeapTotal, External: $External, ArrayBuffers: $ArrayBuffers, Pss: $Pss, Private_Dirty: $Private_Dirty, Private_Clean: $Private_Clean, Shared_Dirty: $Shared_Dirty, Shared_Clean: $Shared_Clean, VmRSS: $VmRSS, VmHWM: $VmHWM, VmSize: $VmSize, VmData: $VmData}')
echo "$JSON"
}
@@ -114,6 +128,10 @@ jobs:
echo "|--------|------:|------:|------:|------:|" >> ./output.md
line() {
if [ "$(echo "$RES" | jq -r ".${1}.${2} == null")" = "true" ]; then
return
fi
METRIC=$2
BASE=$(echo "$RES" | jq -r ".${1}.${2}.base")
HEAD=$(echo "$RES" | jq -r ".${1}.${2}.head")
@@ -125,8 +143,8 @@ jobs:
DIFF_PERCENT="+$DIFF_PERCENT"
fi
# highlight VmRSS
if [ "$2" = "VmRSS" ]; then
# highlight the most useful process and OS memory metrics
if [ "$2" = "HeapUsed" ] || [ "$2" = "Pss" ]; then
METRIC="**${METRIC}**"
BASE="**${BASE}**"
HEAD="**${HEAD}**"
@@ -137,10 +155,11 @@ jobs:
echo "| ${METRIC} | ${BASE} MB | ${HEAD} MB | ${DIFF} MB | ${DIFF_PERCENT}% |" >> ./output.md
}
line $1 VmRSS
line $1 VmHWM
line $1 VmSize
line $1 VmData
line $1 HeapUsed
line $1 HeapTotal
line $1 External
line $1 Pss
line $1 Private_Dirty
}
echo "### Before GC" >> ./output.md
@@ -156,8 +175,9 @@ jobs:
echo >> ./output.md
# Determine if this is a significant change (more than 5% increase)
if [ "$(echo "$RES" | jq -r '.afterGc.VmRSS.diff_percent | tonumber > 5')" = "true" ]; then
echo "⚠️ **Warning**: Memory usage has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
WARNING_METRIC=$(echo "$RES" | jq -r 'if .afterGc.Pss != null then "Pss" elif .afterGc.VmRSS != null then "VmRSS" else empty end')
if [ -n "$WARNING_METRIC" ] && [ "$(echo "$RES" | jq -r ".afterGc.${WARNING_METRIC}.diff_percent | tonumber > 5")" = "true" ]; then
echo "⚠️ **Warning**: Memory usage (${WARNING_METRIC}) has increased by more than 5%. Please verify this is not an unintended change." >> ./output.md
echo >> ./output.md
fi