feat: OTelのHTTPリクエスト自動計装を追加 (#17709)

CHANGELOG.md

@@ -39,6 +39,7 @@
 ### Server
 - Enhance: センシティブメディアの判定を外部サービス ([sensitive-detector](https://github.com/misskey-dev/sensitive-detector)) に分離し、`nsfwjs` / `@tensorflow/tfjs(-node)` の同梱と NSFW 判定モデルを廃止 (#16804)
 - Enhance: バックエンドの `otelForBackend` 設定で OpenTelemetry Traces を OTLP Collector に送信できるように
+- Enhance: OpenTelemetry を単独で利用する際、外向き HTTP リクエストを自動計装するように
 - Enhance: Sentry バックエンドの自動計装を `sentryForBackend.disabledIntegrations` で個別に無効化できるように
 - Enhance: Node.js 22.23.0以降、24.17.0以降、26.4.0以降をサポートするように
 - Enhance: Docker Image の Node.js を 26.4.0 に、Debian を trixie (v13) に更新

packages/backend/src/core/telemetry/adapters/OpenTelemetryAdapter.ts

@@ -7,6 +7,7 @@ import * as os from 'node:os';
 import cluster from 'node:cluster';
 import { envOption } from '@/env.js';
 import { registerDiagLogger } from '@/core/telemetry/telemetry-diag.js';
+import { installHttpClientInstrumentation } from '@/core/telemetry/http-client-instrumentation.js';
 import type { Span, SpanStatusCode, Tracer } from '@opentelemetry/api';
 import type { Resource, ResourceDetector } from '@opentelemetry/resources';
 import type { ParentBasedSampler, Sampler } from '@opentelemetry/sdk-trace-base';
@@ -22,6 +23,7 @@ type OpenTelemetryAdapterDeps = {
 	getActiveSpan: () => Span | undefined;
 	spanStatusCodeError: SpanStatusCode;
 	shutdownTimeout: number;
+	shutdownHttpClientInstrumentation?: () => void;
 };
 
 type CreateSamplerDeps = {
@@ -48,7 +50,7 @@ export class OpenTelemetryAdapter implements TelemetryAdapter {
 
 	public static async create(config: OtelBackendRuntimeConfig): Promise<OpenTelemetryAdapter> {
 		const [
-			{ diag, DiagLogLevel, SpanStatusCode, trace },
+			{ diag, DiagLogLevel, SpanKind, SpanStatusCode, trace },
 			{ W3CTraceContextPropagator },
 			{ OTLPTraceExporter },
 			{ defaultResource, detectResources, envDetector, resourceFromAttributes },
@@ -100,12 +102,19 @@ export class OpenTelemetryAdapter implements TelemetryAdapter {
 		});
 
 		// provider操作をdepsに閉じ込め、span wrapper本体をユニットテストしやすくする。
+		const tracer = provider.getTracer('misskey-backend');
+
 		return new OpenTelemetryAdapter({
-			tracer: provider.getTracer('misskey-backend'),
+			tracer,
 			provider,
 			getActiveSpan: () => trace.getActiveSpan(),
 			spanStatusCodeError: SpanStatusCode.ERROR,
 			shutdownTimeout: DEFAULT_SHUTDOWN_TIMEOUT,
+			shutdownHttpClientInstrumentation: installHttpClientInstrumentation({
+				tracer,
+				spanKindClient: SpanKind.CLIENT,
+				spanStatusCodeError: SpanStatusCode.ERROR,
+			}),
 		});
 	}
 
@@ -157,6 +166,7 @@ export class OpenTelemetryAdapter implements TelemetryAdapter {
 	}
 
 	public async shutdown(): Promise<void> {
+		this.deps.shutdownHttpClientInstrumentation?.();
 		// BatchSpanProcessorのflushが詰まってもプロセス終了を妨げないよう、上限時間を設ける。
 		// タイムアウト側のtimerは、flushが先に終わった場合にイベントループを無駄に引き留めないようclearする。
 		let timer: NodeJS.Timeout | undefined;

packages/backend/src/core/telemetry/http-client-instrumentation.ts

@@ -0,0 +1,122 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { channel } from 'node:diagnostics_channel';
+import type { ClientRequest, IncomingMessage } from 'node:http';
+import type { Span, SpanOptions, SpanStatusCode, Tracer } from '@opentelemetry/api';
+
+const HTTP_CLIENT_REQUEST_CREATED = 'http.client.request.created';
+const HTTP_CLIENT_RESPONSE_FINISH = 'http.client.response.finish';
+const HTTP_CLIENT_REQUEST_ERROR = 'http.client.request.error';
+
+type HttpClientSpan = Pick<Span, 'end' | 'recordException' | 'setAttribute' | 'setStatus'>;
+
+type HttpClientInstrumentationDeps = {
+	tracer: Pick<Tracer, 'startSpan'>;
+	spanKindClient: SpanOptions['kind'];
+	spanStatusCodeError: SpanStatusCode;
+	subscribe: (name: string, listener: (message: unknown) => void) => () => void;
+};
+
+type RequestCreatedMessage = { request: ClientRequest };
+type ResponseFinishMessage = { request: ClientRequest; response: IncomingMessage };
+type RequestErrorMessage = { request: ClientRequest; error: Error };
+
+/**
+ * require フックを使わず、Node.js 組み込み HTTP クライアントの diagnostics channel を計装する。
+ * telemetry 初期化前に読み込まれたモジュールも対象になる。
+ */
+export function createHttpClientInstrumentation(deps: HttpClientInstrumentationDeps): () => void {
+	const spans = new WeakMap<ClientRequest, HttpClientSpan>();
+
+	const unsubscribeCreated = deps.subscribe(HTTP_CLIENT_REQUEST_CREATED, (message: unknown) => {
+		const { request } = message as RequestCreatedMessage;
+		const { url, host, port } = getRequestDetails(request);
+		const method = request.method ?? 'GET';
+		const span = deps.tracer.startSpan(method, {
+			kind: deps.spanKindClient,
+			attributes: {
+				'http.request.method': method,
+				'url.full': url,
+				'server.address': host,
+				'server.port': port,
+			},
+		});
+		spans.set(request, span);
+	});
+
+	const unsubscribeResponseFinish = deps.subscribe(HTTP_CLIENT_RESPONSE_FINISH, (message: unknown) => {
+		const { request, response } = message as ResponseFinishMessage;
+		const span = spans.get(request);
+		if (span == null) return;
+
+		const statusCode = response.statusCode;
+		if (statusCode != null) {
+			span.setAttribute('http.response.status_code', statusCode);
+		}
+		if (response.httpVersion != null) {
+			span.setAttribute('network.protocol.version', response.httpVersion);
+		}
+		if (statusCode != null && statusCode >= 400) {
+			span.setAttribute('error.type', String(statusCode));
+			span.setStatus({ code: deps.spanStatusCodeError });
+		}
+		span.end();
+		spans.delete(request);
+	});
+
+	const unsubscribeRequestError = deps.subscribe(HTTP_CLIENT_REQUEST_ERROR, (message: unknown) => {
+		const { request, error } = message as RequestErrorMessage;
+		const span = spans.get(request);
+		if (span == null) return;
+
+		span.recordException(error);
+		span.setAttribute('error.type', getErrorType(error));
+		span.setStatus({ code: deps.spanStatusCodeError });
+		span.end();
+		spans.delete(request);
+	});
+
+	return () => {
+		unsubscribeCreated();
+		unsubscribeResponseFinish();
+		unsubscribeRequestError();
+	};
+}
+
+export function installHttpClientInstrumentation(deps: Omit<HttpClientInstrumentationDeps, 'subscribe'>): () => void {
+	return createHttpClientInstrumentation({
+		...deps,
+		subscribe: (name, listener) => {
+			const diagnosticChannel = channel(name);
+			diagnosticChannel.subscribe(listener);
+			return () => diagnosticChannel.unsubscribe(listener);
+		},
+	});
+}
+
+function getRequestDetails(request: ClientRequest): { url: string; host: string; port: number } {
+	const protocol = request.protocol ?? 'http:';
+	const host = request.getHeader('host')?.toString() ?? request.host ?? 'localhost';
+	const url = new URL(request.path || '/', `${protocol}//${host}`);
+	// URL 属性には認証情報やクエリ文字列を含めない。
+	url.username = '';
+	url.password = '';
+	url.search = '';
+	url.hash = '';
+
+	return {
+		url: url.toString(),
+		host: url.hostname,
+		// URL.port は既定ポートでは空文字列になるため、スキームから補う。
+		port: url.port === '' ? (url.protocol === 'https:' ? 443 : 80) : Number(url.port),
+	};
+}
+
+function getErrorType(error: Error): string {
+	// Node.js の system error code は安定した低カーディナリティの識別子になる。
+	const code = (error as NodeJS.ErrnoException).code;
+	return code ?? error.name;
+}

packages/backend/test/unit/core/telemetry/http-client-instrumentation.ts

@@ -0,0 +1,108 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { describe, expect, test, vi } from 'vitest';
+import { SpanKind, SpanStatusCode } from '@opentelemetry/api';
+import { createHttpClientInstrumentation } from '@/core/telemetry/http-client-instrumentation.js';
+
+function request() {
+	return {
+		method: 'POST',
+		protocol: 'https:',
+		path: '/inbox?token=secret',
+		host: 'remote.example',
+		getHeader: vi.fn((name: string) => name === 'host' ? 'user:password@remote.example:8443' : undefined),
+	};
+}
+
+describe('http-client-instrumentation', () => {
+	test('creates and completes a sanitized CLIENT span from diagnostics channels', () => {
+		const listeners = new Map<string, (message: unknown) => void>();
+		const span = {
+			end: vi.fn(),
+			recordException: vi.fn(),
+			setAttribute: vi.fn(),
+			setStatus: vi.fn(),
+		};
+		const tracer = { startSpan: vi.fn(() => span) } as any;
+		const unsubscribe = createHttpClientInstrumentation({
+			tracer,
+			spanKindClient: SpanKind.CLIENT,
+			spanStatusCodeError: SpanStatusCode.ERROR,
+			subscribe: (name, listener) => {
+				listeners.set(name, listener);
+				return () => listeners.delete(name);
+			},
+		});
+		const clientRequest = request();
+
+		listeners.get('http.client.request.created')!({ request: clientRequest });
+		listeners.get('http.client.response.finish')!({
+			request: clientRequest,
+			response: { statusCode: 201, httpVersion: '1.1' },
+		});
+
+		expect(tracer.startSpan).toHaveBeenCalledWith('POST', {
+			kind: SpanKind.CLIENT,
+			attributes: {
+				'http.request.method': 'POST',
+				'url.full': 'https://remote.example:8443/inbox',
+				'server.address': 'remote.example',
+				'server.port': 8443,
+			},
+		});
+		expect(span.setAttribute).toHaveBeenCalledWith('http.response.status_code', 201);
+		expect(span.setAttribute).toHaveBeenCalledWith('network.protocol.version', '1.1');
+		expect(span.end).toHaveBeenCalledTimes(1);
+		unsubscribe();
+		expect(listeners).toHaveLength(0);
+	});
+
+	test('records a request error and ends the span once', () => {
+		const listeners = new Map<string, (message: unknown) => void>();
+		const span = { end: vi.fn(), recordException: vi.fn(), setAttribute: vi.fn(), setStatus: vi.fn() };
+		const error = Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' });
+		const clientRequest = request();
+		createHttpClientInstrumentation({
+			tracer: { startSpan: vi.fn(() => span) } as any,
+			spanKindClient: SpanKind.CLIENT,
+			spanStatusCodeError: SpanStatusCode.ERROR,
+			subscribe: (name, listener) => {
+				listeners.set(name, listener);
+				return () => listeners.delete(name);
+			},
+		});
+
+		listeners.get('http.client.request.created')!({ request: clientRequest });
+		listeners.get('http.client.request.error')!({ request: clientRequest, error });
+		listeners.get('http.client.response.finish')!({ request: clientRequest, response: { statusCode: 200 } });
+
+		expect(span.recordException).toHaveBeenCalledWith(error);
+		expect(span.setAttribute).toHaveBeenCalledWith('error.type', 'ECONNREFUSED');
+		expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+		expect(span.end).toHaveBeenCalledTimes(1);
+	});
+
+	test('records the response status code as error.type for an error response', () => {
+		const listeners = new Map<string, (message: unknown) => void>();
+		const span = { end: vi.fn(), recordException: vi.fn(), setAttribute: vi.fn(), setStatus: vi.fn() };
+		const clientRequest = request();
+		createHttpClientInstrumentation({
+			tracer: { startSpan: vi.fn(() => span) } as any,
+			spanKindClient: SpanKind.CLIENT,
+			spanStatusCodeError: SpanStatusCode.ERROR,
+			subscribe: (name, listener) => {
+				listeners.set(name, listener);
+				return () => listeners.delete(name);
+			},
+		});
+
+		listeners.get('http.client.request.created')!({ request: clientRequest });
+		listeners.get('http.client.response.finish')!({ request: clientRequest, response: { statusCode: 502 } });
+
+		expect(span.setAttribute).toHaveBeenCalledWith('error.type', '502');
+		expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+	});
+});