From a12ee2e5d72c40252e11a164b2231667c9697667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=8A=E3=81=95=E3=82=80=E3=81=AE=E3=81=B2=E3=81=A8?= <46447427+samunohito@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:09:37 +0900 Subject: [PATCH] Merge commit from fork --- .../src/server/api/endpoints/fetch-rss.ts | 116 ++++++++++- .../unit/server/api/endpoints/fetch-rss.ts | 192 ++++++++++++++++++ packages/misskey-js/src/autogen/types.ts | 9 + 3 files changed, 306 insertions(+), 11 deletions(-) create mode 100644 packages/backend/test/unit/server/api/endpoints/fetch-rss.ts diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index ba48b0119e..7b777e46c4 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -7,8 +7,11 @@ import Parser from 'rss-parser'; import { Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { ApiError } from '../error.js'; -const rssParser = new Parser(); +const MAX_URL_LENGTH = 8192; +const MAX_RESPONSE_SIZE = 1024 * 1024; +const MAX_CONCURRENT_REQUESTS = 32; export const meta = { tags: ['meta'], @@ -17,6 +20,34 @@ export const meta = { allowGet: true, cacheSec: 60 * 3, + limit: { + duration: 60 * 1000, + max: 300, + }, + + errors: { + invalidUrl: { + message: 'Invalid URL.', + code: 'INVALID_URL', + id: '89b7ee05-ccfc-4bdd-9b13-61172fd1e06c', + httpStatusCode: 400, + }, + fetchRssFailed: { + message: 'Failed to fetch RSS.', + code: 'FETCH_RSS_FAILED', + id: '8db5d3d8-31d7-452f-b0cc-ca3b8925de12', + kind: 'server', + httpStatusCode: 422, + }, + fetchRssUnavailable: { + message: 'RSS fetching is temporarily unavailable.', + code: 'FETCH_RSS_UNAVAILABLE', + id: '91e6ff44-c63f-4725-9ad0-b7a40d7f7655', + kind: 'server', + httpStatusCode: 503, + }, + }, + res: { type: 'object', properties: { @@ -215,21 +246,84 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export + private readonly inFlightRequests = new Map>>>(); + private activeRequestCount = 0; + constructor( private httpRequestService: HttpRequestService, ) { - super(meta, paramDef, async (ps, me) => { - const res = await this.httpRequestService.send(ps.url, { - method: 'GET', - headers: { - Accept: 'application/rss+xml, */*', - }, - timeout: 5000, - }); + super(meta, paramDef, async (ps) => { + const url = this.normalizeUrl(ps.url); + const inFlightRequest = this.inFlightRequests.get(url); + if (inFlightRequest != null) { + return await inFlightRequest; + } - const text = await res.text(); + if (this.activeRequestCount >= MAX_CONCURRENT_REQUESTS) { + throw new ApiError(meta.errors.fetchRssUnavailable); + } - return rssParser.parseString(text); + this.activeRequestCount++; + const request = this.fetchRss(url) + .catch(() => { + throw new ApiError(meta.errors.fetchRssFailed); + }) + .finally(() => { + this.inFlightRequests.delete(url); + this.activeRequestCount--; + }); + this.inFlightRequests.set(url, request); + + return await request; }); } + + private normalizeUrl(input: string): string { + if (input.length === 0 || input.length > MAX_URL_LENGTH) { + throw new ApiError(meta.errors.invalidUrl); + } + + let url: URL; + try { + url = new URL(input); + } catch { + throw new ApiError(meta.errors.invalidUrl); + } + + if ( + (url.protocol !== 'http:' && url.protocol !== 'https:') || + url.username !== '' || + url.password !== '' + ) { + throw new ApiError(meta.errors.invalidUrl); + } + + url.hash = ''; + return url.href; + } + + private async fetchRss(url: string): Promise>> { + const res = await this.httpRequestService.send(url, { + method: 'GET', + headers: { + Accept: 'application/rss+xml, */*', + }, + timeout: 5000, + size: MAX_RESPONSE_SIZE, + }); + + const finalUrl = new URL(res.url); + if (finalUrl.protocol !== 'http:' && finalUrl.protocol !== 'https:') { + throw new Error('Invalid final URL protocol'); + } + + const text = await res.text(); + const rssParser = new Parser({ + xml2js: { + async: true, + }, + }); + + return await rssParser.parseString(text); + } } diff --git a/packages/backend/test/unit/server/api/endpoints/fetch-rss.ts b/packages/backend/test/unit/server/api/endpoints/fetch-rss.ts new file mode 100644 index 0000000000..186b785d9b --- /dev/null +++ b/packages/backend/test/unit/server/api/endpoints/fetch-rss.ts @@ -0,0 +1,192 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import FetchRssEndpoint, { meta } from '@/server/api/endpoints/fetch-rss.js'; +import { ApiError } from '@/server/api/error.js'; +import type { Mocked } from 'vitest'; +import type { Response } from 'node-fetch'; + +const rssParserMocks = vi.hoisted(() => ({ + constructor: vi.fn(), + parseString: vi.fn(), +})); + +vi.mock('rss-parser', () => ({ + default: class { + constructor(options: unknown) { + rssParserMocks.constructor(options); + } + + public parseString(input: string) { + return rssParserMocks.parseString(input); + } + }, +})); + +const RSS = 'Test'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function response(url: string, text = RSS): Response { + return { + url, + text: vi.fn().mockResolvedValue(text), + } as unknown as Response; +} + +describe('fetch-rss endpoint', () => { + let httpRequestService: Mocked; + let endpoint: FetchRssEndpoint; + + beforeEach(() => { + rssParserMocks.constructor.mockReset(); + rssParserMocks.parseString.mockReset(); + rssParserMocks.parseString.mockResolvedValue({ items: [] }); + httpRequestService = { + send: vi.fn(), + } as unknown as Mocked; + endpoint = new FetchRssEndpoint(httpRequestService); + }); + + async function exec(url: string) { + return await endpoint.exec({ url }, null, null); + } + + async function expectApiError(promise: Promise, code: string, status: number) { + await expect(promise).rejects.toMatchObject({ + code, + httpStatusCode: status, + info: undefined, + }); + } + + test.each([ + '', + 'not a URL', + 'ftp://example.com/feed.xml', + `https://example.com/${'a'.repeat(8192)}`, + ])('rejects invalid URL: %s', async (url) => { + await expectApiError(exec(url), 'INVALID_URL', 400); + expect(httpRequestService.send).not.toHaveBeenCalled(); + }); + + test.each([ + 'https://user@example.com/feed.xml', + 'https://user:password@example.com/feed.xml', + ])('rejects URL containing credentials: %s', async (url) => { + await expectApiError(exec(url), 'INVALID_URL', 400); + expect(httpRequestService.send).not.toHaveBeenCalled(); + }); + + test('does not expose details from internal errors', async () => { + httpRequestService.send.mockRejectedValue(new Error('secret upstream details')); + + const promise = exec('https://example.com/feed.xml'); + await expectApiError(promise, 'FETCH_RSS_FAILED', 422); + await expect(promise).rejects.not.toMatchObject({ + message: expect.stringContaining('secret upstream details'), + }); + }); + + test('passes the 1 MiB response limit to HttpRequestService', async () => { + httpRequestService.send.mockResolvedValue(response('https://example.com/feed.xml')); + + await exec('https://example.com/feed.xml'); + + expect(httpRequestService.send).toHaveBeenCalledWith('https://example.com/feed.xml', expect.objectContaining({ + size: 1024 * 1024, + })); + }); + + test('creates an asynchronous RSS parser for every request', async () => { + httpRequestService.send + .mockResolvedValueOnce(response('https://example.com/first.xml')) + .mockResolvedValueOnce(response('https://example.com/second.xml')); + + await exec('https://example.com/first.xml'); + await exec('https://example.com/second.xml'); + + expect(rssParserMocks.constructor).toHaveBeenCalledTimes(2); + expect(rssParserMocks.constructor).toHaveBeenNthCalledWith(1, { xml2js: { async: true } }); + expect(rssParserMocks.constructor).toHaveBeenNthCalledWith(2, { xml2js: { async: true } }); + }); + + test('rejects a non-HTTP final URL without exposing it', async () => { + httpRequestService.send.mockResolvedValue(response('file:///secret/feed.xml')); + + await expectApiError(exec('https://example.com/feed.xml'), 'FETCH_RSS_FAILED', 422); + }); + + test('shares an in-flight request for the same normalized URL', async () => { + const pending = deferred(); + httpRequestService.send.mockReturnValue(pending.promise); + + const first = exec('HTTPS://EXAMPLE.COM:443/feed.xml#first'); + const second = exec('https://example.com/feed.xml#second'); + expect(httpRequestService.send).toHaveBeenCalledTimes(1); + + pending.resolve(response('https://example.com/feed.xml')); + await expect(Promise.all([first, second])).resolves.toHaveLength(2); + expect(httpRequestService.send).toHaveBeenCalledTimes(1); + }); + + test('limits concurrent requests for different URLs to 32', async () => { + const pending = deferred(); + httpRequestService.send.mockReturnValue(pending.promise); + + const requests = Array.from({ length: 32 }, (_, i) => exec(`https://example.com/${i}.xml`)); + expect(httpRequestService.send).toHaveBeenCalledTimes(32); + await expectApiError(exec('https://example.com/overflow.xml'), 'FETCH_RSS_UNAVAILABLE', 503); + expect(httpRequestService.send).toHaveBeenCalledTimes(32); + + pending.resolve(response('https://example.com/feed.xml')); + await Promise.all(requests); + + httpRequestService.send.mockResolvedValue(response('https://example.com/after.xml')); + await expect(exec('https://example.com/after.xml')).resolves.toBeDefined(); + }); + + test('cleans up the in-flight request and concurrency slot after success', async () => { + httpRequestService.send.mockResolvedValue(response('https://example.com/feed.xml')); + + await exec('https://example.com/feed.xml'); + await exec('https://example.com/feed.xml'); + + expect(httpRequestService.send).toHaveBeenCalledTimes(2); + }); + + test('cleans up the in-flight request and concurrency slot after failure', async () => { + httpRequestService.send.mockRejectedValue(new Error('upstream failed')); + const requests = Array.from({ length: 32 }, (_, i) => exec(`https://example.com/${i}.xml`)); + await Promise.allSettled(requests); + + httpRequestService.send.mockResolvedValue(response('https://example.com/0.xml')); + await expect(exec('https://example.com/0.xml')).resolves.toBeDefined(); + expect(httpRequestService.send).toHaveBeenCalledTimes(33); + }); + + test('has the expected rate limit metadata', () => { + expect(meta.limit).toEqual({ + duration: 60 * 1000, + max: 300, + }); + }); + + test('uses only the declared structured API errors', () => { + expect(new ApiError(meta.errors.invalidUrl)).toMatchObject({ code: 'INVALID_URL', httpStatusCode: 400 }); + expect(new ApiError(meta.errors.fetchRssFailed)).toMatchObject({ code: 'FETCH_RSS_FAILED', httpStatusCode: 422 }); + expect(new ApiError(meta.errors.fetchRssUnavailable)).toMatchObject({ code: 'FETCH_RSS_UNAVAILABLE', httpStatusCode: 503 }); + }); +}); diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 0f5db3e205..536e59613f 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -21879,6 +21879,15 @@ export interface operations { 'application/json': components['schemas']['Error']; }; }; + /** @description Too many requests */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + 'application/json': components['schemas']['Error']; + }; + }; /** @description Internal server error */ 500: { headers: {