feat: auto nsfw detection (#8840)

* feat: auto nsfw detection

* ✌️

* Update ja-JP.yml

* Update ja-JP.yml

* ポルノ判定のしきい値を高めに

* エラーハンドリングちゃんとした

* Update ja-JP.yml

* 感度設定を強化

* refactor

* feat: add video support for auto nsfw detection

* rename: image -> media

* .js

* fix: add missing error handling

* fix: use valid pathname instead of using filename due to invalid usage

* perf(nsfw-detection): decode frames

* disable detection of video for some reasons

* perf(nsfw-detection): streamify detection process for video

* disable disallowUploadWhenPredictedAsPorn option

* fix(nsfw-detection): improve reliability

* fix(nsfw-detection): use Math.ceil instead of Math.round

* perf(nsfw-detection): delete tmp frames after used

* fix(nsfw-detection): FSWatcher does not emit ready event

* perf(nsfw-detection): skip black frames

* refactor: strip exists check

* Update package.json

* めっちゃ変えた

* lint

* Update COPYING

* オプションで動画解析できるように

* Update yarn.lock

* Update CHANGELOG.md

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

packages/backend/src/services/detect-sensitive.ts

@@ -0,0 +1,28 @@
+import * as fs from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname } from 'node:path';
+import * as nsfw from 'nsfwjs';
+import * as tf from '@tensorflow/tfjs-node';
+
+const _filename = fileURLToPath(import.meta.url);
+const _dirname = dirname(_filename);
+
+let model: nsfw.NSFWJS;
+
+export async function detectSensitive(path: string): Promise<nsfw.predictionType[] | null> {
+	try {
+		if (model == null) model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
+
+		const buffer = await fs.promises.readFile(path);
+		const image = await tf.node.decodeImage(buffer, 3) as tf.Tensor3D;
+		try {
+			const predictions = await model.classify(image);
+			return predictions;
+		} finally {
+			image.dispose();
+		}
+	} catch (err) {
+		console.error(err);
+		return null;
+	}
+}

packages/backend/src/services/drive/add-file.ts

@@ -16,6 +16,7 @@ import { driveChart, perUserDriveChart, instanceChart } from '@/services/chart/i
 import { genId } from '@/misc/gen-id.js';
 import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
+import { IdentifiableError } from '@/misc/identifiable-error.js';
 import { getS3 } from './s3.js';
 import { InternalStorage } from './internal-storage.js';
 import { IImage, convertSharpToJpeg, convertSharpToWebp, convertSharpToPng } from './image-processor.js';
@@ -349,9 +350,31 @@ export async function addFile({
 	requestIp = null,
 	requestHeaders = null,
 }: AddFileArgs): Promise<DriveFile> {
-	const info = await getFileInfo(path);
+	let skipNsfwCheck = false;
+	const instance = await fetchMeta();
+	if (user == null) skipNsfwCheck = true;
+	if (instance.sensitiveMediaDetection === 'none') skipNsfwCheck = true;
+	if (user && instance.sensitiveMediaDetection === 'local' && Users.isRemoteUser(user)) skipNsfwCheck = true;
+	if (user && instance.sensitiveMediaDetection === 'remote' && Users.isLocalUser(user)) skipNsfwCheck = true;
+
+	const info = await getFileInfo(path, {
+		skipSensitiveDetection: skipNsfwCheck,
+		sensitiveThreshold: // 感度が高いほどしきい値は低くすることになる
+			instance.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 0.1 :
+			instance.sensitiveMediaDetectionSensitivity === 'high' ? 0.3 :
+			instance.sensitiveMediaDetectionSensitivity === 'low' ? 0.7 :
+			instance.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0.9 :
+			0.5,
+		sensitiveThresholdForPorn: 0.75,
+		enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos,
+	});
 	logger.info(`${JSON.stringify(info)}`);
 
+	// 現状 false positive が多すぎて実用に耐えない
+	//if (info.porn && instance.disallowUploadWhenPredictedAsPorn) {
+	//	throw new IdentifiableError('282f77bf-5816-4f72-9264-aa14d8261a21', 'Detected as porn.');
+	//}
+
 	// detect name
 	const detectedName = name || (info.type.ext ? `untitled.${info.type.ext}` : 'untitled');
 
@@ -387,7 +410,7 @@ export async function addFile({
 		// If usage limit exceeded
 		if (usage + info.size > driveCapacity) {
 			if (Users.isLocalUser(user)) {
-				throw new Error('no-free-space');
+				throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.');
 			} else {
 				// (アバターまたはバナーを含まず)最も古いファイルを削除する
 				deleteOldFile(await Users.findOneByOrFail({ id: user.id }) as IRemoteUser);
@@ -441,6 +464,8 @@ export async function addFile({
 	file.isLink = isLink;
 	file.requestIp = requestIp;
 	file.requestHeaders = requestHeaders;
+	file.maybeSensitive = info.sensitive;
+	file.maybePorn = info.porn;
 	file.isSensitive = user
 		? Users.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
 		(sensitive !== null && sensitive !== undefined)
@@ -448,6 +473,9 @@ export async function addFile({
 			: false
 		: false;
 
+	if (info.sensitive && profile!.autoSensitive) file.isSensitive = true;
+	if (info.sensitive && instance.setSensitiveFlagAutomatically) file.isSensitive = true;
+
 	if (url !== null) {
 		file.src = url;