enhance: 絵文字データの参照を自前ライブラリに変更 (#17381)

* wip

* enhance: 絵文字データの参照を自前ライブラリに変更

* fix

* update to v17.0.2

* fix assets handling

* fix

* update mfm-js

* update emoji library

* Update COPYING [ci skip]

* Update Changelog

* Update Changelog

* fix: 端末の絵文字にフォールバックできるように

.gitignore

@@ -81,6 +81,3 @@ vite.config.local-dev.ts.timestamp-*
 
 # VSCode addon
 .favorites.json
-
-# Affinity
-*.af~lock~

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "fluent-emojis"]
-	path = fluent-emojis
-	url = https://github.com/misskey-dev/emojis.git

CHANGELOG.md

@@ -1,10 +1,12 @@
 ## Unreleased
 
 ### General
--
+- Enhance: Unicode 17.0 に収録されている絵文字の処理・表示に対応
+  - Fluent Emojiや端末ネイティブの絵文字を利用している場合は、最新の絵文字に対応しておらず正しく表示できない可能性があります。絵文字が表示できない場合は、表示に使用する絵文字をTwemojiに切り替えてご利用ください。
 
 ### Client
 - Enhance: テーマのプレビュー時、リロードせずにもとのテーマに戻せるように
+- Enhance: Fluent Emojiを更新し、Unicode 15+相当の絵文字の表示に対応
 - Fix: テーマエディター使用時に、最初の変更のみ適用される問題を修正
 - Fix: テーマのプレビュー時、既存のテーマとIDが被っている場合にプレビューできない問題を修正
 - Fix: テーマのインストールエラーの表示を改善

COPYING

@@ -3,13 +3,6 @@ Copyright © 2014-2026 syuilo and contributors
 
 And is distributed under The GNU Affero General Public License Version 3, you should have received a copy of the license file as LICENSE.
 
-
-Misskey includes several third-party Open-Source softwares.
-
-Emoji keywords for Unicode 11 and below by Mu-An Chiou
-License: MIT
-https://github.com/muan/emojilib/blob/master/LICENSE
-
 RsaSignature2017 implementation by Transmute Industries Inc
 License: MIT
 https://github.com/transmute-industries/RsaSignature2017/blob/master/LICENSE

Dockerfile

@@ -103,7 +103,6 @@ COPY --chown=misskey:misskey --from=native-builder /misskey/packages/misskey-rev
 COPY --chown=misskey:misskey --from=native-builder /misskey/packages/misskey-bubble-game/built ./packages/misskey-bubble-game/built
 COPY --chown=misskey:misskey --from=native-builder /misskey/packages/backend/built ./packages/backend/built
 COPY --chown=misskey:misskey --from=native-builder /misskey/packages/i18n/built ./packages/i18n/built
-COPY --chown=misskey:misskey --from=native-builder /misskey/fluent-emojis /misskey/fluent-emojis
 COPY --chown=misskey:misskey . ./
 
 ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so

locales/ja-JP.yml

@@ -3556,17 +3556,3 @@ _qr:
   scanFile: "端末の画像をスキャン"
   raw: "テキスト"
   mfm: "MFM"
-
-_room:
-  snapToGrid: "グリッドにスナップ"
-  gridScale: "グリッドサイズ"
-  thereAreUnsavedChanges: "未保存の変更があります"
-  revertAllChangesConfirmation: "全ての変更を取り消し、部屋を最後に保存した状態まで戻しますか？"
-  graphicsQuality: "グラフィックの品質"
-  frameRate: "フレームレート"
-  resolution: "解像度"
-  yourDeviceNotSupported_title: "お使いのデバイスはMisskeyRoomをサポートしていません。"
-  yourDeviceNotSupported_description: "MisskeyRoomを動作させるには、WebGPUをサポートするデバイスが必要です。"
-  failedToInitialize: "初期化に失敗しました"
-  crushed_description: "バグ、またはデバイスのリソース不足の可能性が考えられます。"
-  antialiasing: "アンチエイリアス"

packages/backend/package.json

@@ -55,7 +55,6 @@
 	"dependencies": {
 		"@aws-sdk/client-s3": "3.1037.0",
 		"@aws-sdk/lib-storage": "3.1037.0",
-		"@discordapp/twemoji": "16.0.1",
 		"@fastify/accepts": "5.0.4",
 		"@fastify/cors": "11.2.0",
 		"@fastify/express": "4.0.5",
@@ -63,6 +62,8 @@
 		"@fastify/multipart": "10.0.0",
 		"@fastify/static": "9.1.3",
 		"@kitajs/html": "4.2.13",
+		"@misskey-dev/emoji-assets": "17.0.3",
+		"@misskey-dev/emoji-data": "17.0.3",
 		"@misskey-dev/sharp-read-bmp": "1.2.0",
 		"@misskey-dev/summaly": "5.3.0",
 		"@napi-rs/canvas": "0.1.100",
@@ -76,7 +77,6 @@
 		"@simplewebauthn/server": "13.3.0",
 		"@sinonjs/fake-timers": "15.3.2",
 		"@smithy/node-http-handler": "4.6.1",
-		"@twemoji/parser": "16.0.0",
 		"accepts": "1.3.8",
 		"ajv": "8.20.0",
 		"archiver": "7.0.1",
@@ -111,7 +111,7 @@
 		"jsonld": "9.0.0",
 		"juice": "11.1.1",
 		"meilisearch": "0.57.0",
-		"mfm-js": "0.25.0",
+		"mfm-js": "0.26.0",
 		"mime-types": "3.0.2",
 		"misskey-js": "workspace:*",
 		"misskey-reversi": "workspace:*",

packages/backend/src/server/web/ClientServerService.ts

@@ -71,7 +71,7 @@ export class ClientServerService {
 	private readonly clientAssets: string;
 	private readonly assets: string;
 	private readonly swAssets: string;
-	private readonly fluentEmojisDir: string;
+	private readonly fluentEmojiDir: string;
 	private readonly twemojiDir: string;
 	private readonly frontendViteOut: string;
 	private readonly frontendEmbedViteOut: string;
@@ -135,8 +135,8 @@ export class ClientServerService {
 		this.clientAssets = resolve(frontendRootdir, 'assets');
 		this.assets = resolve(this.config.rootDir, 'built/_frontend_dist_');
 		this.swAssets = resolve(this.config.rootDir, 'built/_sw_dist_');
-		this.fluentEmojisDir = resolve(this.config.rootDir, 'fluent-emojis/dist');
-		this.twemojiDir = resolve(backendRootdir, 'node_modules/@discordapp/twemoji/dist/svg');
+		this.fluentEmojiDir = resolve(backendRootdir, 'node_modules/@misskey-dev/emoji-assets/built/fluent-emoji');
+		this.twemojiDir = resolve(backendRootdir, 'node_modules/@misskey-dev/emoji-assets/built/twemoji');
 		this.frontendViteOut = resolve(this.config.rootDir, 'built/_frontend_vite_');
 		this.frontendEmbedViteOut = resolve(this.config.rootDir, 'built/_frontend_embed_vite_');
 		this.tarball = resolve(this.config.rootDir, 'built/tarball');
@@ -306,7 +306,7 @@ export class ClientServerService {
 
 			reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
 
-			return reply.sendFile(path, this.fluentEmojisDir, {
+			return reply.sendFile(path, this.fluentEmojiDir, {
 				maxAge: ms('30 days'),
 			});
 		});

packages/frontend-embed/package.json

@@ -10,10 +10,8 @@
 		"lint": "pnpm typecheck && pnpm eslint"
 	},
 	"dependencies": {
-		"@discordapp/twemoji": "16.0.1",
 		"@rollup/plugin-json": "6.1.0",
 		"@rollup/pluginutils": "5.3.0",
-		"@twemoji/parser": "16.0.0",
 		"@vitejs/plugin-vue": "6.0.6",
 		"buraha": "0.0.1",
 		"estree-walker": "3.0.3",
@@ -21,7 +19,7 @@
 		"i18n": "workspace:*",
 		"icons-subsetter": "workspace:*",
 		"json5": "2.2.3",
-		"mfm-js": "0.25.0",
+		"mfm-js": "0.26.0",
 		"misskey-js": "workspace:*",
 		"punycode.js": "2.3.1",
 		"rollup": "4.60.2",
@@ -31,6 +29,7 @@
 		"vue": "3.5.33"
 	},
 	"devDependencies": {
+		"@misskey-dev/emoji-assets": "17.0.3",
 		"@misskey-dev/summaly": "5.3.0",
 		"@tabler/icons-webfont": "3.35.0",
 		"@testing-library/vue": "8.1.0",

packages/frontend-embed/src/style.scss

@@ -21,7 +21,7 @@ html {
 	accent-color: var(--MI_THEME-accent);
 	overflow: clip;
 	overflow-wrap: break-word;
-	font-family: 'Hiragino Maru Gothic Pro', "BIZ UDGothic", Roboto, HelveticaNeue, Arial, sans-serif;
+	font-family: 'Hiragino Maru Gothic Pro', "BIZ UDGothic", Roboto, HelveticaNeue, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
 	font-size: 14px;
 	line-height: 1.35;
 	text-size-adjust: 100%;

packages/frontend-embed/vite.config.ts

@@ -98,7 +98,8 @@ export function getConfig(): UserConfig {
 				'@/': __dirname + '/src/',
 				'@@/': __dirname + '/../frontend-shared/',
 				'/client-assets/': __dirname + '/assets/',
-				'/static-assets/': __dirname + '/../backend/assets/'
+				'/static-assets/': __dirname + '/../backend/assets/',
+				'/fluent-emoji/': '@misskey-dev/emoji-assets/fluent-emoji/',
 			},
 		},
 

packages/frontend-shared/js/emoji-base.ts

@@ -19,7 +19,7 @@ export function char2fluentEmojiFilePath(char: string): string {
 	// Fluent Emojiは国旗非対応 https://github.com/microsoft/fluentui-emoji/issues/25
 	if (codes[0]?.startsWith('1f1')) return char2twemojiFilePath(char);
 	if (!codes.includes('200d')) codes = codes.filter(x => x !== 'fe0f');
-	codes = codes.filter(x => x != null && x.length > 0);
-	const fileName = (codes as string[]).map(x => x.padStart(4, '0')).join('-');
+	codes = codes.filter(x => x && x.length);
+	const fileName = codes.join('-');
 	return `${fluentEmojiPngBase}/${fileName}.png`;
 }

packages/frontend-shared/js/emojilist.ts

@@ -11,8 +11,7 @@ export type UnicodeEmojiDef = {
 	category: typeof unicodeEmojiCategories[number];
 };
 
-// initial converted from https://github.com/muan/emojilib/commit/242fe68be86ed6536843b83f7e32f376468b38fb
-import _emojilist from './emojilist.json' with { type: 'json' };
+import _emojilist from '@misskey-dev/emoji-data/emojilist.json';
 
 export const emojilist: UnicodeEmojiDef[] = _emojilist.map(x => ({
 	name: x[1] as string,

packages/frontend-shared/package.json

@@ -19,6 +19,7 @@
 		"js-built"
 	],
 	"dependencies": {
+		"@misskey-dev/emoji-data": "17.0.3",
 		"i18n": "workspace:*",
 		"json5": "2.2.3",
 		"misskey-js": "workspace:*",

packages/frontend/.storybook/changes.ts

@@ -51,7 +51,6 @@ await fs.readFile(
 	if (
 		micromatch(Array.from(modules), [
 			'../../assets/**',
-			'../../fluent-emojis/**',
 			'../../locales/ja-JP.yml',
 			'assets/**',
 			'public/**',

packages/frontend/.storybook/mocks.ts

@@ -7,7 +7,7 @@ import { type SharedOptions, http, HttpResponse } from 'msw';
 
 export const onUnhandledRequest = ((req, print) => {
 	const url = new URL(req.url);
-	if (url.hostname !== 'localhost' || /^\/(?:client-assets\/|fluent-emojis?\/|iframe.html$|node_modules\/|src\/|sb-|static-assets\/|vite\/)/.test(url.pathname)) {
+	if (url.hostname !== 'localhost' || /^\/(?:client-assets\/|fluent-emoji\/|iframe.html$|node_modules\/|src\/|sb-|static-assets\/|vite\/)/.test(url.pathname)) {
 		return
 	}
 	print.warning()
@@ -16,16 +16,7 @@ export const onUnhandledRequest = ((req, print) => {
 export const commonHandlers = [
 	http.get('/fluent-emoji/:codepoints.png', async ({ params }) => {
 		const { codepoints } = params;
-		const value = await fetch(`https://raw.githubusercontent.com/misskey-dev/emojis/main/dist/${codepoints}.png`).then((response) => response.blob());
-		return new HttpResponse(value, {
-			headers: {
-				'Content-Type': 'image/png',
-			},
-		});
-	}),
-	http.get('/fluent-emojis/:codepoints.png', async ({ params }) => {
-		const { codepoints } = params;
-		const value = await fetch(`https://raw.githubusercontent.com/misskey-dev/emojis/main/dist/${codepoints}.png`).then((response) => response.blob());
+		const value = await fetch(`https://unpkg.com/@misskey-dev/emoji-assets@17.0.3/built/fluent-emoji/${codepoints}.png`).then((response) => response.blob());
 		return new HttpResponse(value, {
 			headers: {
 				'Content-Type': 'image/png',
@@ -34,7 +25,7 @@ export const commonHandlers = [
 	}),
 	http.get('/twemoji/:codepoints.svg', async ({ params }) => {
 		const { codepoints } = params;
-		const value = await fetch(`https://unpkg.com/@discordapp/twemoji@16.0.1/dist/svg/${codepoints}.svg`).then((response) => response.blob());
+		const value = await fetch(`https://unpkg.com/@misskey-dev/emoji-assets@17.0.3/built/twemoji/${codepoints}.svg`).then((response) => response.blob());
 		return new HttpResponse(value, {
 			headers: {
 				'Content-Type': 'image/svg+xml',

packages/frontend/assets/room/object-thumbs/README.md

@@ -1 +0,0 @@
-これらのサムネイルはdev buildでRoomのカタログダイアログを表示し家具を選択した状態でブラウザのコンソールで`takeScreenshot();`を叩くと生成・ダウンロードできます