enhance: Proxy custom emojis to reduce image size and accelerate the frontend (#9431)

* fix(server): /emoji to accept `@.` host expression

* fix(client): use MkEmoji for custom emoji in MkEmojiPicker

* change convertToWebp

* nanka iroiro

* remove

* fix

* nearLosslessは労多くして益少なしなのでやめる

* do not cleanup tmp for development

* update sharp.js to 0.31.3

* mixed: true

* fix MkAutocomplete of 912791b3ab

* clean up

* https://github.com/misskey-dev/misskey/pull/9431#discussion_r1059215943

packages/backend/package.json

@@ -105,7 +105,7 @@
 		"sanitize-html": "2.8.1",
 		"seedrandom": "^3.0.5",
 		"semver": "7.3.8",
-		"sharp": "0.29.3",
+		"sharp": "0.31.3",
 		"speakeasy": "2.0.0",
 		"strict-event-emitter-types": "2.0.0",
 		"stringz": "2.1.0",

packages/backend/src/core/CustomEmojiService.ts

@@ -2,12 +2,10 @@ import { Inject, Injectable } from '@nestjs/common';
 import { DataSource, In, IsNull } from 'typeorm';
 import { GlobalEventService } from '@/core/GlobalEventService.js';
 import { DI } from '@/di-symbols.js';
-import type { Config } from '@/config.js';
 import { IdService } from '@/core/IdService.js';
 import type { DriveFile } from '@/models/entities/DriveFile.js';
 import type { Emoji } from '@/models/entities/Emoji.js';
 import { Cache } from '@/misc/cache.js';
-import { query } from '@/misc/prelude/url.js';
 import type { Note } from '@/models/entities/Note.js';
 import type { EmojisRepository } from '@/models/index.js';
 import { UtilityService } from '@/core/UtilityService.js';
@@ -27,9 +25,6 @@ export class CustomEmojiService {
 	private cache: Cache<Emoji | null>;
 
 	constructor(
-		@Inject(DI.config)
-		private config: Config,
-
 		@Inject(DI.db)
 		private db: DataSource,
 
@@ -117,7 +112,7 @@ export class CustomEmojiService {
 
 		const isLocal = emoji.host == null;
 		const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため
-		const url = isLocal ? emojiUrl : `${this.config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`;
+		const url = emojiUrl;
 
 		return {
 			name: emojiName,

packages/backend/src/core/DownloadService.ts

@@ -33,7 +33,7 @@ export class DownloadService {
 
 	@bindThis
 	public async downloadUrl(url: string, path: string): Promise<void> {
-		this.logger.info(`Downloading ${chalk.cyan(url)} ...`);
+		this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
 	
 		const timeout = 30 * 1000;
 		const operationTimeout = 60 * 1000;

packages/backend/src/core/ImageProcessingService.ts

@@ -8,6 +8,16 @@ export type IImage = {
 	ext: string | null;
 	type: string;
 };
+
+export const webpDefault: sharp.WebpOptions = {
+	quality: 85,
+	alphaQuality: 95,
+	lossless: false,
+	nearLossless: false,
+	smartSubsample: true,
+	mixed: true,
+};
+
 import { bindThis } from '@/decorators.js';
 
 @Injectable()
@@ -53,21 +63,19 @@ export class ImageProcessingService {
 	 *   with resize, remove metadata, resolve orientation, stop animation
 	 */
 	@bindThis
-	public async convertToWebp(path: string, width: number, height: number, quality = 85): Promise<IImage> {
-		return this.convertSharpToWebp(await sharp(path), width, height, quality);
+	public async convertToWebp(path: string, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
+		return this.convertSharpToWebp(await sharp(path), width, height, options);
 	}
 
 	@bindThis
-	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, quality = 85): Promise<IImage> {
+	public async convertSharpToWebp(sharp: sharp.Sharp, width: number, height: number, options: sharp.WebpOptions = webpDefault): Promise<IImage> {
 		const data = await sharp
 			.resize(width, height, {
 				fit: 'inside',
 				withoutEnlargement: true,
 			})
 			.rotate()
-			.webp({
-				quality,
-			})
+			.webp(options)
 			.toBuffer();
 
 		return {

packages/backend/src/misc/create-temp.ts

@@ -4,7 +4,7 @@ export function createTemp(): Promise<[string, () => void]> {
 	return new Promise<[string, () => void]>((res, rej) => {
 		tmp.file((e, path, fd, cleanup) => {
 			if (e) return rej(e);
-			res([path, cleanup]);
+			res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
 		});
 	});
 }
@@ -17,7 +17,7 @@ export function createTempDir(): Promise<[string, () => void]> {
 			},
 			(e, path, cleanup) => {
 				if (e) return rej(e);
-				res([path, cleanup]);
+				res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]);
 			},
 		);
 	});

packages/backend/src/misc/prelude/url.ts

@@ -1,3 +1,8 @@
+/* objを検査して
+ * 1. 配列に何も入っていない時はクエリを付けない
+ * 2. プロパティがundefinedの時はクエリを付けない
+ * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない）
+ */ 
 export function query(obj: Record<string, unknown>): string {
 	const params = Object.entries(obj)
 		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

packages/backend/src/server/MediaProxyServerService.ts

@@ -9,7 +9,7 @@ import type { Config } from '@/config.js';
 import { isMimeImage } from '@/misc/is-mime-image.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { DownloadService } from '@/core/DownloadService.js';
-import { ImageProcessingService } from '@/core/ImageProcessingService.js';
+import { ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js';
 import type { IImage } from '@/core/ImageProcessingService.js';
 import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
 import { StatusError } from '@/misc/status-error.js';
@@ -81,8 +81,21 @@ export class MediaProxyServerService {
 			const isConvertibleImage = isMimeImage(mime, 'sharp-convertible-image');
 	
 			let image: IImage;
+			if ('emoji' in request.query && isConvertibleImage) {
+				const data = await sharp(path, { animated: !('static' in request.query) })
+					.resize({
+						height: 128,
+						withoutEnlargement: true,
+					})
+					.webp(webpDefault)
+					.toBuffer();
 
-			if ('static' in request.query && isConvertibleImage) {
+				image = {
+					data,
+					ext: 'webp',
+					type: 'image/webp',
+				};
+			} else if ('static' in request.query && isConvertibleImage) {
 				image = await this.imageProcessingService.convertToWebp(path, 498, 280);
 			} else if ('preview' in request.query && isConvertibleImage) {
 				image = await this.imageProcessingService.convertToWebp(path, 200, 200);
@@ -122,7 +135,7 @@ export class MediaProxyServerService {
 					type: 'image/png',
 				};
 			} else if (mime === 'image/svg+xml') {
-				image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, 1);
+				image = await this.imageProcessingService.convertToWebp(path, 2048, 2048, webpDefault);
 			} else if (!mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(mime)) {
 				throw new StatusError('Rejected type', 403, 'Rejected type');
 			} else {

packages/backend/src/server/web/ClientServerService.ts

@@ -220,7 +220,7 @@ export class ClientServerService {
 			return reply.sendFile('/apple-touch-icon.png', staticAssets);
 		});
 
-		fastify.get<{ Params: { path: string } }>('/emoji/:path(.*)', async (request, reply) => {
+		fastify.get<{ Params: { path: string }; Querystring: { static?: any; }; }>('/emoji/:path(.*)', async (request, reply) => {
 			const path = request.params.path;
 
 			if (!path.match(/^[a-zA-Z0-9\-_@\.]+?\.webp$/)) {
@@ -244,8 +244,15 @@ export class ClientServerService {
 
 			reply.header('Content-Security-Policy', 'default-src \'none\'; style-src \'unsafe-inline\'');
 
-			// ?? emoji.originalUrl してるのは後方互換性のため
-			return await reply.redirect(301, emoji.publicUrl ?? emoji.originalUrl);
+			const url = new URL("/proxy/emoji.webp", this.config.url);
+			url.searchParams.set('url', emoji.publicUrl ?? emoji.originalUrl); // ?? emoji.originalUrl してるのは後方互換性のため
+			url.searchParams.set('emoji', '1');
+			if ('static' in request.query) url.searchParams.set('static', '1');
+
+			return await reply.redirect(
+				301,
+				url.toString(),
+			);
 		});
 
 		fastify.get<{ Params: { path: string } }>('/fluent-emoji/:path(.*)', async (request, reply) => {

packages/frontend/src/components/MkAutocomplete.vue

@@ -16,12 +16,12 @@
 		</li>
 	</ol>
 	<ol v-else-if="emojis.length > 0" ref="suggests" class="emojis">
-		<li v-for="emoji in emojis" tabindex="-1" @click="complete(type, emoji.emoji)" @keydown="onKeydown">
-			<span v-if="emoji.isCustomEmoji" class="emoji"><img :src="`/emoji/${emoji.name}.webp`" :alt="emoji.emoji"/></span>
-			<span v-else-if="defaultStore.state.emojiStyle != 'native'" class="emoji"><img :src="emoji.url" :alt="emoji.emoji"/></span>
-			<span v-else class="emoji">{{ emoji.emoji }}</span>
+		<li v-for="emoji in emojis" tabindex="-1" :key="emoji.emoji" @click="complete(type, emoji.emoji)" @keydown="onKeydown">
+			<div class="emoji">
+				<MkEmoji :emoji="emoji.emoji" />
+			</div>
 			<!-- eslint-disable-next-line vue/no-v-html -->
-			<span class="name" v-html="emoji.name.replace(q, `<b>${q}</b>`)"></span>
+			<span class="name" v-html="emoji.name.replace(q ?? '', `<b>${q}</b>`)"></span>
 			<span v-if="emoji.aliasOf" class="alias">({{ emoji.aliasOf }})</span>
 		</li>
 	</ol>
@@ -37,7 +37,6 @@
 import { markRaw, ref, onUpdated, onMounted, onBeforeUnmount, nextTick, watch } from 'vue';
 import contains from '@/scripts/contains';
 import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
 import { acct } from '@/filters/user';
 import * as os from '@/os';
 import { MFM_TAGS } from '@/scripts/mfm-tags';
@@ -49,9 +48,13 @@ import { i18n } from '@/i18n';
 type EmojiDef = {
 	emoji: string;
 	name: string;
+	url: string;
 	aliasOf?: string;
-	url?: string;
-	isCustomEmoji?: boolean;
+} | {
+	emoji: string;
+	name: string;
+	aliasOf?: string;
+	isCustomEmoji?: true;
 };
 
 const lib = emojilist.filter(x => x.category !== 'flags');
@@ -87,7 +90,6 @@ for (const x of customEmojis) {
 	emojiDefinitions.push({
 		name: x.name,
 		emoji: `:${x.name}:`,
-		url: x.url,
 		isCustomEmoji: true,
 	});
 
@@ -97,7 +99,6 @@ for (const x of customEmojis) {
 				name: alias,
 				aliasOf: x.name,
 				emoji: `:${x.name}:`,
-				url: x.url,
 				isCustomEmoji: true,
 			});
 		}
@@ -452,14 +453,20 @@ onBeforeUnmount(() => {
 	> .emojis > li {
 
 		.emoji {
-			display: inline-block;
+			display: flex;
 			margin: 0 4px 0 0;
+			height: 24px;
 			width: 24px;
+			justify-content: center;
+			align-items: center;
+			font-size: 20px;
 
 			> img {
+				height: 24px;
 				width: 24px;
-				vertical-align: bottom;
+				object-fit: scale-down;
 			}
+
 		}
 
 		.alias {

packages/frontend/src/components/MkEmojiPicker.vue

@@ -81,7 +81,6 @@ import { ref, computed, watch, onMounted } from 'vue';
 import * as Misskey from 'misskey-js';
 import XSection from '@/components/MkEmojiPicker.section.vue';
 import { emojilist, UnicodeEmojiDef, unicodeEmojiCategories as categories } from '@/scripts/emojilist';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
 import Ripple from '@/components/MkRipple.vue';
 import * as os from '@/os';
 import { isTouchUsing } from '@/scripts/touch';

packages/frontend/src/components/MkMediaImage.vue

@@ -23,7 +23,7 @@
 <script lang="ts" setup>
 import { watch } from 'vue';
 import * as misskey from 'misskey-js';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import ImgWithBlurhash from '@/components/MkImgWithBlurhash.vue';
 import { defaultStore } from '@/store';
 

packages/frontend/src/components/global/MkAvatar.vue

@@ -12,7 +12,7 @@
 <script lang="ts" setup>
 import { onMounted, watch } from 'vue';
 import * as misskey from 'misskey-js';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import { extractAvgColorFromBlurhash } from '@/scripts/extract-avg-color-from-blurhash';
 import { acct, userPage } from '@/filters/user';
 import MkUserOnlineIndicator from '@/components/MkUserOnlineIndicator.vue';

packages/frontend/src/components/global/MkEmoji.vue

@@ -7,7 +7,7 @@
 
 <script lang="ts" setup>
 import { computed } from 'vue';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import { char2twemojiFilePath, char2fluentEmojiFilePath } from '@/scripts/emoji-base';
 import { defaultStore } from '@/store';
 import { getEmojiName } from '@/scripts/emojilist';

packages/frontend/src/pages/user/index.photos.vue

@@ -21,7 +21,7 @@
 <script lang="ts" setup>
 import { onMounted } from 'vue';
 import * as misskey from 'misskey-js';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import { notePage } from '@/filters/note';
 import * as os from '@/os';
 import MkContainer from '@/components/MkContainer.vue';

packages/frontend/src/scripts/get-static-image-url.ts

@@ -1,19 +0,0 @@
-import { url as instanceUrl } from '@/config';
-import * as url from '@/scripts/url';
-
-export function getStaticImageUrl(baseUrl: string): string {
-	const u = new URL(baseUrl);
-	if (u.href.startsWith(`${instanceUrl}/proxy/`)) {
-		// もう既にproxyっぽそうだったらsearchParams付けるだけ
-		u.searchParams.set('static', '1');
-		return u.href;
-	}
-
-	// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
-	const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
-
-	return `${instanceUrl}/proxy/${dummy}?${url.query({
-		url: u.href,
-		static: '1',
-	})}`;
-}

packages/frontend/src/scripts/media-proxy.ts

@@ -1,7 +1,15 @@
-import { query } from '@/scripts/url';
+import { query, appendQuery } from '@/scripts/url';
 import { url } from '@/config';
 
 export function getProxiedImageUrl(imageUrl: string, type?: 'preview'): string {
+	if (imageUrl.startsWith(`${url}/proxy/`) || imageUrl.startsWith('/proxy/')) {
+		// もう既にproxyっぽそうだったらsearchParams付けるだけ
+		return appendQuery(imageUrl, query({
+			fallback: '1',
+			...(type ? { [type]: '1' } : {}),
+		}));
+	}
+
 	return `${url}/proxy/image.webp?${query({
 		url: imageUrl,
 		fallback: '1',
@@ -13,3 +21,27 @@ export function getProxiedImageUrlNullable(imageUrl: string | null | undefined,
 	if (imageUrl == null) return null;
 	return getProxiedImageUrl(imageUrl, type);
 }
+
+export function getStaticImageUrl(baseUrl: string): string {
+	const u = baseUrl.startsWith('http') ? new URL(baseUrl) : new URL(baseUrl, url);
+
+	if (u.href.startsWith(`${url}/proxy/`)) {
+		// もう既にproxyっぽそうだったらsearchParams付けるだけ
+		u.searchParams.set('static', '1');
+		return u.href;
+	}
+
+	if (u.href.startsWith(`${url}/emoji/`)) {
+		// もう既にemojiっぽそうだったらsearchParams付けるだけ
+		u.searchParams.set('static', '1');
+		return u.href;
+	}
+
+	// 拡張子がないとキャッシュしてくれないCDNがあるのでダミーの名前を指定する
+	const dummy = `${encodeURIComponent(`${u.host}${u.pathname}`)}.webp`;
+
+	return `${url}/proxy/${dummy}?${query({
+		url: u.href,
+		static: '1',
+	})}`;
+}

packages/frontend/src/scripts/url.ts

@@ -1,3 +1,8 @@
+/* objを検査して
+ * 1. 配列に何も入っていない時はクエリを付けない
+ * 2. プロパティがundefinedの時はクエリを付けない
+ * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない）
+ */ 
 export function query(obj: Record<string, any>): string {
 	const params = Object.entries(obj)
 		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

packages/frontend/src/widgets/photos.vue

@@ -20,7 +20,7 @@ import { onMounted, onUnmounted, reactive, ref } from 'vue';
 import { useWidgetPropsManager, Widget, WidgetComponentEmits, WidgetComponentExpose, WidgetComponentProps } from './widget';
 import { GetFormResultType } from '@/scripts/form';
 import { stream } from '@/stream';
-import { getStaticImageUrl } from '@/scripts/get-static-image-url';
+import { getStaticImageUrl } from '@/scripts/media-proxy';
 import * as os from '@/os';
 import MkContainer from '@/components/MkContainer.vue';
 import { defaultStore } from '@/store';

packages/sw/src/scripts/url.ts

@@ -1,3 +1,8 @@
+/* objを検査して
+ * 1. 配列に何も入っていない時はクエリを付けない
+ * 2. プロパティがundefinedの時はクエリを付けない
+ * （new URLSearchParams(obj)ではそこまで丁寧なことをしてくれない）
+ */ 
 export function query(obj: object): string {
 	const params = Object.entries(obj)
 		.filter(([, v]) => Array.isArray(v) ? v.length : v !== undefined)

yarn.lock

@@ -4257,7 +4257,7 @@ __metadata:
     sanitize-html: 2.8.1
     seedrandom: ^3.0.5
     semver: 7.3.8
-    sharp: 0.29.3
+    sharp: 0.31.3
     speakeasy: 2.0.0
     strict-event-emitter-types: 2.0.0
     stringz: 2.1.0
@@ -5367,7 +5367,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"color@npm:^4.0.1":
+"color@npm:^4.2.3":
   version: 4.2.3
   resolution: "color@npm:4.2.3"
   dependencies:
@@ -6161,16 +6161,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"detect-libc@npm:^1.0.3":
-  version: 1.0.3
-  resolution: "detect-libc@npm:1.0.3"
-  bin:
-    detect-libc: ./bin/detect-libc.js
-  checksum: daaaed925ffa7889bd91d56e9624e6c8033911bb60f3a50a74a87500680652969dbaab9526d1e200a4c94acf80fc862a22131841145a0a8482d60a99c24f4a3e
-  languageName: node
-  linkType: hard
-
-"detect-libc@npm:^2.0.0":
+"detect-libc@npm:^2.0.0, detect-libc@npm:^2.0.1":
   version: 2.0.1
   resolution: "detect-libc@npm:2.0.1"
   checksum: ccb05fcabbb555beb544d48080179c18523a343face9ee4e1a86605a8715b4169f94d663c21a03c310ac824592f2ba9a5270218819bb411ad7be578a527593d7
@@ -12167,12 +12158,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"node-addon-api@npm:^4.2.0":
-  version: 4.3.0
-  resolution: "node-addon-api@npm:4.3.0"
+"node-addon-api@npm:^5.0.0":
+  version: 5.0.0
+  resolution: "node-addon-api@npm:5.0.0"
   dependencies:
     node-gyp: latest
-  checksum: 3de396e23cc209f539c704583e8e99c148850226f6e389a641b92e8967953713228109f919765abc1f4355e801e8f41842f96210b8d61c7dcc10a477002dcf00
+  checksum: 7c5e2043ac37f6108784d94ed73a44ae6d3e68eb968de60680922fc6bc3d17fa69448c0feb4e0c9d3f4c74a0324822e566a8340a56916d9d6f23cb3e85620334
   languageName: node
   linkType: hard
 
@@ -13672,7 +13663,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prebuild-install@npm:^7.0.0":
+"prebuild-install@npm:^7.1.1":
   version: 7.1.1
   resolution: "prebuild-install@npm:7.1.1"
   dependencies:
@@ -15053,7 +15044,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"semver@npm:7.3.8, semver@npm:^7.3.2, semver@npm:^7.3.5, semver@npm:^7.3.6, semver@npm:^7.3.7":
+"semver@npm:7.3.8, semver@npm:^7.3.2, semver@npm:^7.3.5, semver@npm:^7.3.6, semver@npm:^7.3.7, semver@npm:^7.3.8":
   version: 7.3.8
   resolution: "semver@npm:7.3.8"
   dependencies:
@@ -15146,20 +15137,20 @@ __metadata:
   languageName: node
   linkType: hard
 
-"sharp@npm:0.29.3":
-  version: 0.29.3
-  resolution: "sharp@npm:0.29.3"
+"sharp@npm:0.31.3":
+  version: 0.31.3
+  resolution: "sharp@npm:0.31.3"
   dependencies:
-    color: ^4.0.1
-    detect-libc: ^1.0.3
-    node-addon-api: ^4.2.0
+    color: ^4.2.3
+    detect-libc: ^2.0.1
+    node-addon-api: ^5.0.0
     node-gyp: latest
-    prebuild-install: ^7.0.0
-    semver: ^7.3.5
-    simple-get: ^4.0.0
+    prebuild-install: ^7.1.1
+    semver: ^7.3.8
+    simple-get: ^4.0.1
     tar-fs: ^2.1.1
     tunnel-agent: ^0.6.0
-  checksum: d496cdd546c9abe743aebcee013731295f735687819a18c2bdcbba6f31a6b259f3da95af5c11260a8fedc9d4ab95697f5f8c4f3cd65232792b5cfb876bea7c9a
+  checksum: 29fd1dfbc616c6389f53f366cec342b4353d9f2a37e98952ca273db38dca57dfa0f336322d6d763f0fae876042ead22fd86ffe26d70c32ade2458d421db60d04
   languageName: node
   linkType: hard
 
@@ -15204,7 +15195,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"simple-get@npm:^4.0.0":
+"simple-get@npm:^4.0.0, simple-get@npm:^4.0.1":
   version: 4.0.1
   resolution: "simple-get@npm:4.0.1"
   dependencies: