From 0a64d121d9a1345e4a5dd71686fd3be400558595 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 7 Feb 2021 10:43:34 +0900 Subject: [PATCH] Resolve #7149 --- src/misc/normalize-for-search.ts | 6 ++++++ src/remote/activitypub/models/person.ts | 5 +++-- src/server/api/endpoints/hashtags/show.ts | 3 ++- src/server/api/endpoints/hashtags/trend.ts | 3 ++- src/server/api/endpoints/hashtags/users.ts | 3 ++- src/server/api/endpoints/i/update.ts | 3 ++- src/server/api/endpoints/notes/search-by-tag.ts | 5 +++-- src/server/api/stream/channels/hashtag.ts | 3 ++- src/services/note/create.ts | 5 +++-- src/services/update-hashtag.ts | 3 ++- 10 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 src/misc/normalize-for-search.ts diff --git a/src/misc/normalize-for-search.ts b/src/misc/normalize-for-search.ts new file mode 100644 index 000000000..200540566 --- /dev/null +++ b/src/misc/normalize-for-search.ts @@ -0,0 +1,6 @@ +export function normalizeForSearch(tag: string): string { + // ref. + // - https://analytics-note.xyz/programming/unicode-normalization-forms/ + // - https://maku77.github.io/js/string/normalize.html + return tag.normalize('NFKC').toLowerCase(); +} diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index 8f1483ab4..73a2ebc02 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -27,6 +27,7 @@ import { getConnection } from 'typeorm'; import { ensure } from '../../../prelude/ensure'; import { toArray } from '../../../prelude/array'; import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata'; +import { normalizeForSearch } from '../../../misc/normalize-for-search'; const logger = apLogger; @@ -134,7 +135,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise tag.toLowerCase()).splice(0, 32); + const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32); const isBot = object.type === 'Service'; @@ -323,7 +324,7 @@ export async function updatePerson(uri: string, resolver?: Resolver | null, hint const { fields } = analyzeAttachments(person.attachment || []); - const tags = extractApHashtags(person.tag).map(tag => tag.toLowerCase()).splice(0, 32); + const tags = extractApHashtags(person.tag).map(tag => normalizeForSearch(tag)).splice(0, 32); const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); diff --git a/src/server/api/endpoints/hashtags/show.ts b/src/server/api/endpoints/hashtags/show.ts index 9462342aa..49aa36e05 100644 --- a/src/server/api/endpoints/hashtags/show.ts +++ b/src/server/api/endpoints/hashtags/show.ts @@ -2,6 +2,7 @@ import $ from 'cafy'; import define from '../../define'; import { ApiError } from '../../error'; import { Hashtags } from '../../../../models'; +import { normalizeForSearch } from '../../../../misc/normalize-for-search'; export const meta = { desc: { @@ -38,7 +39,7 @@ export const meta = { }; export default define(meta, async (ps, user) => { - const hashtag = await Hashtags.findOne({ name: ps.tag.toLowerCase() }); + const hashtag = await Hashtags.findOne({ name: normalizeForSearch(ps.tag) }); if (hashtag == null) { throw new ApiError(meta.errors.noSuchHashtag); } diff --git a/src/server/api/endpoints/hashtags/trend.ts b/src/server/api/endpoints/hashtags/trend.ts index cfa97d147..3b5dd3c0c 100644 --- a/src/server/api/endpoints/hashtags/trend.ts +++ b/src/server/api/endpoints/hashtags/trend.ts @@ -4,6 +4,7 @@ import { fetchMeta } from '../../../../misc/fetch-meta'; import { Notes } from '../../../../models'; import { Note } from '../../../../models/entities/note'; import { safeForSql } from '../../../../misc/safe-for-sql'; +import { normalizeForSearch } from '../../../../misc/normalize-for-search'; /* トレンドに載るためには「『直近a分間のユニーク投稿数が今からa分前～今からb分前の間のユニーク投稿数のn倍以上』のハッシュタグの上位5位以内に入る」ことが必要 @@ -54,7 +55,7 @@ export const meta = { export default define(meta, async () => { const instance = await fetchMeta(true); - const hiddenTags = instance.hiddenTags.map(t => t.toLowerCase()); + const hiddenTags = instance.hiddenTags.map(t => normalizeForSearch(t)); const now = new Date(); // 5分単位で丸めた現在日時 now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0); diff --git a/src/server/api/endpoints/hashtags/users.ts b/src/server/api/endpoints/hashtags/users.ts index 532a490d9..d2f599868 100644 --- a/src/server/api/endpoints/hashtags/users.ts +++ b/src/server/api/endpoints/hashtags/users.ts @@ -1,6 +1,7 @@ import $ from 'cafy'; import define from '../../define'; import { Users } from '../../../../models'; +import { normalizeForSearch } from '../../../../misc/normalize-for-search'; export const meta = { requireCredential: false as const, @@ -59,7 +60,7 @@ export const meta = { export default define(meta, async (ps, me) => { const query = Users.createQueryBuilder('user') - .where(':tag = ANY(user.tags)', { tag: ps.tag.toLowerCase() }); + .where(':tag = ANY(user.tags)', { tag: normalizeForSearch(ps.tag) }); const recent = new Date(Date.now() - (1000 * 60 * 60 * 24 * 5)); diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts index 6552f9b76..e4c0e8cec 100644 --- a/src/server/api/endpoints/i/update.ts +++ b/src/server/api/endpoints/i/update.ts @@ -15,6 +15,7 @@ import { User } from '../../../../models/entities/user'; import { UserProfile } from '../../../../models/entities/user-profile'; import { ensure } from '../../../../prelude/ensure'; import { notificationTypes } from '../../../../types'; +import { normalizeForSearch } from '../../../../misc/normalize-for-search'; export const meta = { desc: { @@ -286,7 +287,7 @@ export default define(meta, async (ps, user, token) => { if (newDescription != null) { const tokens = parse(newDescription); emojis = emojis.concat(extractEmojis(tokens!)); - tags = extractHashtags(tokens!).map(tag => tag.toLowerCase()).splice(0, 32); + tags = extractHashtags(tokens!).map(tag => normalizeForSearch(tag)).splice(0, 32); } updates.emojis = emojis; diff --git a/src/server/api/endpoints/notes/search-by-tag.ts b/src/server/api/endpoints/notes/search-by-tag.ts index 446beb32d..e0f7f4d62 100644 --- a/src/server/api/endpoints/notes/search-by-tag.ts +++ b/src/server/api/endpoints/notes/search-by-tag.ts @@ -7,6 +7,7 @@ import { generateMutedUserQuery } from '../../common/generate-muted-user-query'; import { generateVisibilityQuery } from '../../common/generate-visibility-query'; import { Brackets } from 'typeorm'; import { safeForSql } from '../../../../misc/safe-for-sql'; +import { normalizeForSearch } from '../../../../misc/normalize-for-search'; export const meta = { desc: { @@ -101,7 +102,7 @@ export default define(meta, async (ps, me) => { if (ps.tag) { if (!safeForSql(ps.tag)) return; - query.andWhere(`'{"${ps.tag.toLowerCase()}"}' <@ note.tags`); + query.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`); } else { let i = 0; query.andWhere(new Brackets(qb => { @@ -109,7 +110,7 @@ export default define(meta, async (ps, me) => { qb.orWhere(new Brackets(qb => { for (const tag of tags) { if (!safeForSql(tag)) return; - qb.andWhere(`'{"${tag.toLowerCase()}"}' <@ note.tags`); + qb.andWhere(`'{"${normalizeForSearch(ps.tag)}"}' <@ note.tags`); i++; } })); diff --git a/src/server/api/stream/channels/hashtag.ts b/src/server/api/stream/channels/hashtag.ts index 32d8111f7..41447039d 100644 --- a/src/server/api/stream/channels/hashtag.ts +++ b/src/server/api/stream/channels/hashtag.ts @@ -3,6 +3,7 @@ import { isMutedUserRelated } from '../../../../misc/is-muted-user-related'; import Channel from '../channel'; import { Notes } from '../../../../models'; import { PackedNote } from '../../../../models/repositories/note'; +import { normalizeForSearch } from '../../../../misc/normalize-for-search'; export default class extends Channel { public readonly chName = 'hashtag'; @@ -23,7 +24,7 @@ export default class extends Channel { @autobind private async onNote(note: PackedNote) { const noteTags = note.tags ? note.tags.map((t: string) => t.toLowerCase()) : []; - const matched = this.q.some(tags => tags.every(tag => noteTags.includes(tag.toLowerCase()))); + const matched = this.q.some(tags => tags.every(tag => noteTags.includes(normalizeForSearch(tag)))); if (!matched) return; // Renoteなら再pack diff --git a/src/services/note/create.ts b/src/services/note/create.ts index f6593996e..62ec92f2b 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -33,6 +33,7 @@ import { addNoteToAntenna } from '../add-note-to-antenna'; import { countSameRenotes } from '../../misc/count-same-renotes'; import { deliverToRelays } from '../relay'; import { Channel } from '../../models/entities/channel'; +import { normalizeForSearch } from '../../misc/normalize-for-search'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -460,7 +461,7 @@ async function insertNote(user: User, data: Option, tags: string[], emojis: stri text: data.text, hasPoll: data.poll != null, cw: data.cw == null ? null : data.cw, - tags: tags.map(tag => tag.toLowerCase()), + tags: tags.map(tag => normalizeForSearch(tag)), emojis, userId: user.id, viaMobile: data.viaMobile!, @@ -547,7 +548,7 @@ function index(note: Note) { index: config.elasticsearch.index || 'misskey_note', id: note.id.toString(), body: { - text: note.text.toLowerCase(), + text: normalizeForSearch(note.text), userId: note.userId, userHost: note.userHost } diff --git a/src/services/update-hashtag.ts b/src/services/update-hashtag.ts index 1c67ef881..1dcb58279 100644 --- a/src/services/update-hashtag.ts +++ b/src/services/update-hashtag.ts @@ -3,6 +3,7 @@ import { Hashtags, Users } from '../models'; import { hashtagChart } from './chart'; import { genId } from '../misc/gen-id'; import { Hashtag } from '../models/entities/hashtag'; +import { normalizeForSearch } from '../misc/normalize-for-search'; export async function updateHashtags(user: User, tags: string[]) { for (const tag of tags) { @@ -21,7 +22,7 @@ export async function updateUsertags(user: User, tags: string[]) { } export async function updateHashtag(user: User, tag: string, isUserAttached = false, inc = true) { - tag = tag.toLowerCase(); + tag = normalizeForSearch(tag); const index = await Hashtags.findOne({ name: tag });