perf(server): reduce db query

This commit is contained in:
syuilo 2022-03-21 21:10:41 +09:00
parent 8500fb86c7
commit e8037b7e7d
9 changed files with 36 additions and 34 deletions

View File

@ -262,3 +262,5 @@ export type CacheableRemoteUser = {
featured: IRemoteUser['featured']; featured: IRemoteUser['featured'];
followersUri: IRemoteUser['followersUri']; followersUri: IRemoteUser['followersUri'];
}; };
export type CacheableUser = CacheableLocalUser | CacheableRemoteUser;

View File

@ -3,14 +3,14 @@ import Resolver from './resolver.js';
import { resolvePerson } from './models/person.js'; import { resolvePerson } from './models/person.js';
import { unique, concat } from '@/prelude/array.js'; import { unique, concat } from '@/prelude/array.js';
import promiseLimit from 'promise-limit'; import promiseLimit from 'promise-limit';
import { User, CacheableRemoteUser } from '@/models/entities/user.js'; import { User, CacheableRemoteUser, CacheableUser } from '@/models/entities/user.js';
type Visibility = 'public' | 'home' | 'followers' | 'specified'; type Visibility = 'public' | 'home' | 'followers' | 'specified';
type AudienceInfo = { type AudienceInfo = {
visibility: Visibility, visibility: Visibility,
mentionedUsers: User[], mentionedUsers: CacheableUser[],
visibleUsers: User[], visibleUsers: CacheableUser[],
}; };
export async function parseAudience(actor: CacheableRemoteUser, to?: ApObject, cc?: ApObject, resolver?: Resolver): Promise<AudienceInfo> { export async function parseAudience(actor: CacheableRemoteUser, to?: ApObject, cc?: ApObject, resolver?: Resolver): Promise<AudienceInfo> {
@ -19,10 +19,10 @@ export async function parseAudience(actor: CacheableRemoteUser, to?: ApObject, c
const others = unique(concat([toGroups.other, ccGroups.other])); const others = unique(concat([toGroups.other, ccGroups.other]));
const limit = promiseLimit<User | null>(2); const limit = promiseLimit<CacheableUser | null>(2);
const mentionedUsers = (await Promise.all( const mentionedUsers = (await Promise.all(
others.map(id => limit(() => resolvePerson(id, resolver).catch(() => null))) others.map(id => limit(() => resolvePerson(id, resolver).catch(() => null)))
)).filter((x): x is User => x != null); )).filter((x): x is CacheableUser => x != null);
if (toGroups.public.length > 0) { if (toGroups.public.length > 0) {
return { return {

View File

@ -109,10 +109,11 @@ export default class DbResolver {
user: CacheableRemoteUser; user: CacheableRemoteUser;
key?: UserPublickey; key?: UserPublickey;
} | null> { } | null> {
const user = await resolvePerson(uri) as IRemoteUser; const user = await resolvePerson(uri) as CacheableRemoteUser;
if (user == null) return null; if (user == null) return null;
// TODO: cache
const key = await UserPublickeys.findOne(user.id); const key = await UserPublickeys.findOne(user.id);
return { return {

View File

@ -1,10 +1,10 @@
import { uploadFromUrl } from '@/services/drive/upload-from-url.js'; import { uploadFromUrl } from '@/services/drive/upload-from-url.js';
import { IRemoteUser } from '@/models/entities/user.js'; import { CacheableRemoteUser, IRemoteUser } from '@/models/entities/user.js';
import Resolver from '../resolver.js'; import Resolver from '../resolver.js';
import { fetchMeta } from '@/misc/fetch-meta.js'; import { fetchMeta } from '@/misc/fetch-meta.js';
import { apLogger } from '../logger.js'; import { apLogger } from '../logger.js';
import { DriveFile } from '@/models/entities/drive-file.js'; import { DriveFile } from '@/models/entities/drive-file.js';
import { DriveFiles } from '@/models/index.js'; import { DriveFiles, Users } from '@/models/index.js';
import { truncate } from '@/misc/truncate.js'; import { truncate } from '@/misc/truncate.js';
import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js'; import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/misc/hard-limits.js';
@ -13,9 +13,9 @@ const logger = apLogger;
/** /**
* Imageを作成します * Imageを作成します
*/ */
export async function createImage(actor: IRemoteUser, value: any): Promise<DriveFile> { export async function createImage(actor: CacheableRemoteUser, value: any): Promise<DriveFile> {
// 投稿者が凍結されていたらスキップ // 投稿者が凍結されていたらスキップ
if (actor.isSuspended) { if (Users.checkSuspended(actor.id)) {
throw new Error('actor has been suspended'); throw new Error('actor has been suspended');
} }
@ -60,7 +60,7 @@ export async function createImage(actor: IRemoteUser, value: any): Promise<Drive
* Misskeyに対象のImageが登録されていればそれを返し * Misskeyに対象のImageが登録されていればそれを返し
* Misskeyに登録しそれを返します * Misskeyに登録しそれを返します
*/ */
export async function resolveImage(actor: IRemoteUser, value: any): Promise<DriveFile> { export async function resolveImage(actor: CacheableRemoteUser, value: any): Promise<DriveFile> {
// TODO // TODO
// リモートサーバーからフェッチしてきて登録 // リモートサーバーからフェッチしてきて登録

View File

@ -3,17 +3,17 @@ import { IObject, isMention, IApMention } from '../type.js';
import { resolvePerson } from './person.js'; import { resolvePerson } from './person.js';
import promiseLimit from 'promise-limit'; import promiseLimit from 'promise-limit';
import Resolver from '../resolver.js'; import Resolver from '../resolver.js';
import { User } from '@/models/entities/user.js'; import { CacheableUser, User } from '@/models/entities/user.js';
export async function extractApMentions(tags: IObject | IObject[] | null | undefined) { export async function extractApMentions(tags: IObject | IObject[] | null | undefined) {
const hrefs = unique(extractApMentionObjects(tags).map(x => x.href as string)); const hrefs = unique(extractApMentionObjects(tags).map(x => x.href as string));
const resolver = new Resolver(); const resolver = new Resolver();
const limit = promiseLimit<User | null>(2); const limit = promiseLimit<CacheableUser | null>(2);
const mentionedUsers = (await Promise.all( const mentionedUsers = (await Promise.all(
hrefs.map(x => limit(() => resolvePerson(x, resolver).catch(() => null))) hrefs.map(x => limit(() => resolvePerson(x, resolver).catch(() => null)))
)).filter((x): x is User => x != null); )).filter((x): x is CacheableUser => x != null);
return mentionedUsers; return mentionedUsers;
} }

View File

@ -5,7 +5,7 @@ import Resolver from '../resolver.js';
import post from '@/services/note/create.js'; import post from '@/services/note/create.js';
import { resolvePerson, updatePerson } from './person.js'; import { resolvePerson, updatePerson } from './person.js';
import { resolveImage } from './image.js'; import { resolveImage } from './image.js';
import { IRemoteUser } from '@/models/entities/user.js'; import { CacheableRemoteUser, IRemoteUser } from '@/models/entities/user.js';
import { htmlToMfm } from '../misc/html-to-mfm.js'; import { htmlToMfm } from '../misc/html-to-mfm.js';
import { extractApHashtags } from './tag.js'; import { extractApHashtags } from './tag.js';
import { unique, toArray, toSingle } from '@/prelude/array.js'; import { unique, toArray, toSingle } from '@/prelude/array.js';
@ -15,7 +15,7 @@ import { apLogger } from '../logger.js';
import { DriveFile } from '@/models/entities/drive-file.js'; import { DriveFile } from '@/models/entities/drive-file.js';
import { deliverQuestionUpdate } from '@/services/note/polls/update.js'; import { deliverQuestionUpdate } from '@/services/note/polls/update.js';
import { extractDbHost, toPuny } from '@/misc/convert-host.js'; import { extractDbHost, toPuny } from '@/misc/convert-host.js';
import { Emojis, Polls, MessagingMessages } from '@/models/index.js'; import { Emojis, Polls, MessagingMessages, Users } from '@/models/index.js';
import { Note } from '@/models/entities/note.js'; import { Note } from '@/models/entities/note.js';
import { IObject, getOneApId, getApId, getOneApHrefNullable, validPost, IPost, isEmoji, getApType } from '../type.js'; import { IObject, getOneApId, getApId, getOneApHrefNullable, validPost, IPost, isEmoji, getApType } from '../type.js';
import { Emoji } from '@/models/entities/emoji.js'; import { Emoji } from '@/models/entities/emoji.js';
@ -90,10 +90,10 @@ export async function createNote(value: string | IObject, resolver?: Resolver, s
logger.info(`Creating the Note: ${note.id}`); logger.info(`Creating the Note: ${note.id}`);
// 投稿者をフェッチ // 投稿者をフェッチ
const actor = await resolvePerson(getOneApId(note.attributedTo), resolver) as IRemoteUser; const actor = await resolvePerson(getOneApId(note.attributedTo), resolver) as CacheableRemoteUser;
// 投稿者が凍結されていたらスキップ // 投稿者が凍結されていたらスキップ
if (actor.isSuspended) { if (Users.checkSuspended(actor.id)) {
throw new Error('actor has been suspended'); throw new Error('actor has been suspended');
} }
@ -230,11 +230,6 @@ export async function createNote(value: string | IObject, resolver?: Resolver, s
const poll = await extractPollFromQuestion(note, resolver).catch(() => undefined); const poll = await extractPollFromQuestion(note, resolver).catch(() => undefined);
// ユーザーの情報が古かったらついでに更新しておく
if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
if (actor.uri) updatePerson(actor.uri);
}
if (isTalk) { if (isTalk) {
for (const recipient of visibleUsers) { for (const recipient of visibleUsers) {
await createMessage(actor, recipient, undefined, text || undefined, (files && files.length > 0) ? files[0] : null, object.id); await createMessage(actor, recipient, undefined, text || undefined, (files && files.length > 0) ? files[0] : null, object.id);

View File

@ -15,7 +15,7 @@ import { apLogger } from '../logger.js';
import { Note } from '@/models/entities/note.js'; import { Note } from '@/models/entities/note.js';
import { updateUsertags } from '@/services/update-hashtag.js'; import { updateUsertags } from '@/services/update-hashtag.js';
import { Users, Instances, DriveFiles, Followings, UserProfiles, UserPublickeys } from '@/models/index.js'; import { Users, Instances, DriveFiles, Followings, UserProfiles, UserPublickeys } from '@/models/index.js';
import { User, IRemoteUser } from '@/models/entities/user.js'; import { User, IRemoteUser, CacheableUser } from '@/models/entities/user.js';
import { Emoji } from '@/models/entities/emoji.js'; import { Emoji } from '@/models/entities/emoji.js';
import { UserNotePining } from '@/models/entities/user-note-pining.js'; import { UserNotePining } from '@/models/entities/user-note-pining.js';
import { genId } from '@/misc/gen-id.js'; import { genId } from '@/misc/gen-id.js';
@ -32,7 +32,7 @@ import { truncate } from '@/misc/truncate.js';
import { StatusError } from '@/misc/fetch.js'; import { StatusError } from '@/misc/fetch.js';
import { Cache } from '@/misc/cache.js'; import { Cache } from '@/misc/cache.js';
const uriPersonCache = new Cache<User | null>(Infinity); const uriPersonCache = new Cache<CacheableUser | null>(Infinity);
const logger = apLogger; const logger = apLogger;
@ -93,22 +93,26 @@ function validateActor(x: IObject, uri: string): IActor {
* Personをフェッチします * Personをフェッチします
* *
* Misskeyに対象のPersonが登録されていればそれを返します * Misskeyに対象のPersonが登録されていればそれを返します
*
* TODO: cache
*/ */
export async function fetchPerson(uri: string, resolver?: Resolver): Promise<User | null> { export async function fetchPerson(uri: string, resolver?: Resolver): Promise<CacheableUser | null> {
if (typeof uri !== 'string') throw new Error('uri is not string'); if (typeof uri !== 'string') throw new Error('uri is not string');
const cached = uriPersonCache.get(uri);
if (cached) return cached;
// URIがこのサーバーを指しているならデータベースからフェッチ // URIがこのサーバーを指しているならデータベースからフェッチ
if (uri.startsWith(config.url + '/')) { if (uri.startsWith(config.url + '/')) {
const id = uri.split('/').pop(); const id = uri.split('/').pop();
return await Users.findOne(id).then(x => x || null); const u = await Users.findOne(id).then(x => x || null); // TODO: typeorm 3.0 にしたら .then(x => x || null) を消す
if (u) uriPersonCache.set(uri, u);
return u;
} }
//#region このサーバーに既に登録されていたらそれを返す //#region このサーバーに既に登録されていたらそれを返す
const exist = await Users.findOne({ uri }); const exist = await Users.findOne({ uri });
if (exist) { if (exist) {
uriPersonCache.set(uri, exist);
return exist; return exist;
} }
//#endregion //#endregion
@ -376,7 +380,7 @@ export async function updatePerson(uri: string, resolver?: Resolver | null, hint
* Misskeyに対象のPersonが登録されていればそれを返し * Misskeyに対象のPersonが登録されていればそれを返し
* Misskeyに登録しそれを返します * Misskeyに登録しそれを返します
*/ */
export async function resolvePerson(uri: string, resolver?: Resolver): Promise<User> { export async function resolvePerson(uri: string, resolver?: Resolver): Promise<CacheableUser> {
if (typeof uri !== 'string') throw new Error('uri is not string'); if (typeof uri !== 'string') throw new Error('uri is not string');
//#region このサーバーに既に登録されていたらそれを返す //#region このサーバーに既に登録されていたらそれを返す

View File

@ -1,4 +1,4 @@
import { User } from '@/models/entities/user.js'; import { CacheableUser, User } from '@/models/entities/user.js';
import { UserGroup } from '@/models/entities/user-group.js'; import { UserGroup } from '@/models/entities/user-group.js';
import { DriveFile } from '@/models/entities/drive-file.js'; import { DriveFile } from '@/models/entities/drive-file.js';
import { MessagingMessages, UserGroupJoinings, Mutings, Users } from '@/models/index.js'; import { MessagingMessages, UserGroupJoinings, Mutings, Users } from '@/models/index.js';
@ -13,7 +13,7 @@ import renderCreate from '@/remote/activitypub/renderer/create.js';
import { renderActivity } from '@/remote/activitypub/renderer/index.js'; import { renderActivity } from '@/remote/activitypub/renderer/index.js';
import { deliver } from '@/queue/index.js'; import { deliver } from '@/queue/index.js';
export async function createMessage(user: { id: User['id']; host: User['host']; }, recipientUser: User | undefined, recipientGroup: UserGroup | undefined, text: string | null | undefined, file: DriveFile | null, uri?: string) { export async function createMessage(user: { id: User['id']; host: User['host']; }, recipientUser: CacheableUser | undefined, recipientGroup: UserGroup | undefined, text: string | null | undefined, file: DriveFile | null, uri?: string) {
const message = { const message = {
id: genId(), id: genId(),
createdAt: new Date(), createdAt: new Date(),

View File

@ -1,12 +1,12 @@
import { publishNoteStream } from '@/services/stream.js'; import { publishNoteStream } from '@/services/stream.js';
import { User } from '@/models/entities/user.js'; import { CacheableUser, User } from '@/models/entities/user.js';
import { Note } from '@/models/entities/note.js'; import { Note } from '@/models/entities/note.js';
import { PollVotes, NoteWatchings, Polls, Blockings } from '@/models/index.js'; import { PollVotes, NoteWatchings, Polls, Blockings } from '@/models/index.js';
import { Not } from 'typeorm'; import { Not } from 'typeorm';
import { genId } from '@/misc/gen-id.js'; import { genId } from '@/misc/gen-id.js';
import { createNotification } from '../../create-notification.js'; import { createNotification } from '../../create-notification.js';
export default async function(user: User, note: Note, choice: number) { export default async function(user: CacheableUser, note: Note, choice: number) {
const poll = await Polls.findOne(note.id); const poll = await Polls.findOne(note.id);
if (poll == null) throw new Error('poll not found'); if (poll == null) throw new Error('poll not found');