From 9c454e59450fe7850175a5c117d2f337aaa23141 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 20 Apr 2019 01:23:48 +0900 Subject: [PATCH] Revert "Clean up" This reverts commit a1f7b009810cd94c3d43752ae035d4ff123dd4de. --- binding.gyp | 9 + gulpfile.ts | 1 + package.json | 2 + src/crypto_key.cc | 111 ++++++++++ src/crypto_key.d.ts | 2 + src/migrate.ts | 502 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 627 insertions(+) create mode 100644 binding.gyp create mode 100644 src/crypto_key.cc create mode 100644 src/crypto_key.d.ts create mode 100644 src/migrate.ts diff --git a/binding.gyp b/binding.gyp new file mode 100644 index 000000000..0349526d5 --- /dev/null +++ b/binding.gyp @@ -0,0 +1,9 @@ +{ + 'targets': [ + { + 'target_name': 'crypto_key', + 'sources': ['src/crypto_key.cc'], + 'include_dirs': [' gulp.task('build:copy', gulp.parallel('build:copy:views', () => gulp.src([ + './build/Release/crypto_key.node', './src/const.json', './src/server/web/views/**/*', './src/**/assets/**/*', diff --git a/package.json b/package.json index 409bdda15..1584899c2 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "scripts": { "start": "node ./index.js", "init": "node ./built/init.js", + "migrate": "node ./built/migrate.js", "build": "webpack && gulp build", "webpack": "webpack", "watch": "webpack --watch", @@ -166,6 +167,7 @@ "moji": "0.5.1", "moment": "2.24.0", "ms": "2.1.1", + "nan": "2.12.1", "nested-property": "0.0.7", "node-fetch": "2.3.0", "nodemailer": "6.1.0", diff --git a/src/crypto_key.cc b/src/crypto_key.cc new file mode 100644 index 000000000..658586bae --- /dev/null +++ b/src/crypto_key.cc @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +#include +#include + +NAN_METHOD(extractPublic) +{ + const auto sourceString = info[0]->ToString(Nan::GetCurrentContext()).ToLocalChecked(); + if (!sourceString->IsOneByte()) { + Nan::ThrowError("Malformed character found"); + return; + } + + size_t sourceLength = sourceString->Length(); + const auto sourceBuf = new char[sourceLength]; + + Nan::DecodeWrite(sourceBuf, sourceLength, sourceString); + + const auto source = BIO_new_mem_buf(sourceBuf, sourceLength); + if (source == nullptr) { + Nan::ThrowError("Memory allocation failed"); + delete[] sourceBuf; + return; + } + + const auto rsa = PEM_read_bio_RSAPrivateKey(source, nullptr, nullptr, nullptr); + + BIO_free(source); + delete[] sourceBuf; + + if (rsa == nullptr) { + Nan::ThrowError("Decode failed"); + return; + } + + const auto destination = BIO_new(BIO_s_mem()); + if (destination == nullptr) { + Nan::ThrowError("Memory allocation failed"); + return; + } + + const auto result = PEM_write_bio_RSAPublicKey(destination, rsa); + + RSA_free(rsa); + + if (result != 1) { + Nan::ThrowError("Public key extraction failed"); + BIO_free(destination); + return; + } + + char *pem; + const auto pemLength = BIO_get_mem_data(destination, &pem); + + info.GetReturnValue().Set(Nan::Encode(pem, pemLength)); + BIO_free(destination); +} + +NAN_METHOD(generate) +{ + const auto exponent = BN_new(); + const auto mem = BIO_new(BIO_s_mem()); + const auto rsa = RSA_new(); + char *data; + long result; + + if (exponent == nullptr || mem == nullptr || rsa == nullptr) { + Nan::ThrowError("Memory allocation failed"); + goto done; + } + + result = BN_set_word(exponent, 65537); + if (result != 1) { + Nan::ThrowError("Exponent setting failed"); + goto done; + } + + result = RSA_generate_key_ex(rsa, 2048, exponent, nullptr); + if (result != 1) { + Nan::ThrowError("Key generation failed"); + goto done; + } + + result = PEM_write_bio_RSAPrivateKey(mem, rsa, NULL, NULL, 0, NULL, NULL); + if (result != 1) { + Nan::ThrowError("Key export failed"); + goto done; + } + + result = BIO_get_mem_data(mem, &data); + info.GetReturnValue().Set(Nan::Encode(data, result)); + +done: + RSA_free(rsa); + BIO_free(mem); + BN_free(exponent); +} + +NAN_MODULE_INIT(InitAll) +{ + Nan::Set(target, Nan::New("extractPublic").ToLocalChecked(), + Nan::GetFunction(Nan::New(extractPublic)).ToLocalChecked()); + + Nan::Set(target, Nan::New("generate").ToLocalChecked(), + Nan::GetFunction(Nan::New(generate)).ToLocalChecked()); +} + +NODE_MODULE(crypto_key, InitAll); diff --git a/src/crypto_key.d.ts b/src/crypto_key.d.ts new file mode 100644 index 000000000..9aa81a687 --- /dev/null +++ b/src/crypto_key.d.ts @@ -0,0 +1,2 @@ +export function extractPublic(keypair: string): string; +export function generate(): string; diff --git a/src/migrate.ts b/src/migrate.ts new file mode 100644 index 000000000..18217f408 --- /dev/null +++ b/src/migrate.ts @@ -0,0 +1,502 @@ +process.env.NODE_ENV = 'production'; + +import monk from 'monk'; +import * as mongo from 'mongodb'; +import * as fs from 'fs'; +import * as uuid from 'uuid'; +import chalk from 'chalk'; +import config from './config'; +import { initDb } from './db/postgre'; +import { User } from './models/entities/user'; +import { getRepository } from 'typeorm'; +import generateUserToken from './server/api/common/generate-native-user-token'; +import { DriveFile } from './models/entities/drive-file'; +import { DriveFolder } from './models/entities/drive-folder'; +import { InternalStorage } from './services/drive/internal-storage'; +import { createTemp } from './misc/create-temp'; +import { Note } from './models/entities/note'; +import { Following } from './models/entities/following'; +import { Poll } from './models/entities/poll'; +import { PollVote } from './models/entities/poll-vote'; +import { NoteFavorite } from './models/entities/note-favorite'; +import { NoteReaction } from './models/entities/note-reaction'; +import { UserPublickey } from './models/entities/user-publickey'; +import { UserKeypair } from './models/entities/user-keypair'; +import { extractPublic } from './crypto_key'; +import { Emoji } from './models/entities/emoji'; +import { toPuny as _toPuny } from './misc/convert-host'; +import { UserProfile } from './models/entities/user-profile'; + +function toPuny(x: string | null): string | null { + if (x == null) return null; + return _toPuny(x); +} + +const u = (config as any).mongodb.user ? encodeURIComponent((config as any).mongodb.user) : null; +const p = (config as any).mongodb.pass ? encodeURIComponent((config as any).mongodb.pass) : null; + +const uri = `mongodb://${u && p ? `${u}:${p}@` : ''}${(config as any).mongodb.host}:${(config as any).mongodb.port}/${(config as any).mongodb.db}`; + +const db = monk(uri); +let mdb: mongo.Db; + +const test = false; +const limit = 500; + +const nativeDbConn = async (): Promise => { + if (mdb) return mdb; + + const db = await ((): Promise => new Promise((resolve, reject) => { + mongo.MongoClient.connect(uri, { useNewUrlParser: true }, (e: Error, client: any) => { + if (e) return reject(e); + resolve(client.db((config as any).mongodb.db)); + }); + }))(); + + mdb = db; + + return db; +}; + +const _User = db.get('users'); +const _DriveFile = db.get('driveFiles.files'); +const _DriveFolder = db.get('driveFolders'); +const _Note = db.get('notes'); +const _Following = db.get('following'); +const _PollVote = db.get('pollVotes'); +const _Favorite = db.get('favorites'); +const _NoteReaction = db.get('noteReactions'); +const _Emoji = db.get('emoji'); +const getDriveFileBucket = async (): Promise => { + const db = await nativeDbConn(); + const bucket = new mongo.GridFSBucket(db, { + bucketName: 'driveFiles' + }); + return bucket; +}; + +async function main() { + await initDb(); + const Users = getRepository(User); + const UserProfiles = getRepository(UserProfile); + const DriveFiles = getRepository(DriveFile); + const DriveFolders = getRepository(DriveFolder); + const Notes = getRepository(Note); + const Followings = getRepository(Following); + const Polls = getRepository(Poll); + const PollVotes = getRepository(PollVote); + const NoteFavorites = getRepository(NoteFavorite); + const NoteReactions = getRepository(NoteReaction); + const UserPublickeys = getRepository(UserPublickey); + const UserKeypairs = getRepository(UserKeypair); + const Emojis = getRepository(Emoji); + + async function migrateUser(user: any) { + await Users.save({ + id: user._id.toHexString(), + createdAt: user.createdAt || new Date(), + username: user.username, + usernameLower: user.username.toLowerCase(), + host: toPuny(user.host), + token: generateUserToken(), + isAdmin: user.isAdmin || false, + name: user.name, + followersCount: user.followersCount || 0, + followingCount: user.followingCount || 0, + notesCount: user.notesCount || 0, + isBot: user.isBot || false, + isCat: user.isCat || false, + isVerified: user.isVerified || false, + inbox: user.inbox, + sharedInbox: user.sharedInbox, + uri: user.uri, + }); + await UserProfiles.save({ + userId: user._id.toHexString(), + description: user.description, + userHost: toPuny(user.host), + autoAcceptFollowed: true, + autoWatch: false, + password: user.password, + location: user.profile ? user.profile.location : null, + birthday: user.profile ? user.profile.birthday : null, + }); + if (user.publicKey) { + await UserPublickeys.save({ + userId: user._id.toHexString(), + keyId: user.publicKey.id, + keyPem: user.publicKey.publicKeyPem + }); + } + if (user.keypair) { + await UserKeypairs.save({ + userId: user._id.toHexString(), + publicKey: extractPublic(user.keypair), + privateKey: user.keypair, + }); + } + } + + async function migrateFollowing(following: any) { + await Followings.save({ + id: following._id.toHexString(), + createdAt: new Date(), + followerId: following.followerId.toHexString(), + followeeId: following.followeeId.toHexString(), + + // 非正規化 + followerHost: following._follower ? toPuny(following._follower.host) : null, + followerInbox: following._follower ? following._follower.inbox : null, + followerSharedInbox: following._follower ? following._follower.sharedInbox : null, + followeeHost: following._followee ? toPuny(following._followee.host) : null, + followeeInbox: following._followee ? following._followee.inbox : null, + followeeSharedInbox: following._followee ? following._followee.sharedInbo : null + }); + } + + async function migrateDriveFolder(folder: any) { + await DriveFolders.save({ + id: folder._id.toHexString(), + createdAt: folder.createdAt || new Date(), + name: folder.name, + parentId: folder.parentId ? folder.parentId.toHexString() : null, + }); + } + + async function migrateDriveFile(file: any) { + const user = await _User.findOne({ + _id: file.metadata.userId + }); + if (user == null) return; + if (file.metadata.storage && file.metadata.storage.key) { // when object storage + await DriveFiles.save({ + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties || {}, + size: file.length, + url: file.metadata.url, + uri: file.metadata.uri, + accessKey: file.metadata.storage.key, + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + storedInternal: false, + isLink: false + }); + } else if (!file.metadata.isLink) { + const [temp, clean] = await createTemp(); + await new Promise(async (res, rej) => { + const bucket = await getDriveFileBucket(); + const readable = bucket.openDownloadStream(file._id); + const dest = fs.createWriteStream(temp); + readable.pipe(dest); + readable.on('end', () => { + dest.end(); + res(); + }); + }); + + const key = uuid.v4(); + const url = InternalStorage.saveFromPath(key, temp); + await DriveFiles.save({ + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties, + size: file.length, + url: url, + uri: file.metadata.uri, + accessKey: key, + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + storedInternal: true, + isLink: false + }); + clean(); + } else { + await DriveFiles.save({ + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties, + size: file.length, + url: file.metadata.url, + uri: file.metadata.uri, + accessKey: null, + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + storedInternal: false, + isLink: true + }); + } + } + + async function migrateNote(note: any) { + await Notes.save({ + id: note._id.toHexString(), + createdAt: note.createdAt || new Date(), + text: note.text, + cw: note.cw || null, + tags: note.tags || [], + userId: note.userId.toHexString(), + viaMobile: note.viaMobile || false, + geo: note.geo, + appId: null, + visibility: note.visibility || 'public', + visibleUserIds: note.visibleUserIds ? note.visibleUserIds.map((id: any) => id.toHexString()) : [], + replyId: note.replyId ? note.replyId.toHexString() : null, + renoteId: note.renoteId ? note.renoteId.toHexString() : null, + userHost: null, + fileIds: note.fileIds ? note.fileIds.map((id: any) => id.toHexString()) : [], + localOnly: note.localOnly || false, + hasPoll: note.poll != null + }); + + if (note.poll) { + await Polls.save({ + noteId: note._id.toHexString(), + choices: note.poll.choices.map((x: any) => x.text), + expiresAt: note.poll.expiresAt, + multiple: note.poll.multiple, + votes: note.poll.choices.map((x: any) => x.votes), + noteVisibility: note.visibility, + userId: note.userId.toHexString(), + userHost: null + }); + } + } + + async function migratePollVote(vote: any) { + await PollVotes.save({ + id: vote._id.toHexString(), + createdAt: vote.createdAt, + noteId: vote.noteId.toHexString(), + userId: vote.userId.toHexString(), + choice: vote.choice + }); + } + + async function migrateNoteFavorite(favorite: any) { + await NoteFavorites.save({ + id: favorite._id.toHexString(), + createdAt: favorite.createdAt, + noteId: favorite.noteId.toHexString(), + userId: favorite.userId.toHexString(), + }); + } + + async function migrateNoteReaction(reaction: any) { + await NoteReactions.save({ + id: reaction._id.toHexString(), + createdAt: reaction.createdAt, + noteId: reaction.noteId.toHexString(), + userId: reaction.userId.toHexString(), + reaction: reaction.reaction + }); + } + + async function reMigrateUser(user: any) { + const u = await _User.findOne({ + _id: new mongo.ObjectId(user.id) + }); + const avatar = u.avatarId ? await DriveFiles.findOne(u.avatarId.toHexString()) : null; + const banner = u.bannerId ? await DriveFiles.findOne(u.bannerId.toHexString()) : null; + await Users.update(user.id, { + avatarId: avatar ? avatar.id : null, + bannerId: banner ? banner.id : null, + avatarUrl: avatar ? avatar.url : null, + bannerUrl: banner ? banner.url : null + }); + } + + async function migrateEmoji(emoji: any) { + await Emojis.save({ + id: emoji._id.toHexString(), + updatedAt: emoji.createdAt, + aliases: emoji.aliases, + url: emoji.url, + uri: emoji.uri, + host: toPuny(emoji.host), + name: emoji.name + }); + } + + let allUsersCount = await _User.count({ + deletedAt: { $exists: false } + }); + if (test && allUsersCount > limit) allUsersCount = limit; + for (let i = 0; i < allUsersCount; i++) { + const user = await _User.findOne({ + deletedAt: { $exists: false } + }, { + skip: i + }); + try { + await migrateUser(user); + console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allFollowingsCount = await _Following.count(); + if (test && allFollowingsCount > limit) allFollowingsCount = limit; + for (let i = 0; i < allFollowingsCount; i++) { + const following = await _Following.findOne({}, { + skip: i + }); + try { + await migrateFollowing(following); + console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allDriveFoldersCount = await _DriveFolder.count(); + if (test && allDriveFoldersCount > limit) allDriveFoldersCount = limit; + for (let i = 0; i < allDriveFoldersCount; i++) { + const folder = await _DriveFolder.findOne({}, { + skip: i + }); + try { + await migrateDriveFolder(folder); + console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allDriveFilesCount = await _DriveFile.count({ + 'metadata._user.host': null, + 'metadata.deletedAt': { $exists: false } + }); + if (test && allDriveFilesCount > limit) allDriveFilesCount = limit; + for (let i = 0; i < allDriveFilesCount; i++) { + const file = await _DriveFile.findOne({ + 'metadata._user.host': null, + 'metadata.deletedAt': { $exists: false } + }, { + skip: i + }); + try { + await migrateDriveFile(file); + console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allNotesCount = await _Note.count({ + '_user.host': null, + 'metadata.deletedAt': { $exists: false } + }); + if (test && allNotesCount > limit) allNotesCount = limit; + for (let i = 0; i < allNotesCount; i++) { + const note = await _Note.findOne({ + '_user.host': null, + 'metadata.deletedAt': { $exists: false } + }, { + skip: i + }); + try { + await migrateNote(note); + console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allPollVotesCount = await _PollVote.count(); + if (test && allPollVotesCount > limit) allPollVotesCount = limit; + for (let i = 0; i < allPollVotesCount; i++) { + const vote = await _PollVote.findOne({}, { + skip: i + }); + try { + await migratePollVote(vote); + console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allNoteFavoritesCount = await _Favorite.count(); + if (test && allNoteFavoritesCount > limit) allNoteFavoritesCount = limit; + for (let i = 0; i < allNoteFavoritesCount; i++) { + const favorite = await _Favorite.findOne({}, { + skip: i + }); + try { + await migrateNoteFavorite(favorite); + console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allNoteReactionsCount = await _NoteReaction.count(); + if (test && allNoteReactionsCount > limit) allNoteReactionsCount = limit; + for (let i = 0; i < allNoteReactionsCount; i++) { + const reaction = await _NoteReaction.findOne({}, { + skip: i + }); + try { + await migrateNoteReaction(reaction); + console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allActualUsersCount = await Users.count(); + if (test && allActualUsersCount > limit) allActualUsersCount = limit; + for (let i = 0; i < allActualUsersCount; i++) { + const [user] = await Users.find({ + take: 1, + skip: i + }); + try { + await reMigrateUser(user); + console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + const allEmojisCount = await _Emoji.count(); + for (let i = 0; i < allEmojisCount; i++) { + const emoji = await _Emoji.findOne({}, { + skip: i + }); + try { + await migrateEmoji(emoji); + console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + console.log('DONE :)'); +} + +main();