fix some of migration problems (#4903)

* migration: adds missing packages (ref:#4808)

* migrate add missing: user

* migrate add missing: file

* migrate add missing: notes

* migrate add validation: note

- remove attachments which are already deleted
  - => (v11 can't hold them, causes error)
- sets attachementTypes by already migrated
- skips note migration when referenced note does not exist

* migrate add validation: vote, favorite, reaction

* migrate: typo

package.json

@@ -65,6 +65,7 @@
 		"@types/lolex": "3.1.1",
 		"@types/minio": "7.0.1",
 		"@types/mocha": "5.2.6",
+		"@types/mongodb": "3.1.26",
 		"@types/node": "11.13.4",
 		"@types/nodemailer": "4.6.7",
 		"@types/nprogress": "0.0.29",
@@ -166,6 +167,8 @@
 		"mocha": "6.1.3",
 		"moji": "0.5.1",
 		"moment": "2.24.0",
+		"mongodb": "3.2.3",
+		"monk": "6.0.6",
 		"ms": "2.1.1",
 		"nan": "2.12.1",
 		"nested-property": "0.0.7",

src/migrate.ts

@@ -81,6 +81,8 @@ const getDriveFileBucket = async (): Promise<mongo.GridFSBucket> => {
 	return bucket;
 };
 
+const isMigrateRemoteNote = false; // making this true will try to migrate remote notes (possibly could cause errors)
+
 async function main() {
 	await initDb();
 	const Users = getRepository(User);
@@ -100,10 +102,21 @@ async function main() {
 	const Emojis = getRepository(Emoji);
 	const MessagingMessages = getRepository(MessagingMessage);
 
+	async function validateNoteExistOnMigrated(noteId: string) {
+		if (!isMigrateRemoteNote) {
+			const noteMigrated = await Notes.findOne(noteId);
+
+			if (noteMigrated === undefined) {
+				throw `=> ${chalk.yellow('SKIP')}: referenced note does not exist in migrated notes: ${noteId}`;
+			}
+		}
+	}
+
 	async function migrateUser(user: any) {
 		await Users.save({
 			id: user._id.toHexString(),
 			createdAt: typeof user.createdAt === 'number' ? new Date(user.createdAt) : (user.createdAt || new Date()),
+			updatedAt: typeof user.updatedAt === 'number' ? new Date(user.updatedAt) : (user.updatedAt || null),
 			username: user.username,
 			usernameLower: user.username.toLowerCase(),
 			host: toPuny(user.host),
@@ -119,17 +132,59 @@ async function main() {
 			inbox: user.inbox,
 			sharedInbox: user.sharedInbox,
 			uri: user.uri,
+			emojis: user.emojis || [] as string[],
+			tags: user.tags || [] as string[],
+			isSuspended: user.isSuspended,
+			isSilenced: user.isSilenced,
+			isLocked: user.isLocked || false,
 		});
-		await UserProfiles.save({
+
+		const userProfileToSave: any = {
 			userId: user._id.toHexString(),
 			description: user.description,
 			userHost: toPuny(user.host),
 			autoAcceptFollowed: true,
 			autoWatch: false,
+			alwaysMarkNsfw: user.settings ? user.settings.alwaysMarkNsfw : false,
 			password: user.password,
 			location: user.profile ? user.profile.location : null,
 			birthday: user.profile ? user.profile.birthday : null,
-		});
+			email: user.email,
+			emailVerified: user.emailVerified || false,
+			emailVerifyCode: user.emailVerifyCode,
+			twoFactorSecret: user.twoFactorSecret,
+			twoFactorEnabled: user.twoFactorEnabled,
+			twoFactorTempSecret: user.twoFactorTempSecret,
+			carefulBot: user.carefulBot
+		};
+
+		if (user.twitter) {
+			userProfileToSave.twitter = true;
+			userProfileToSave.twitterAccessToken = user.twitter.accessToken;
+			userProfileToSave.twitterAccessTokenSecret = user.twitter.accessTokenSecret;
+			userProfileToSave.twitterUserId = user.twitter.userId;
+			userProfileToSave.twitterScreenName = user.twitter.screenName;
+		}
+
+		if (user.github) {
+			userProfileToSave.github = true;
+			userProfileToSave.githubAccessToken = user.github.accessToken;
+			userProfileToSave.githubId = Number(user.github.id);
+			userProfileToSave.githubLogin = user.github.login;
+		}
+
+		if (user.discord) {
+			userProfileToSave.discord = true;
+			userProfileToSave.discordAccessToken = user.discord.accessToken;
+			userProfileToSave.discordrefreshToken = user.discord.refreshToken;
+			userProfileToSave.discordExpiresDate = user.discord.expiresDate; // number.
+			userProfileToSave.discordId = user.discord.id;
+			userProfileToSave.discordUsername = user.discord.username;
+			userProfileToSave.discordDiscriminator = user.discord.discriminator;
+		}
+
+		await UserProfiles.save(userProfileToSave);
+
 		if (user.publicKey) {
 			await UserPublickeys.save({
 				userId: user._id.toHexString(),
@@ -196,8 +251,8 @@ async function main() {
 			_id: file.metadata.userId
 		});
 		if (user == null) return;
-		if (file.metadata.storageProps && file.metadata.storageProps.key) { // when object storage
+
-			await DriveFiles.save({
+		const fileToSave: any = {
 			id: file._id.toHexString(),
 			userId: user._id.toHexString(),
 			userHost: toPuny(user.host),
@@ -207,13 +262,27 @@ async function main() {
 			type: file.contentType,
 			properties: file.metadata.properties || {},
 			size: file.length,
-				url: file.metadata.url,
+			// url: [different],
 			uri: file.metadata.uri,
-				accessKey: file.metadata.storageProps.key,
+			// accessKey: [different],
 			folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null,
-				storedInternal: false,
+			// storedInternal: [different],
-				isLink: false
+			// isLink: [different],
-			});
+			isSensitive: file.metadata.isSensitive === true,
+			comment: file.metadata.comment && (file.metadata.comment.length > 0) && file.metadata.comment || null,
+			thumbnailUrl: file.metadata.thumbnailUrl,
+			thumbnailAccessKey: file.metadata.storageProps && file.metadata.storageProps.thumbnailAccessKey || null,
+			webpublicUrl: file.metadata.webpublicUrl,
+			webpublicAccessKey: file.metadata.storageProps && file.metadata.storageProps.webpublicAccessKey || null,
+		};
+
+		if (file.metadata.storageProps && file.metadata.storageProps.key) { // when object storage
+			fileToSave.url = file.metadata.url;
+			fileToSave.accessKey = file.metadata.storageProps.key;
+			fileToSave.storedInternal = false;
+			fileToSave.isLink = false;
+
+			await DriveFiles.save(fileToSave);
 		} else if (!file.metadata.isLink) {
 			const [temp, clean] = await createTemp();
 			await new Promise(async (res, rej) => {
@@ -229,47 +298,26 @@ async function main() {
 
 			const key = uuid.v4();
 			const url = InternalStorage.saveFromPath(key, temp);
-			await DriveFiles.save({
+
-				id: file._id.toHexString(),
+			fileToSave.url = url;
-				userId: user._id.toHexString(),
+			fileToSave.accessKey = key;
-				userHost: toPuny(user.host),
+			fileToSave.storedInternal = true;
-				createdAt: file.uploadDate || new Date(),
+			fileToSave.isLink = false;
-				md5: file.md5,
+
-				name: file.filename,
+			await DriveFiles.save(fileToSave);
-				type: file.contentType,
-				properties: file.metadata.properties,
-				size: file.length,
-				url: url,
-				uri: file.metadata.uri,
-				accessKey: key,
-				folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null,
-				storedInternal: true,
-				isLink: false
-			});
 			clean();
 		} else {
-			await DriveFiles.save({
+			fileToSave.url = file.metadata.url;
-				id: file._id.toHexString(),
+			fileToSave.accessKey = null;
-				userId: user._id.toHexString(),
+			fileToSave.storedInternal = false;
-				userHost: toPuny(user.host),
+			fileToSave.isLink = true;
-				createdAt: file.uploadDate || new Date(),
+
-				md5: file.md5,
+			await DriveFiles.save(fileToSave);
-				name: file.filename,
-				type: file.contentType,
-				properties: file.metadata.properties,
-				size: file.length,
-				url: file.metadata.url,
-				uri: file.metadata.uri,
-				accessKey: null,
-				folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null,
-				storedInternal: false,
-				isLink: true
-			});
 		}
 	}
 
 	async function migrateNote(note: any) {
-		await Notes.save({
+		const noteToSave = {
 			id: note._id.toHexString(),
 			createdAt: note.createdAt || new Date(),
 			text: note.text,
@@ -279,24 +327,74 @@ async function main() {
 			viaMobile: note.viaMobile || false,
 			geo: note.geo,
 			appId: null,
-			visibility: note.visibility || 'public',
+			visibility: note.visibility && (note.visibility === 'private' ? 'specified' : note.visibility) || 'public', // there is no 'private' visibility more.
 			visibleUserIds: note.visibleUserIds ? note.visibleUserIds.map((id: any) => id.toHexString()) : [],
 			replyId: note.replyId ? note.replyId.toHexString() : null,
 			renoteId: note.renoteId ? note.renoteId.toHexString() : null,
 			userHost: null,
 			fileIds: note.fileIds ? note.fileIds.map((id: any) => id.toHexString()) : [],
+			attachedFileTypes: ([] as string[]), // see below
 			localOnly: note.localOnly || false,
-			hasPoll: note.poll != null
+			hasPoll: note.poll != null,
+			name: note.name && (note.name.length > 0) && note.name || null,
+			emojis: note.emojis || ([] as string[]),
+			renoteCount: note.renoteCount || 0,
+			repliesCount: note.repliesCount || 0,
+			mentions: note.mentions && note.mentions.map((id: any) => id.toHexString()) || [],
+			mentionedRemoteUsers: note.mentionedRemoteUsers && JSON.stringify(note.mentionedRemoteUsers) || '[]',
+			score: note.score || 0,
+			uri: note.uri || null
+		};
+
+		// validate existance of referenced notes (on migrated)
+		if ((!isMigrateRemoteNote) && (noteToSave.replyId !== null || noteToSave.renoteId !== null)) {
+			// skip when reply does not exist on local
+			if (noteToSave.replyId !== null) {
+				const mongoReplyNoteLocal = await _Note.findOne({
+					'_user.host': null,
+					'_id': note.replyId
 				});
 
+				if (mongoReplyNoteLocal === null) {
+					throw `=> ${chalk.yellow('SKIP')}: referenced "local" reply note does not exist: ${note.replyId}`;
+				}
+			}
+
+			// skip when reply does not exist on local
+			if (noteToSave.renoteId !== null) {
+				const mongoRenoteNoteLocal = await _Note.findOne({
+					'_user.host': null,
+					'_id': note.renoteId
+				});
+
+				if (mongoRenoteNoteLocal === null) {
+					throw `=> ${chalk.yellow('SKIP')}: referenced "local" renote note does not exist: ${note.renoteId}`;
+				}
+			}
+		}
+
+		if (noteToSave.fileIds.length !== 0) {
+			const filesMigrated = await DriveFiles.findByIds(noteToSave.fileIds);
+
+			// remove attachments which user removed after creating note
+			if (noteToSave.fileIds.length !== filesMigrated.length) {
+				console.warn(`NOTE ${noteToSave.id} ${chalk.yellow('MODIFIED')}: file count is different: before: ${noteToSave.fileIds.length} => after: ${filesMigrated.length}`);
+				noteToSave.fileIds = filesMigrated.map(file => file.id) || [];
+			}
+
+			noteToSave.attachedFileTypes = filesMigrated.map(file => file.type);
+		}
+
+		await Notes.save(noteToSave);
+
 		if (note.poll) {
 			await Polls.save({
 				noteId: note._id.toHexString(),
 				choices: note.poll.choices.map((x: any) => x.text),
 				expiresAt: note.poll.expiresAt,
-				multiple: note.poll.multiple,
+				multiple: note.poll.multiple || false,
 				votes: note.poll.choices.map((x: any) => x.votes),
-				noteVisibility: note.visibility,
+				noteVisibility: note.visibility && (note.visibility === 'private' ? 'specified' : note.visibility) || 'public', // there is no 'private' visibility more.
 				userId: note.userId.toHexString(),
 				userHost: null
 			});
@@ -304,32 +402,44 @@ async function main() {
 	}
 
 	async function migratePollVote(vote: any) {
-		await PollVotes.save({
+		const voteToSave = {
 			id: vote._id.toHexString(),
 			createdAt: vote.createdAt,
 			noteId: vote.noteId.toHexString(),
 			userId: vote.userId.toHexString(),
 			choice: vote.choice
-		});
+		};
+
+		await validateNoteExistOnMigrated(voteToSave.noteId);
+
+		await PollVotes.save(voteToSave);
 	}
 
 	async function migrateNoteFavorite(favorite: any) {
-		await NoteFavorites.save({
+		const favoriteToSave = {
 			id: favorite._id.toHexString(),
 			createdAt: favorite.createdAt,
 			noteId: favorite.noteId.toHexString(),
 			userId: favorite.userId.toHexString(),
-		});
+		};
+
+		await validateNoteExistOnMigrated(favoriteToSave.noteId);
+
+		await NoteFavorites.save(favoriteToSave);
 	}
 
 	async function migrateNoteReaction(reaction: any) {
-		await NoteReactions.save({
+		const reactionToSave = {
 			id: reaction._id.toHexString(),
 			createdAt: reaction.createdAt,
 			noteId: reaction.noteId.toHexString(),
 			userId: reaction.userId.toHexString(),
 			reaction: reaction.reaction
-		});
+		};
+
+		await validateNoteExistOnMigrated(reactionToSave.noteId);
+
+		await NoteReactions.save(reactionToSave);
 	}
 
 	async function reMigrateUser(user: any) {
@@ -470,16 +580,18 @@ async function main() {
 		}
 	}
 
-	let allNotesCount = await _Note.count({
+	const noteCondition = {
 		'_user.host': null,
 		'metadata.deletedAt': { $exists: false }
-	});
+	};
+	if (isMigrateRemoteNote) {
+		delete noteCondition['_user.host'];
+	}
+
+	let allNotesCount = await _Note.count(noteCondition);
 	if (test && allNotesCount > limit) allNotesCount = limit;
 	for (let i = 0; i < allNotesCount; i++) {
-		const note = await _Note.findOne({
+		const note = await _Note.findOne(noteCondition, {
-			'_user.host': null,
-			'metadata.deletedAt': { $exists: false }
-		}, {
 			skip: i
 		});
 		try {