mirror of
https://github.com/monkeytypegame/monkeytype.git
synced 2025-02-04 21:09:18 +08:00
443 lines
15 KiB
JavaScript
443 lines
15 KiB
JavaScript
const { config } = require("dotenv");
|
|
const path = require("path");
|
|
config({ path: path.join(__dirname, ".env") });
|
|
const { mongoDB } = require("./init/mongodb");
|
|
const { connectDB } = require("./init/mongodb");
|
|
const { ObjectID } = require("mongodb");
|
|
const { performance } = require("perf_hooks");
|
|
const fs = require("fs");
|
|
|
|
// const { QuerySnapshotData } = require("firebase-firestore");
|
|
|
|
console.log(config());
|
|
|
|
const admin = require("firebase-admin");
|
|
|
|
// const { User } = require("./models/user");
|
|
// const { Leaderboard } = require("./models/leaderboard");
|
|
// const { BotCommand } = require("./models/bot-command");
|
|
|
|
const serviceAccount = require("./credentials/serviceAccountKey.json");
|
|
|
|
admin.initializeApp({
|
|
credential: admin.credential.cert(serviceAccount),
|
|
});
|
|
|
|
var db = admin.firestore();
|
|
var auth = admin.auth();
|
|
|
|
process.on("SIGTERM", async () => {
|
|
console.info("SIGTERM signal received. Stopping after this user is done");
|
|
await currentUserPromise;
|
|
process.exit(1);
|
|
});
|
|
|
|
process.on("exit", async () => {
|
|
console.info("exit signal received. Stopping after this user is done");
|
|
await currentUserPromise;
|
|
process.exit(1);
|
|
});
|
|
|
|
process.on("SIGINT", async () => {
|
|
console.info("SIGINT signal received. Stopping after this user is done");
|
|
await currentUserPromise;
|
|
process.exit(1);
|
|
});
|
|
|
|
// Database should be completely clear before this is ran in order to prevent overlapping documents
|
|
// Migrate users
|
|
|
|
let currentUserPromise = null;
|
|
|
|
let resolveUser = null;
|
|
|
|
async function migrateUsers() {
|
|
// let UIDOVERRIDE = "ugbG1GiSHxVEYMDmMeLV9byeukl2";
|
|
let UIDOVERRIDE = undefined;
|
|
let lastId;
|
|
let usersSoFar = 0;
|
|
let totalUsers = 330000;
|
|
let totalCompletionTime = 0;
|
|
let averageCompletionTime = 0;
|
|
try {
|
|
let migrationStats = JSON.parse(
|
|
fs.readFileSync("./migrationStats.txt", "utf8")
|
|
);
|
|
lastId = migrationStats.uid;
|
|
usersSoFar = migrationStats.usersSoFar;
|
|
totalCompletionTime = migrationStats.totalCompletionTime;
|
|
averageCompletionTime = migrationStats.averageCompletionTime;
|
|
} catch (e) {}
|
|
let querySnapshot;
|
|
let limit = 1000;
|
|
do {
|
|
console.log("starting another loop, getting users");
|
|
if (lastId) {
|
|
let lastSnapshot = await db.collection("users").doc(lastId).get();
|
|
querySnapshot = await db
|
|
.collection("users")
|
|
.where("banned", "==", true)
|
|
.orderBy("name")
|
|
.startAfter(lastSnapshot)
|
|
.limit(limit)
|
|
.get();
|
|
} else {
|
|
querySnapshot = await db
|
|
.collection("users")
|
|
.where("banned", "==", true)
|
|
.orderBy("name")
|
|
.limit(limit)
|
|
.get();
|
|
}
|
|
// console.log('start of foreach');
|
|
console.log(`migrating ${querySnapshot.docs.length} users`);
|
|
let fulllog = false;
|
|
for (const userDoc of querySnapshot.docs) {
|
|
let userstart = performance.now();
|
|
currentUserPromise = null;
|
|
currentUserPromise = new Promise((resolve, reject) => {
|
|
resolveUser = resolve;
|
|
});
|
|
let userData = userDoc.data();
|
|
let uid = userDoc.id;
|
|
try {
|
|
let userAuth = await auth.getUser(uid);
|
|
let email = userAuth.email;
|
|
let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime();
|
|
|
|
let mongoUser = {
|
|
name: userData.name,
|
|
email: email,
|
|
addedAt: userCreatedAt,
|
|
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
|
|
oldTypingStats: {},
|
|
};
|
|
|
|
if (userData.completedTests)
|
|
mongoUser.oldTypingStats.completedTests = userData.completedTests;
|
|
if (userData.discordId) mongoUser.discordId = userData.discordId;
|
|
if (userData.banned) mongoUser.banned = userData.banned;
|
|
if (userData.verified) mongoUser.verified = userData.verified;
|
|
//banned
|
|
//verified
|
|
if (userData.startedTests)
|
|
mongoUser.oldTypingStats.startedTests = userData.startedTests;
|
|
if (userData.timeTyping)
|
|
mongoUser.oldTypingStats.timeTyping = userData.timeTyping;
|
|
|
|
if (userData.personalBests)
|
|
mongoUser.personalBests = userData.personalBests;
|
|
|
|
let tagPairs = {};
|
|
|
|
let mongoUserTags = [];
|
|
|
|
if (fulllog) console.log(`${uid} migrating tags`);
|
|
let tagsSnapshot = await db.collection(`users/${uid}/tags`).get();
|
|
await tagsSnapshot.forEach(async (tagDoc) => {
|
|
let tagData = tagDoc.data();
|
|
let tagId = tagDoc.id;
|
|
let new_id = ObjectID();
|
|
tagPairs[tagId] = new_id;
|
|
let tagtopush = { _id: new_id, name: tagData.name };
|
|
if (tagData.personalBests)
|
|
tagtopush.personalBests = tagData.personalBests;
|
|
mongoUserTags.push(tagtopush);
|
|
});
|
|
|
|
mongoUser.tags = mongoUserTags;
|
|
|
|
// if (fulllog) console.log(`${uid} migrating config`);
|
|
// if (userData.config) {
|
|
// await mongoDB()
|
|
// .collection("configs")
|
|
// .updateOne(
|
|
// { uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
|
|
// {
|
|
// $set: {
|
|
// uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
|
|
// config: userData.config,
|
|
// },
|
|
// },
|
|
// { upsert: true }
|
|
// );
|
|
// }
|
|
|
|
// if (fulllog) console.log(`${uid} migrating presets`);
|
|
// let presetsSnapshot = await db.collection(`users/${uid}/presets`).get();
|
|
// await presetsSnapshot.forEach(async (presetDoc) => {
|
|
// let presetData = presetDoc.data();
|
|
// let newpreset = {
|
|
// uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
|
|
// name: presetData.name,
|
|
// };
|
|
// if (presetData.config) newpreset.config = presetData.config;
|
|
// await mongoDB().collection("presets").insertOne(newpreset);
|
|
// });
|
|
|
|
// let lastcount = 0;
|
|
// let limit = 1000;
|
|
// let lastdoc = "start";
|
|
// let total = 0;
|
|
// let newStats = {
|
|
// completedTests: 0,
|
|
// startedTests: 0,
|
|
// timeTyping: 0,
|
|
// };
|
|
// if (fulllog) console.log(`${uid} migrating results`);
|
|
// do {
|
|
// if (fulllog) console.log(`${total} so far`);
|
|
// let resultsSnapshot;
|
|
// if (lastdoc === "start") {
|
|
// resultsSnapshot = await db
|
|
// .collection(`users/${uid}/results`)
|
|
// .orderBy("timestamp", "desc")
|
|
// .limit(limit)
|
|
// .get();
|
|
// } else {
|
|
// resultsSnapshot = await db
|
|
// .collection(`users/${uid}/results`)
|
|
// .orderBy("timestamp", "desc")
|
|
// .startAfter(lastdoc)
|
|
// .limit(limit)
|
|
// .get();
|
|
// }
|
|
// await resultsSnapshot.forEach(async (resultDoc) => {
|
|
// let resultData = resultDoc.data();
|
|
// resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid;
|
|
// if (resultData.tags && resultData.tags.length > 0) {
|
|
// resultData.tags = resultData.tags.map((tag) => tagPairs[tag]);
|
|
// }
|
|
// if (!resultData.charStats) {
|
|
// resultData.charStats = [
|
|
// resultData.correctChars,
|
|
// resultData.incorrectChars,
|
|
// ];
|
|
// }
|
|
// delete resultData.correctChars;
|
|
// delete resultData.incorrectChars;
|
|
// delete resultData.allChars;
|
|
// delete resultData.keySpacing;
|
|
// delete resultData.keyDuration;
|
|
// delete resultData.theme;
|
|
// delete resultData.name;
|
|
|
|
// //remove the default fields here
|
|
// if (resultData.bailedOut === false) delete resultData.bailedOut;
|
|
// if (resultData.blindMode === false) delete resultData.blindMode;
|
|
// if (resultData.difficulty === "normal")
|
|
// delete resultData.difficulty;
|
|
// if (resultData.funbox === "none") delete resultData.funbox;
|
|
// if (resultData.language === "english") delete resultData.language;
|
|
// if (resultData.numbers === false) delete resultData.numbers;
|
|
// if (resultData.punctuation === false) delete resultData.punctuation;
|
|
|
|
// if (resultData.mode !== "custom") delete resultData.customText;
|
|
|
|
// newStats.completedTests++;
|
|
// if (resultData.restartCount) {
|
|
// newStats.startedTests += resultData.restartCount + 1;
|
|
// } else {
|
|
// newStats.startedTests++;
|
|
// }
|
|
// if (resultData.testDuration) {
|
|
// newStats.timeTyping += parseFloat(resultData.testDuration);
|
|
// }
|
|
// if (resultData.incompleteTestSeconds) {
|
|
// newStats.timeTyping += resultData.incompleteTestSeconds;
|
|
// }
|
|
// await mongoDB().collection("results").insertOne(resultData);
|
|
// });
|
|
// lastcount = resultsSnapshot.docs.length;
|
|
// lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1];
|
|
// total += lastcount;
|
|
// } while (lastcount > 0);
|
|
|
|
// if (fulllog) console.log(`${uid} migrated ${total} results`);
|
|
|
|
// mongoUser.completedTests = newStats.completedTests;
|
|
// mongoUser.startedTests = newStats.startedTests;
|
|
// mongoUser.timeTyping = newStats.timeTyping;
|
|
|
|
if (fulllog) console.log(`${uid} migrating user doc`);
|
|
await mongoDB()
|
|
.collection("users")
|
|
.updateOne(
|
|
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
|
|
{
|
|
$set: mongoUser,
|
|
},
|
|
{ upsert: true }
|
|
);
|
|
|
|
console.log(`${uid} migrated \t\t ${userData.name} \t\t`);
|
|
fs.appendFileSync(
|
|
"log_success.txt",
|
|
`${uid}\t\t${userData.name}\t\t`,
|
|
"utf8"
|
|
);
|
|
} catch (err) {
|
|
console.log(`${uid} failed`);
|
|
console.log(err);
|
|
fs.appendFileSync(
|
|
"log_failed.txt",
|
|
`${uid}\t\t${err.message}\n`,
|
|
"utf8"
|
|
);
|
|
}
|
|
lastId = uid;
|
|
let userend = performance.now();
|
|
let time = (userend - userstart) / 1000;
|
|
totalCompletionTime += time;
|
|
// console.log(`${uid} took ${time} seconds`);
|
|
averageCompletionTime = totalCompletionTime / usersSoFar + 1;
|
|
usersSoFar++;
|
|
let estimateSecondsLeft =
|
|
averageCompletionTime * (totalUsers - usersSoFar);
|
|
console.log(
|
|
`${usersSoFar}/${totalUsers} users | estimated ${secondsToString(
|
|
estimateSecondsLeft,
|
|
true
|
|
)} left`
|
|
);
|
|
|
|
fs.writeFileSync(
|
|
"migrationStats.txt",
|
|
JSON.stringify({
|
|
uid,
|
|
usersSoFar,
|
|
totalCompletionTime,
|
|
averageCompletionTime,
|
|
}),
|
|
"utf8"
|
|
);
|
|
|
|
resolveUser();
|
|
|
|
// console.log(userData);
|
|
// let newUser;
|
|
// try{
|
|
// let data = userDoc.data();
|
|
// data._id = userDoc.id;
|
|
// newUser = new User(data);
|
|
// newUser.uid = userDoc.id;
|
|
// newUser.globalStats = {
|
|
// started: userDoc.data().startedTests,
|
|
// completed: userDoc.data().completedTests,
|
|
// time: userDoc.data().timeTyping,
|
|
// };
|
|
// let tagIdDict = {};
|
|
// let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get();
|
|
// tagsSnapshot.forEach((tagDoc) => {
|
|
// let formattedTag = tagDoc.data();
|
|
// formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId
|
|
// tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags
|
|
// newUser.tags.push(formattedTag);
|
|
// console.log(`Tag ${tagDoc.id} saved for user ${userCount}`);
|
|
// });
|
|
// let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get();
|
|
// let resCount = 1;
|
|
// resultsSnapshot.forEach((result) => {
|
|
// let formattedResult = result.data();
|
|
// if(formattedResult.tags != undefined){
|
|
// formattedResult.tags.forEach((tag, index) => {
|
|
// if (tagIdDict[tag])
|
|
// formattedResult.tags[index] = tagIdDict[tag];
|
|
// });
|
|
// }
|
|
// newUser.results.push(formattedResult);
|
|
// console.log(`Result ${resCount} saved for user ${userCount}`);
|
|
// resCount++;
|
|
// });
|
|
// newUser.results.sort((a, b) => {
|
|
// return a.timestamp - b.timestamp;
|
|
// });
|
|
// let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get();
|
|
// presetsSnapshot.forEach((preset) => {
|
|
// newUser.presets.push(preset.data());
|
|
// });
|
|
// await newUser.save();
|
|
// console.log(`User ${userCount} (${newUser.uid}) saved`);
|
|
// userCount++;
|
|
// }catch(e){
|
|
// // throw e;
|
|
// console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`);
|
|
// userCount++;
|
|
// }
|
|
}
|
|
} while (querySnapshot.docs.length > 0);
|
|
|
|
console.log("Migration complete");
|
|
// console.log('end of foreach');
|
|
}
|
|
// //not tested because I can't get leaderboards to work on my fork for some reason
|
|
// db.collection("leaderboards")
|
|
// .get()
|
|
// .then((leaderboardsSnapshot) => {
|
|
// leaderboardsSnapshot.forEach((lbDoc) => {
|
|
// let newLb = new Leaderboard(lbDoc.data());
|
|
// newLb.save();
|
|
// });
|
|
// });
|
|
|
|
// //migrate bot-commands
|
|
// db.collection("bot-commands")
|
|
// .get()
|
|
// .then((botCommandsSnapshot) => {
|
|
// botCommandsSnapshot.forEach((bcDoc) => {
|
|
// let newBotCommand = new BotCommand(bcDoc.data());
|
|
// newBotCommand.save();
|
|
// });
|
|
// });
|
|
|
|
//migrate public stat
|
|
async function migratePublicStats() {
|
|
db.collection("public")
|
|
.doc("stats")
|
|
.get()
|
|
.then((ret) => {
|
|
let stats = ret.data();
|
|
mongoDB()
|
|
.collection("public")
|
|
.updateOne(
|
|
{ type: "stats" },
|
|
{
|
|
$set: {
|
|
completedTests: stats.completedTests,
|
|
startedTests: stats.startedTests,
|
|
timeTyping: stats.timeTyping,
|
|
},
|
|
},
|
|
{ upsert: true }
|
|
);
|
|
});
|
|
}
|
|
|
|
async function init() {
|
|
await connectDB();
|
|
// await migratePublicStats();
|
|
await migrateUsers();
|
|
// process.exit(1);
|
|
}
|
|
|
|
function secondsToString(sec, full = false) {
|
|
const hours = Math.floor(sec / 3600);
|
|
const minutes = Math.floor((sec % 3600) / 60);
|
|
const seconds = Math.round((sec % 3600) % 60);
|
|
let hoursString;
|
|
let minutesString;
|
|
let secondsString;
|
|
hours < 10 ? (hoursString = "0" + hours) : (hoursString = hours);
|
|
minutes < 10 ? (minutesString = "0" + minutes) : (minutesString = minutes);
|
|
seconds < 10 && (minutes > 0 || hours > 0 || full)
|
|
? (secondsString = "0" + seconds)
|
|
: (secondsString = seconds);
|
|
|
|
let ret = "";
|
|
if (hours > 0 || full) ret += hoursString + ":";
|
|
if (minutes > 0 || hours > 0 || full) ret += minutesString + ":";
|
|
ret += secondsString;
|
|
return ret;
|
|
}
|
|
|
|
init();
|