mirror of
https://github.com/monkeytypegame/monkeytype.git
synced 2025-12-29 11:26:13 +08:00
migration changes
added uid override for testing saving old stats in a separate object and recalculating new stats based on results reducing logs paginating results calculating estimated time remaining
This commit is contained in:
Miodec
parent
6d9fbec7a2
commit
565cf1ea62
1 changed files with 238 additions and 145 deletions
|
|
@ -4,6 +4,7 @@ config({ path: path.join(__dirname, ".env") });
|
|||
const { mongoDB } = require("./init/mongodb");
|
||||
const { connectDB } = require("./init/mongodb");
|
||||
const { ObjectID } = require("mongodb");
|
||||
const { performance } = require("perf_hooks");
|
||||
|
||||
console.log(config());
|
||||
|
||||
|
|
@ -13,7 +14,7 @@ const admin = require("firebase-admin");
|
|||
// const { Leaderboard } = require("./models/leaderboard");
|
||||
// const { BotCommand } = require("./models/bot-command");
|
||||
|
||||
const serviceAccount = require("../functions/serviceAccountKey.json");
|
||||
const serviceAccount = require("../functions/serviceAccountKey_live.json");
|
||||
|
||||
admin.initializeApp({
|
||||
credential: admin.credential.cert(serviceAccount),
|
||||
|
|
@ -25,165 +26,237 @@ var auth = admin.auth();
|
|||
// Database should be completely clear before this is ran in order to prevent overlapping documents
|
||||
// Migrate users
|
||||
async function migrateUsers() {
|
||||
await db
|
||||
// let UIDOVERRIDE = "ugbG1GiSHxVEYMDmMeLV9byeukl2";
|
||||
let UIDOVERRIDE = undefined;
|
||||
let querySnapshot = await db
|
||||
.collection("users")
|
||||
.where("name", "==", "mio")
|
||||
.get()
|
||||
.then((querySnapshot) => {
|
||||
// console.log('start of foreach');
|
||||
querySnapshot.forEach(async (userDoc) => {
|
||||
let userData = userDoc.data();
|
||||
let uid = userDoc.id;
|
||||
try {
|
||||
let userAuth = await auth.getUser(uid);
|
||||
let email = userAuth.email;
|
||||
let userCreatedAt = new Date(
|
||||
userAuth.metadata.creationTime
|
||||
).getTime();
|
||||
// .where("name", "==", "Miodec")
|
||||
.limit(10000)
|
||||
.get();
|
||||
// console.log('start of foreach');
|
||||
console.log(`migrating ${querySnapshot.docs.length} users`);
|
||||
let fulllog = false;
|
||||
let currentIndex = 0;
|
||||
let totalUsers = querySnapshot.docs.length;
|
||||
let totalCompletionTime = 0;
|
||||
let averageCompletionTime = 0;
|
||||
for (const userDoc of querySnapshot.docs) {
|
||||
let userstart = performance.now();
|
||||
let userData = userDoc.data();
|
||||
let uid = userDoc.id;
|
||||
try {
|
||||
let userAuth = await auth.getUser(uid);
|
||||
let email = userAuth.email;
|
||||
let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime();
|
||||
|
||||
let mongoUser = {
|
||||
name: userData.name,
|
||||
email: email,
|
||||
addedAt: userCreatedAt,
|
||||
uid: uid,
|
||||
};
|
||||
let mongoUser = {
|
||||
name: userData.name,
|
||||
email: email,
|
||||
addedAt: userCreatedAt,
|
||||
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
|
||||
oldTypingStats: {},
|
||||
};
|
||||
|
||||
if (userData.completedTests)
|
||||
mongoUser.completedTests = userData.completedTests;
|
||||
if (userData.discordId) mongoUser.discordId = userData.discordId;
|
||||
if (userData.startedTests)
|
||||
mongoUser.startedTests = userData.startedTests;
|
||||
if (userData.timeTyping) mongoUser.timeTyping = userData.timeTyping;
|
||||
if (userData.completedTests)
|
||||
mongoUser.oldTypingStats.completedTests = userData.completedTests;
|
||||
if (userData.discordId) mongoUser.discordId = userData.discordId;
|
||||
if (userData.startedTests)
|
||||
mongoUser.oldTypingStats.startedTests = userData.startedTests;
|
||||
if (userData.timeTyping)
|
||||
mongoUser.oldTypingStats.timeTyping = userData.timeTyping;
|
||||
|
||||
if (userData.personalBests)
|
||||
mongoUser.personalBests = userData.personalBests;
|
||||
if (userData.personalBests)
|
||||
mongoUser.personalBests = userData.personalBests;
|
||||
|
||||
let tagPairs = {};
|
||||
let tagPairs = {};
|
||||
|
||||
let mongoUserTags = [];
|
||||
let mongoUserTags = [];
|
||||
|
||||
let tagsSnapshot = await db.collection(`users/${uid}/tags`).get();
|
||||
tagsSnapshot.forEach((tagDoc) => {
|
||||
let tagData = tagDoc.data();
|
||||
let tagId = tagDoc.id;
|
||||
console.log(tagData);
|
||||
let new_id = ObjectID();
|
||||
tagPairs[tagId] = new_id;
|
||||
let tagtopush = { _id: new_id, name: tagData.name };
|
||||
if (tagData.personalBests)
|
||||
tagtopush.personalBests = tagData.personalBests;
|
||||
mongoUserTags.push(tagtopush);
|
||||
});
|
||||
if (fulllog) console.log(`${uid} migrating tags`);
|
||||
let tagsSnapshot = await db.collection(`users/${uid}/tags`).get();
|
||||
await tagsSnapshot.forEach(async (tagDoc) => {
|
||||
let tagData = tagDoc.data();
|
||||
let tagId = tagDoc.id;
|
||||
let new_id = ObjectID();
|
||||
tagPairs[tagId] = new_id;
|
||||
let tagtopush = { _id: new_id, name: tagData.name };
|
||||
if (tagData.personalBests)
|
||||
tagtopush.personalBests = tagData.personalBests;
|
||||
mongoUserTags.push(tagtopush);
|
||||
});
|
||||
|
||||
mongoUser.tags = mongoUserTags;
|
||||
mongoUser.tags = mongoUserTags;
|
||||
|
||||
await mongoDB().collection("users").updateOne(
|
||||
{ uid: uid },
|
||||
if (fulllog) console.log(`${uid} migrating config`);
|
||||
if (userData.config) {
|
||||
await mongoDB()
|
||||
.collection("configs")
|
||||
.updateOne(
|
||||
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
|
||||
{
|
||||
$set: mongoUser,
|
||||
$set: {
|
||||
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
|
||||
config: userData.config,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
}
|
||||
|
||||
if (userData.config) {
|
||||
await mongoDB()
|
||||
.collection("configs")
|
||||
.updateOne(
|
||||
{ uid: uid },
|
||||
{
|
||||
$set: {
|
||||
uid: uid,
|
||||
config: userData.config,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
}
|
||||
|
||||
let presetsSnapshot = await db
|
||||
.collection(`users/${uid}/presets`)
|
||||
.get();
|
||||
presetsSnapshot.forEach(async (presetDoc) => {
|
||||
let presetData = presetDoc.data();
|
||||
let newpreset = {
|
||||
uid: uid,
|
||||
name: presetData.name,
|
||||
};
|
||||
if (presetData.config) newpreset.config = presetData.config;
|
||||
await mongoDB().collection("presets").insertOne(newpreset);
|
||||
});
|
||||
|
||||
let resultsSnapshot = await db
|
||||
.collection(`users/${uid}/results`)
|
||||
.get();
|
||||
resultsSnapshot.forEach(async (resultDoc) => {
|
||||
let resultData = resultDoc.data();
|
||||
resultData.uid = uid;
|
||||
if (resultData.tags && resultData.tags.length > 0) {
|
||||
resultData.tags = resultData.tags.map((tag) => tagPairs[tag]);
|
||||
}
|
||||
await mongoDB().collection("results").insertOne(resultData);
|
||||
});
|
||||
|
||||
console.log(`${uid} migrated`);
|
||||
} catch (err) {
|
||||
console.log(`${uid} failed`);
|
||||
console.log(err);
|
||||
}
|
||||
|
||||
// console.log(userData);
|
||||
// let newUser;
|
||||
// try{
|
||||
// let data = userDoc.data();
|
||||
// data._id = userDoc.id;
|
||||
// newUser = new User(data);
|
||||
// newUser.uid = userDoc.id;
|
||||
// newUser.globalStats = {
|
||||
// started: userDoc.data().startedTests,
|
||||
// completed: userDoc.data().completedTests,
|
||||
// time: userDoc.data().timeTyping,
|
||||
// };
|
||||
// let tagIdDict = {};
|
||||
// let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get();
|
||||
// tagsSnapshot.forEach((tagDoc) => {
|
||||
// let formattedTag = tagDoc.data();
|
||||
// formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId
|
||||
// tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags
|
||||
// newUser.tags.push(formattedTag);
|
||||
// console.log(`Tag ${tagDoc.id} saved for user ${userCount}`);
|
||||
// });
|
||||
// let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get();
|
||||
// let resCount = 1;
|
||||
// resultsSnapshot.forEach((result) => {
|
||||
// let formattedResult = result.data();
|
||||
// if(formattedResult.tags != undefined){
|
||||
// formattedResult.tags.forEach((tag, index) => {
|
||||
// if (tagIdDict[tag])
|
||||
// formattedResult.tags[index] = tagIdDict[tag];
|
||||
// });
|
||||
// }
|
||||
// newUser.results.push(formattedResult);
|
||||
// console.log(`Result ${resCount} saved for user ${userCount}`);
|
||||
// resCount++;
|
||||
// });
|
||||
// newUser.results.sort((a, b) => {
|
||||
// return a.timestamp - b.timestamp;
|
||||
// });
|
||||
// let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get();
|
||||
// presetsSnapshot.forEach((preset) => {
|
||||
// newUser.presets.push(preset.data());
|
||||
// });
|
||||
// await newUser.save();
|
||||
// console.log(`User ${userCount} (${newUser.uid}) saved`);
|
||||
// userCount++;
|
||||
// }catch(e){
|
||||
// // throw e;
|
||||
// console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`);
|
||||
// userCount++;
|
||||
// }
|
||||
if (fulllog) console.log(`${uid} migrating presets`);
|
||||
let presetsSnapshot = await db.collection(`users/${uid}/presets`).get();
|
||||
await presetsSnapshot.forEach(async (presetDoc) => {
|
||||
let presetData = presetDoc.data();
|
||||
let newpreset = {
|
||||
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
|
||||
name: presetData.name,
|
||||
};
|
||||
if (presetData.config) newpreset.config = presetData.config;
|
||||
await mongoDB().collection("presets").insertOne(newpreset);
|
||||
});
|
||||
// console.log('end of foreach');
|
||||
});
|
||||
|
||||
let lastcount = 0;
|
||||
let limit = 1000;
|
||||
let lastdoc = "start";
|
||||
let total = 0;
|
||||
let newStats = {
|
||||
completedTests: 0,
|
||||
startedTests: 0,
|
||||
timeTyping: 0,
|
||||
};
|
||||
if (fulllog) console.log(`${uid} migrating results`);
|
||||
do {
|
||||
if (fulllog) console.log(`${total} so far`);
|
||||
let resultsSnapshot;
|
||||
if (lastdoc === "start") {
|
||||
resultsSnapshot = await db
|
||||
.collection(`users/${uid}/results`)
|
||||
.orderBy("timestamp", "desc")
|
||||
.limit(limit)
|
||||
.get();
|
||||
} else {
|
||||
resultsSnapshot = await db
|
||||
.collection(`users/${uid}/results`)
|
||||
.orderBy("timestamp", "desc")
|
||||
.startAfter(lastdoc)
|
||||
.limit(limit)
|
||||
.get();
|
||||
}
|
||||
await resultsSnapshot.forEach(async (resultDoc) => {
|
||||
let resultData = resultDoc.data();
|
||||
resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid;
|
||||
if (resultData.tags && resultData.tags.length > 0) {
|
||||
resultData.tags = resultData.tags.map((tag) => tagPairs[tag]);
|
||||
}
|
||||
newStats.completedTests++;
|
||||
if (resultData.restartCount) {
|
||||
newStats.startedTests += resultData.restartCount + 1;
|
||||
} else {
|
||||
newStats.startedTests++;
|
||||
}
|
||||
if (resultData.testDuration) {
|
||||
newStats.timeTyping += parseFloat(resultData.testDuration);
|
||||
}
|
||||
if (resultData.incompleteTestSeconds) {
|
||||
newStats.timeTyping += resultData.incompleteTestSeconds;
|
||||
}
|
||||
await mongoDB().collection("results").insertOne(resultData);
|
||||
});
|
||||
lastcount = resultsSnapshot.docs.length;
|
||||
lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1];
|
||||
total += lastcount;
|
||||
} while (lastcount > 0);
|
||||
|
||||
if (fulllog) console.log(`${uid} migrated ${total} results`);
|
||||
|
||||
mongoUser.completedTests = newStats.completedTests;
|
||||
mongoUser.startedTests = newStats.startedTests;
|
||||
mongoUser.timeTyping = newStats.timeTyping;
|
||||
|
||||
if (fulllog) console.log(`${uid} migrating user doc`);
|
||||
await mongoDB()
|
||||
.collection("users")
|
||||
.updateOne(
|
||||
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
|
||||
{
|
||||
$set: mongoUser,
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
|
||||
console.log(`${uid} migrated`);
|
||||
} catch (err) {
|
||||
console.log(`${uid} failed`);
|
||||
console.log(err);
|
||||
}
|
||||
let userend = performance.now();
|
||||
let time = (userend - userstart) / 1000;
|
||||
totalCompletionTime += time;
|
||||
// console.log(`${uid} took ${time} seconds`);
|
||||
averageCompletionTime = totalCompletionTime / currentIndex + 1;
|
||||
currentIndex++;
|
||||
let estimateSecondsLeft =
|
||||
averageCompletionTime * (totalUsers - currentIndex);
|
||||
console.log(
|
||||
`${currentIndex}/${totalUsers} users | estimated ${secondsToString(
|
||||
estimateSecondsLeft,
|
||||
true
|
||||
)} left`
|
||||
);
|
||||
|
||||
// console.log(userData);
|
||||
// let newUser;
|
||||
// try{
|
||||
// let data = userDoc.data();
|
||||
// data._id = userDoc.id;
|
||||
// newUser = new User(data);
|
||||
// newUser.uid = userDoc.id;
|
||||
// newUser.globalStats = {
|
||||
// started: userDoc.data().startedTests,
|
||||
// completed: userDoc.data().completedTests,
|
||||
// time: userDoc.data().timeTyping,
|
||||
// };
|
||||
// let tagIdDict = {};
|
||||
// let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get();
|
||||
// tagsSnapshot.forEach((tagDoc) => {
|
||||
// let formattedTag = tagDoc.data();
|
||||
// formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId
|
||||
// tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags
|
||||
// newUser.tags.push(formattedTag);
|
||||
// console.log(`Tag ${tagDoc.id} saved for user ${userCount}`);
|
||||
// });
|
||||
// let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get();
|
||||
// let resCount = 1;
|
||||
// resultsSnapshot.forEach((result) => {
|
||||
// let formattedResult = result.data();
|
||||
// if(formattedResult.tags != undefined){
|
||||
// formattedResult.tags.forEach((tag, index) => {
|
||||
// if (tagIdDict[tag])
|
||||
// formattedResult.tags[index] = tagIdDict[tag];
|
||||
// });
|
||||
// }
|
||||
// newUser.results.push(formattedResult);
|
||||
// console.log(`Result ${resCount} saved for user ${userCount}`);
|
||||
// resCount++;
|
||||
// });
|
||||
// newUser.results.sort((a, b) => {
|
||||
// return a.timestamp - b.timestamp;
|
||||
// });
|
||||
// let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get();
|
||||
// presetsSnapshot.forEach((preset) => {
|
||||
// newUser.presets.push(preset.data());
|
||||
// });
|
||||
// await newUser.save();
|
||||
// console.log(`User ${userCount} (${newUser.uid}) saved`);
|
||||
// userCount++;
|
||||
// }catch(e){
|
||||
// // throw e;
|
||||
// console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`);
|
||||
// userCount++;
|
||||
// }
|
||||
}
|
||||
console.log("Migration complete");
|
||||
// console.log('end of foreach');
|
||||
}
|
||||
// //not tested because I can't get leaderboards to work on my fork for some reason
|
||||
// db.collection("leaderboards")
|
||||
|
|
@ -235,4 +308,24 @@ async function init() {
|
|||
// process.exit(1);
|
||||
}
|
||||
|
||||
function secondsToString(sec, full = false) {
|
||||
const hours = Math.floor(sec / 3600);
|
||||
const minutes = Math.floor((sec % 3600) / 60);
|
||||
const seconds = Math.round((sec % 3600) % 60);
|
||||
let hoursString;
|
||||
let minutesString;
|
||||
let secondsString;
|
||||
hours < 10 ? (hoursString = "0" + hours) : (hoursString = hours);
|
||||
minutes < 10 ? (minutesString = "0" + minutes) : (minutesString = minutes);
|
||||
seconds < 10 && (minutes > 0 || hours > 0 || full)
|
||||
? (secondsString = "0" + seconds)
|
||||
: (secondsString = seconds);
|
||||
|
||||
let ret = "";
|
||||
if (hours > 0 || full) ret += hoursString + ":";
|
||||
if (minutes > 0 || hours > 0 || full) ret += minutesString + ":";
|
||||
ret += secondsString;
|
||||
return ret;
|
||||
}
|
||||
|
||||
init();
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue