From 1277d4df561c64978f90bd667c89766109879344 Mon Sep 17 00:00:00 2001 From: Miodec Date: Mon, 9 Aug 2021 23:39:13 +0100 Subject: [PATCH] more migration script updates: catching script stopping and waiting for the current user to finish running the script until everything is done --- .gitignore | 3 +- backend/migrate.js | 536 +++++++++++++++++++++++++-------------------- 2 files changed, 297 insertions(+), 242 deletions(-) diff --git a/.gitignore b/.gitignore index e7adbd09f..4edee553c 100644 --- a/.gitignore +++ b/.gitignore @@ -89,4 +89,5 @@ purgeCfCache.sh backend/credentials backend/.env -static/adtest.html \ No newline at end of file +static/adtest.html +backend/migrationStats.txt diff --git a/backend/migrate.js b/backend/migrate.js index 03cdf9b56..30dddde72 100644 --- a/backend/migrate.js +++ b/backend/migrate.js @@ -26,272 +26,326 @@ admin.initializeApp({ var db = admin.firestore(); var auth = admin.auth(); +process.on("SIGTERM", async () => { + console.info("SIGTERM signal received. Stopping after this user is done"); + await currentUserPromise; + process.exit(1); +}); + +process.on("exit", async () => { + console.info("exit signal received. Stopping after this user is done"); + await currentUserPromise; + process.exit(1); +}); + +process.on("SIGINT", async () => { + console.info("SIGINT signal received. Stopping after this user is done"); + await currentUserPromise; + process.exit(1); +}); + // Database should be completely clear before this is ran in order to prevent overlapping documents // Migrate users + +let currentUserPromise = null; + +let resolveUser = null; + async function migrateUsers() { // let UIDOVERRIDE = "ugbG1GiSHxVEYMDmMeLV9byeukl2"; let UIDOVERRIDE = undefined; let lastId; - try { - lastId = fs.readFileSync("./lastId.txt", "utf8"); - } catch (e) {} - let querySnapshot; - if (lastId) { - let lastSnapshot = await db.collection("users").doc(lastId).get(); - querySnapshot = await db - .collection("users") - // .where("name", "==", "Miodec") - .orderBy("name") - .startAfter(lastSnapshot) - .limit(1) - .get(); - } else { - querySnapshot = await db - .collection("users") - // .where("name", "==", "Miodec") - .orderBy("name") - .limit(1) - .get(); - } - // console.log('start of foreach'); - console.log(`migrating ${querySnapshot.docs.length} users`); - let fulllog = false; - let currentIndex = 0; - let totalUsers = querySnapshot.docs.length; + let usersSoFar = 0; + let totalUsers = 330000; let totalCompletionTime = 0; let averageCompletionTime = 0; - for (const userDoc of querySnapshot.docs) { - let userstart = performance.now(); - let userData = userDoc.data(); - let uid = userDoc.id; - try { - let userAuth = await auth.getUser(uid); - let email = userAuth.email; - let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime(); - - let mongoUser = { - name: userData.name, - email: email, - addedAt: userCreatedAt, - uid: UIDOVERRIDE ? UIDOVERRIDE : uid, - oldTypingStats: {}, - }; - - if (userData.completedTests) - mongoUser.oldTypingStats.completedTests = userData.completedTests; - if (userData.discordId) mongoUser.discordId = userData.discordId; - if (userData.startedTests) - mongoUser.oldTypingStats.startedTests = userData.startedTests; - if (userData.timeTyping) - mongoUser.oldTypingStats.timeTyping = userData.timeTyping; - - if (userData.personalBests) - mongoUser.personalBests = userData.personalBests; - - let tagPairs = {}; - - let mongoUserTags = []; - - if (fulllog) console.log(`${uid} migrating tags`); - let tagsSnapshot = await db.collection(`users/${uid}/tags`).get(); - await tagsSnapshot.forEach(async (tagDoc) => { - let tagData = tagDoc.data(); - let tagId = tagDoc.id; - let new_id = ObjectID(); - tagPairs[tagId] = new_id; - let tagtopush = { _id: new_id, name: tagData.name }; - if (tagData.personalBests) - tagtopush.personalBests = tagData.personalBests; - mongoUserTags.push(tagtopush); + try { + let migrationStats = JSON.parse( + fs.readFileSync("./migrationStats.txt", "utf8") + ); + lastId = migrationStats.uid; + usersSoFar = migrationStats.usersSoFar; + totalCompletionTime = migrationStats.totalCompletionTime; + averageCompletionTime = migrationStats.averageCompletionTime; + } catch (e) {} + let querySnapshot; + let limit = 1; + do { + console.log("starting another loop, getting users"); + if (lastId) { + let lastSnapshot = await db.collection("users").doc(lastId).get(); + querySnapshot = await db + .collection("users") + // .where("name", "==", "Miodec") + .orderBy("name") + .startAfter(lastSnapshot) + .limit(limit) + .get(); + } else { + querySnapshot = await db + .collection("users") + // .where("name", "==", "Miodec") + .orderBy("name") + .limit(limit) + .get(); + } + // console.log('start of foreach'); + console.log(`migrating ${querySnapshot.docs.length} users`); + let fulllog = false; + for (const userDoc of querySnapshot.docs) { + let userstart = performance.now(); + currentUserPromise = null; + currentUserPromise = new Promise((resolve, reject) => { + resolveUser = resolve; }); + let userData = userDoc.data(); + let uid = userDoc.id; + try { + let userAuth = await auth.getUser(uid); + let email = userAuth.email; + let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime(); - mongoUser.tags = mongoUserTags; + let mongoUser = { + name: userData.name, + email: email, + addedAt: userCreatedAt, + uid: UIDOVERRIDE ? UIDOVERRIDE : uid, + oldTypingStats: {}, + }; - if (fulllog) console.log(`${uid} migrating config`); - if (userData.config) { + if (userData.completedTests) + mongoUser.oldTypingStats.completedTests = userData.completedTests; + if (userData.discordId) mongoUser.discordId = userData.discordId; + if (userData.startedTests) + mongoUser.oldTypingStats.startedTests = userData.startedTests; + if (userData.timeTyping) + mongoUser.oldTypingStats.timeTyping = userData.timeTyping; + + if (userData.personalBests) + mongoUser.personalBests = userData.personalBests; + + let tagPairs = {}; + + let mongoUserTags = []; + + if (fulllog) console.log(`${uid} migrating tags`); + let tagsSnapshot = await db.collection(`users/${uid}/tags`).get(); + await tagsSnapshot.forEach(async (tagDoc) => { + let tagData = tagDoc.data(); + let tagId = tagDoc.id; + let new_id = ObjectID(); + tagPairs[tagId] = new_id; + let tagtopush = { _id: new_id, name: tagData.name }; + if (tagData.personalBests) + tagtopush.personalBests = tagData.personalBests; + mongoUserTags.push(tagtopush); + }); + + mongoUser.tags = mongoUserTags; + + if (fulllog) console.log(`${uid} migrating config`); + if (userData.config) { + await mongoDB() + .collection("configs") + .updateOne( + { uid: UIDOVERRIDE ? UIDOVERRIDE : uid }, + { + $set: { + uid: UIDOVERRIDE ? UIDOVERRIDE : uid, + config: userData.config, + }, + }, + { upsert: true } + ); + } + + if (fulllog) console.log(`${uid} migrating presets`); + let presetsSnapshot = await db.collection(`users/${uid}/presets`).get(); + await presetsSnapshot.forEach(async (presetDoc) => { + let presetData = presetDoc.data(); + let newpreset = { + uid: UIDOVERRIDE ? UIDOVERRIDE : uid, + name: presetData.name, + }; + if (presetData.config) newpreset.config = presetData.config; + await mongoDB().collection("presets").insertOne(newpreset); + }); + + let lastcount = 0; + let limit = 1000; + let lastdoc = "start"; + let total = 0; + let newStats = { + completedTests: 0, + startedTests: 0, + timeTyping: 0, + }; + if (fulllog) console.log(`${uid} migrating results`); + do { + if (fulllog) console.log(`${total} so far`); + let resultsSnapshot; + if (lastdoc === "start") { + resultsSnapshot = await db + .collection(`users/${uid}/results`) + .orderBy("timestamp", "desc") + .limit(limit) + .get(); + } else { + resultsSnapshot = await db + .collection(`users/${uid}/results`) + .orderBy("timestamp", "desc") + .startAfter(lastdoc) + .limit(limit) + .get(); + } + await resultsSnapshot.forEach(async (resultDoc) => { + let resultData = resultDoc.data(); + resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid; + if (resultData.tags && resultData.tags.length > 0) { + resultData.tags = resultData.tags.map((tag) => tagPairs[tag]); + } + if (!resultData.charStats) { + resultData.charStats = [ + resultData.correctChars, + resultData.incorrectChars, + ]; + } + delete resultData.correctChars; + delete resultData.incorrectChars; + resultData.charStats = newStats.completedTests++; + if (resultData.restartCount) { + newStats.startedTests += resultData.restartCount + 1; + } else { + newStats.startedTests++; + } + if (resultData.testDuration) { + newStats.timeTyping += parseFloat(resultData.testDuration); + } + if (resultData.incompleteTestSeconds) { + newStats.timeTyping += resultData.incompleteTestSeconds; + } + await mongoDB().collection("results").insertOne(resultData); + }); + lastcount = resultsSnapshot.docs.length; + lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1]; + total += lastcount; + } while (lastcount > 0); + + if (fulllog) console.log(`${uid} migrated ${total} results`); + + mongoUser.completedTests = newStats.completedTests; + mongoUser.startedTests = newStats.startedTests; + mongoUser.timeTyping = newStats.timeTyping; + + if (fulllog) console.log(`${uid} migrating user doc`); await mongoDB() - .collection("configs") + .collection("users") .updateOne( { uid: UIDOVERRIDE ? UIDOVERRIDE : uid }, { - $set: { - uid: UIDOVERRIDE ? UIDOVERRIDE : uid, - config: userData.config, - }, + $set: mongoUser, }, { upsert: true } ); - } - if (fulllog) console.log(`${uid} migrating presets`); - let presetsSnapshot = await db.collection(`users/${uid}/presets`).get(); - await presetsSnapshot.forEach(async (presetDoc) => { - let presetData = presetDoc.data(); - let newpreset = { - uid: UIDOVERRIDE ? UIDOVERRIDE : uid, - name: presetData.name, - }; - if (presetData.config) newpreset.config = presetData.config; - await mongoDB().collection("presets").insertOne(newpreset); - }); - - let lastcount = 0; - let limit = 1000; - let lastdoc = "start"; - let total = 0; - let newStats = { - completedTests: 0, - startedTests: 0, - timeTyping: 0, - }; - if (fulllog) console.log(`${uid} migrating results`); - do { - if (fulllog) console.log(`${total} so far`); - let resultsSnapshot; - if (lastdoc === "start") { - resultsSnapshot = await db - .collection(`users/${uid}/results`) - .orderBy("timestamp", "desc") - .limit(limit) - .get(); - } else { - resultsSnapshot = await db - .collection(`users/${uid}/results`) - .orderBy("timestamp", "desc") - .startAfter(lastdoc) - .limit(limit) - .get(); - } - await resultsSnapshot.forEach(async (resultDoc) => { - let resultData = resultDoc.data(); - resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid; - if (resultData.tags && resultData.tags.length > 0) { - resultData.tags = resultData.tags.map((tag) => tagPairs[tag]); - } - if (!resultData.charStats) { - resultData.charStats = [ - resultData.correctChars, - resultData.incorrectChars, - ]; - } - delete resultData.correctChars; - delete resultData.incorrectChars; - resultData.charStats = newStats.completedTests++; - if (resultData.restartCount) { - newStats.startedTests += resultData.restartCount + 1; - } else { - newStats.startedTests++; - } - if (resultData.testDuration) { - newStats.timeTyping += parseFloat(resultData.testDuration); - } - if (resultData.incompleteTestSeconds) { - newStats.timeTyping += resultData.incompleteTestSeconds; - } - await mongoDB().collection("results").insertOne(resultData); - }); - lastcount = resultsSnapshot.docs.length; - lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1]; - total += lastcount; - } while (lastcount > 0); - - if (fulllog) console.log(`${uid} migrated ${total} results`); - - mongoUser.completedTests = newStats.completedTests; - mongoUser.startedTests = newStats.startedTests; - mongoUser.timeTyping = newStats.timeTyping; - - if (fulllog) console.log(`${uid} migrating user doc`); - await mongoDB() - .collection("users") - .updateOne( - { uid: UIDOVERRIDE ? UIDOVERRIDE : uid }, - { - $set: mongoUser, - }, - { upsert: true } + console.log( + `${uid} migrated \t\t ${userData.name} \t\t ${total} results` ); - + fs.appendFileSync( + "log_success.txt", + `${uid}\t\t${userData.name}\t\t${total}\n`, + "utf8" + ); + } catch (err) { + console.log(`${uid} failed`); + console.log(err); + fs.appendFileSync( + "log_failed.txt", + `${uid}\t\t${err.message}\n`, + "utf8" + ); + } + lastId = uid; + let userend = performance.now(); + let time = (userend - userstart) / 1000; + totalCompletionTime += time; + // console.log(`${uid} took ${time} seconds`); + averageCompletionTime = totalCompletionTime / usersSoFar + 1; + usersSoFar++; + let estimateSecondsLeft = + averageCompletionTime * (totalUsers - usersSoFar); console.log( - `${uid} migrated \t\t ${userData.name} \t\t ${total} results` + `${usersSoFar}/${totalUsers} users | estimated ${secondsToString( + estimateSecondsLeft, + true + )} left` ); - fs.appendFileSync( - "log_success.txt", - `${uid}\t\t${userData.name}\t\t${total}\n`, + + fs.writeFileSync( + "migrationStats.txt", + JSON.stringify({ + uid, + usersSoFar, + totalCompletionTime, + averageCompletionTime, + }), "utf8" ); - } catch (err) { - console.log(`${uid} failed`); - console.log(err); - fs.appendFileSync("log_failed.txt", `${uid}\t\t${err.message}\n`, "utf8"); - } - fs.writeFileSync("lastId.txt", uid, "utf8"); - let userend = performance.now(); - let time = (userend - userstart) / 1000; - totalCompletionTime += time; - // console.log(`${uid} took ${time} seconds`); - averageCompletionTime = totalCompletionTime / currentIndex + 1; - currentIndex++; - let estimateSecondsLeft = - averageCompletionTime * (totalUsers - currentIndex); - console.log( - `${currentIndex}/${totalUsers} users | estimated ${secondsToString( - estimateSecondsLeft, - true - )} left` - ); - // console.log(userData); - // let newUser; - // try{ - // let data = userDoc.data(); - // data._id = userDoc.id; - // newUser = new User(data); - // newUser.uid = userDoc.id; - // newUser.globalStats = { - // started: userDoc.data().startedTests, - // completed: userDoc.data().completedTests, - // time: userDoc.data().timeTyping, - // }; - // let tagIdDict = {}; - // let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get(); - // tagsSnapshot.forEach((tagDoc) => { - // let formattedTag = tagDoc.data(); - // formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId - // tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags - // newUser.tags.push(formattedTag); - // console.log(`Tag ${tagDoc.id} saved for user ${userCount}`); - // }); - // let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get(); - // let resCount = 1; - // resultsSnapshot.forEach((result) => { - // let formattedResult = result.data(); - // if(formattedResult.tags != undefined){ - // formattedResult.tags.forEach((tag, index) => { - // if (tagIdDict[tag]) - // formattedResult.tags[index] = tagIdDict[tag]; - // }); - // } - // newUser.results.push(formattedResult); - // console.log(`Result ${resCount} saved for user ${userCount}`); - // resCount++; - // }); - // newUser.results.sort((a, b) => { - // return a.timestamp - b.timestamp; - // }); - // let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get(); - // presetsSnapshot.forEach((preset) => { - // newUser.presets.push(preset.data()); - // }); - // await newUser.save(); - // console.log(`User ${userCount} (${newUser.uid}) saved`); - // userCount++; - // }catch(e){ - // // throw e; - // console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`); - // userCount++; - // } - } + resolveUser(); + + // console.log(userData); + // let newUser; + // try{ + // let data = userDoc.data(); + // data._id = userDoc.id; + // newUser = new User(data); + // newUser.uid = userDoc.id; + // newUser.globalStats = { + // started: userDoc.data().startedTests, + // completed: userDoc.data().completedTests, + // time: userDoc.data().timeTyping, + // }; + // let tagIdDict = {}; + // let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get(); + // tagsSnapshot.forEach((tagDoc) => { + // let formattedTag = tagDoc.data(); + // formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId + // tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags + // newUser.tags.push(formattedTag); + // console.log(`Tag ${tagDoc.id} saved for user ${userCount}`); + // }); + // let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get(); + // let resCount = 1; + // resultsSnapshot.forEach((result) => { + // let formattedResult = result.data(); + // if(formattedResult.tags != undefined){ + // formattedResult.tags.forEach((tag, index) => { + // if (tagIdDict[tag]) + // formattedResult.tags[index] = tagIdDict[tag]; + // }); + // } + // newUser.results.push(formattedResult); + // console.log(`Result ${resCount} saved for user ${userCount}`); + // resCount++; + // }); + // newUser.results.sort((a, b) => { + // return a.timestamp - b.timestamp; + // }); + // let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get(); + // presetsSnapshot.forEach((preset) => { + // newUser.presets.push(preset.data()); + // }); + // await newUser.save(); + // console.log(`User ${userCount} (${newUser.uid}) saved`); + // userCount++; + // }catch(e){ + // // throw e; + // console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`); + // userCount++; + // } + } + } while (querySnapshot.docs.length > 0); console.log("Migration complete"); // console.log('end of foreach');