more migration script updates:

catching script stopping and waiting for the current user to finish
running the script until everything is done
This commit is contained in:
Miodec 2021-08-09 23:39:13 +01:00
parent 26d698c366
commit 1277d4df56
2 changed files with 297 additions and 242 deletions

3
.gitignore vendored
View file

@ -89,4 +89,5 @@ purgeCfCache.sh
backend/credentials
backend/.env
static/adtest.html
static/adtest.html
backend/migrationStats.txt

View file

@ -26,272 +26,326 @@ admin.initializeApp({
var db = admin.firestore();
var auth = admin.auth();
process.on("SIGTERM", async () => {
console.info("SIGTERM signal received. Stopping after this user is done");
await currentUserPromise;
process.exit(1);
});
process.on("exit", async () => {
console.info("exit signal received. Stopping after this user is done");
await currentUserPromise;
process.exit(1);
});
process.on("SIGINT", async () => {
console.info("SIGINT signal received. Stopping after this user is done");
await currentUserPromise;
process.exit(1);
});
// Database should be completely clear before this is ran in order to prevent overlapping documents
// Migrate users
let currentUserPromise = null;
let resolveUser = null;
async function migrateUsers() {
// let UIDOVERRIDE = "ugbG1GiSHxVEYMDmMeLV9byeukl2";
let UIDOVERRIDE = undefined;
let lastId;
try {
lastId = fs.readFileSync("./lastId.txt", "utf8");
} catch (e) {}
let querySnapshot;
if (lastId) {
let lastSnapshot = await db.collection("users").doc(lastId).get();
querySnapshot = await db
.collection("users")
// .where("name", "==", "Miodec")
.orderBy("name")
.startAfter(lastSnapshot)
.limit(1)
.get();
} else {
querySnapshot = await db
.collection("users")
// .where("name", "==", "Miodec")
.orderBy("name")
.limit(1)
.get();
}
// console.log('start of foreach');
console.log(`migrating ${querySnapshot.docs.length} users`);
let fulllog = false;
let currentIndex = 0;
let totalUsers = querySnapshot.docs.length;
let usersSoFar = 0;
let totalUsers = 330000;
let totalCompletionTime = 0;
let averageCompletionTime = 0;
for (const userDoc of querySnapshot.docs) {
let userstart = performance.now();
let userData = userDoc.data();
let uid = userDoc.id;
try {
let userAuth = await auth.getUser(uid);
let email = userAuth.email;
let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime();
let mongoUser = {
name: userData.name,
email: email,
addedAt: userCreatedAt,
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
oldTypingStats: {},
};
if (userData.completedTests)
mongoUser.oldTypingStats.completedTests = userData.completedTests;
if (userData.discordId) mongoUser.discordId = userData.discordId;
if (userData.startedTests)
mongoUser.oldTypingStats.startedTests = userData.startedTests;
if (userData.timeTyping)
mongoUser.oldTypingStats.timeTyping = userData.timeTyping;
if (userData.personalBests)
mongoUser.personalBests = userData.personalBests;
let tagPairs = {};
let mongoUserTags = [];
if (fulllog) console.log(`${uid} migrating tags`);
let tagsSnapshot = await db.collection(`users/${uid}/tags`).get();
await tagsSnapshot.forEach(async (tagDoc) => {
let tagData = tagDoc.data();
let tagId = tagDoc.id;
let new_id = ObjectID();
tagPairs[tagId] = new_id;
let tagtopush = { _id: new_id, name: tagData.name };
if (tagData.personalBests)
tagtopush.personalBests = tagData.personalBests;
mongoUserTags.push(tagtopush);
try {
let migrationStats = JSON.parse(
fs.readFileSync("./migrationStats.txt", "utf8")
);
lastId = migrationStats.uid;
usersSoFar = migrationStats.usersSoFar;
totalCompletionTime = migrationStats.totalCompletionTime;
averageCompletionTime = migrationStats.averageCompletionTime;
} catch (e) {}
let querySnapshot;
let limit = 1;
do {
console.log("starting another loop, getting users");
if (lastId) {
let lastSnapshot = await db.collection("users").doc(lastId).get();
querySnapshot = await db
.collection("users")
// .where("name", "==", "Miodec")
.orderBy("name")
.startAfter(lastSnapshot)
.limit(limit)
.get();
} else {
querySnapshot = await db
.collection("users")
// .where("name", "==", "Miodec")
.orderBy("name")
.limit(limit)
.get();
}
// console.log('start of foreach');
console.log(`migrating ${querySnapshot.docs.length} users`);
let fulllog = false;
for (const userDoc of querySnapshot.docs) {
let userstart = performance.now();
currentUserPromise = null;
currentUserPromise = new Promise((resolve, reject) => {
resolveUser = resolve;
});
let userData = userDoc.data();
let uid = userDoc.id;
try {
let userAuth = await auth.getUser(uid);
let email = userAuth.email;
let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime();
mongoUser.tags = mongoUserTags;
let mongoUser = {
name: userData.name,
email: email,
addedAt: userCreatedAt,
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
oldTypingStats: {},
};
if (fulllog) console.log(`${uid} migrating config`);
if (userData.config) {
if (userData.completedTests)
mongoUser.oldTypingStats.completedTests = userData.completedTests;
if (userData.discordId) mongoUser.discordId = userData.discordId;
if (userData.startedTests)
mongoUser.oldTypingStats.startedTests = userData.startedTests;
if (userData.timeTyping)
mongoUser.oldTypingStats.timeTyping = userData.timeTyping;
if (userData.personalBests)
mongoUser.personalBests = userData.personalBests;
let tagPairs = {};
let mongoUserTags = [];
if (fulllog) console.log(`${uid} migrating tags`);
let tagsSnapshot = await db.collection(`users/${uid}/tags`).get();
await tagsSnapshot.forEach(async (tagDoc) => {
let tagData = tagDoc.data();
let tagId = tagDoc.id;
let new_id = ObjectID();
tagPairs[tagId] = new_id;
let tagtopush = { _id: new_id, name: tagData.name };
if (tagData.personalBests)
tagtopush.personalBests = tagData.personalBests;
mongoUserTags.push(tagtopush);
});
mongoUser.tags = mongoUserTags;
if (fulllog) console.log(`${uid} migrating config`);
if (userData.config) {
await mongoDB()
.collection("configs")
.updateOne(
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
{
$set: {
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
config: userData.config,
},
},
{ upsert: true }
);
}
if (fulllog) console.log(`${uid} migrating presets`);
let presetsSnapshot = await db.collection(`users/${uid}/presets`).get();
await presetsSnapshot.forEach(async (presetDoc) => {
let presetData = presetDoc.data();
let newpreset = {
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
name: presetData.name,
};
if (presetData.config) newpreset.config = presetData.config;
await mongoDB().collection("presets").insertOne(newpreset);
});
let lastcount = 0;
let limit = 1000;
let lastdoc = "start";
let total = 0;
let newStats = {
completedTests: 0,
startedTests: 0,
timeTyping: 0,
};
if (fulllog) console.log(`${uid} migrating results`);
do {
if (fulllog) console.log(`${total} so far`);
let resultsSnapshot;
if (lastdoc === "start") {
resultsSnapshot = await db
.collection(`users/${uid}/results`)
.orderBy("timestamp", "desc")
.limit(limit)
.get();
} else {
resultsSnapshot = await db
.collection(`users/${uid}/results`)
.orderBy("timestamp", "desc")
.startAfter(lastdoc)
.limit(limit)
.get();
}
await resultsSnapshot.forEach(async (resultDoc) => {
let resultData = resultDoc.data();
resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid;
if (resultData.tags && resultData.tags.length > 0) {
resultData.tags = resultData.tags.map((tag) => tagPairs[tag]);
}
if (!resultData.charStats) {
resultData.charStats = [
resultData.correctChars,
resultData.incorrectChars,
];
}
delete resultData.correctChars;
delete resultData.incorrectChars;
resultData.charStats = newStats.completedTests++;
if (resultData.restartCount) {
newStats.startedTests += resultData.restartCount + 1;
} else {
newStats.startedTests++;
}
if (resultData.testDuration) {
newStats.timeTyping += parseFloat(resultData.testDuration);
}
if (resultData.incompleteTestSeconds) {
newStats.timeTyping += resultData.incompleteTestSeconds;
}
await mongoDB().collection("results").insertOne(resultData);
});
lastcount = resultsSnapshot.docs.length;
lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1];
total += lastcount;
} while (lastcount > 0);
if (fulllog) console.log(`${uid} migrated ${total} results`);
mongoUser.completedTests = newStats.completedTests;
mongoUser.startedTests = newStats.startedTests;
mongoUser.timeTyping = newStats.timeTyping;
if (fulllog) console.log(`${uid} migrating user doc`);
await mongoDB()
.collection("configs")
.collection("users")
.updateOne(
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
{
$set: {
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
config: userData.config,
},
$set: mongoUser,
},
{ upsert: true }
);
}
if (fulllog) console.log(`${uid} migrating presets`);
let presetsSnapshot = await db.collection(`users/${uid}/presets`).get();
await presetsSnapshot.forEach(async (presetDoc) => {
let presetData = presetDoc.data();
let newpreset = {
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
name: presetData.name,
};
if (presetData.config) newpreset.config = presetData.config;
await mongoDB().collection("presets").insertOne(newpreset);
});
let lastcount = 0;
let limit = 1000;
let lastdoc = "start";
let total = 0;
let newStats = {
completedTests: 0,
startedTests: 0,
timeTyping: 0,
};
if (fulllog) console.log(`${uid} migrating results`);
do {
if (fulllog) console.log(`${total} so far`);
let resultsSnapshot;
if (lastdoc === "start") {
resultsSnapshot = await db
.collection(`users/${uid}/results`)
.orderBy("timestamp", "desc")
.limit(limit)
.get();
} else {
resultsSnapshot = await db
.collection(`users/${uid}/results`)
.orderBy("timestamp", "desc")
.startAfter(lastdoc)
.limit(limit)
.get();
}
await resultsSnapshot.forEach(async (resultDoc) => {
let resultData = resultDoc.data();
resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid;
if (resultData.tags && resultData.tags.length > 0) {
resultData.tags = resultData.tags.map((tag) => tagPairs[tag]);
}
if (!resultData.charStats) {
resultData.charStats = [
resultData.correctChars,
resultData.incorrectChars,
];
}
delete resultData.correctChars;
delete resultData.incorrectChars;
resultData.charStats = newStats.completedTests++;
if (resultData.restartCount) {
newStats.startedTests += resultData.restartCount + 1;
} else {
newStats.startedTests++;
}
if (resultData.testDuration) {
newStats.timeTyping += parseFloat(resultData.testDuration);
}
if (resultData.incompleteTestSeconds) {
newStats.timeTyping += resultData.incompleteTestSeconds;
}
await mongoDB().collection("results").insertOne(resultData);
});
lastcount = resultsSnapshot.docs.length;
lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1];
total += lastcount;
} while (lastcount > 0);
if (fulllog) console.log(`${uid} migrated ${total} results`);
mongoUser.completedTests = newStats.completedTests;
mongoUser.startedTests = newStats.startedTests;
mongoUser.timeTyping = newStats.timeTyping;
if (fulllog) console.log(`${uid} migrating user doc`);
await mongoDB()
.collection("users")
.updateOne(
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
{
$set: mongoUser,
},
{ upsert: true }
console.log(
`${uid} migrated \t\t ${userData.name} \t\t ${total} results`
);
fs.appendFileSync(
"log_success.txt",
`${uid}\t\t${userData.name}\t\t${total}\n`,
"utf8"
);
} catch (err) {
console.log(`${uid} failed`);
console.log(err);
fs.appendFileSync(
"log_failed.txt",
`${uid}\t\t${err.message}\n`,
"utf8"
);
}
lastId = uid;
let userend = performance.now();
let time = (userend - userstart) / 1000;
totalCompletionTime += time;
// console.log(`${uid} took ${time} seconds`);
averageCompletionTime = totalCompletionTime / usersSoFar + 1;
usersSoFar++;
let estimateSecondsLeft =
averageCompletionTime * (totalUsers - usersSoFar);
console.log(
`${uid} migrated \t\t ${userData.name} \t\t ${total} results`
`${usersSoFar}/${totalUsers} users | estimated ${secondsToString(
estimateSecondsLeft,
true
)} left`
);
fs.appendFileSync(
"log_success.txt",
`${uid}\t\t${userData.name}\t\t${total}\n`,
fs.writeFileSync(
"migrationStats.txt",
JSON.stringify({
uid,
usersSoFar,
totalCompletionTime,
averageCompletionTime,
}),
"utf8"
);
} catch (err) {
console.log(`${uid} failed`);
console.log(err);
fs.appendFileSync("log_failed.txt", `${uid}\t\t${err.message}\n`, "utf8");
}
fs.writeFileSync("lastId.txt", uid, "utf8");
let userend = performance.now();
let time = (userend - userstart) / 1000;
totalCompletionTime += time;
// console.log(`${uid} took ${time} seconds`);
averageCompletionTime = totalCompletionTime / currentIndex + 1;
currentIndex++;
let estimateSecondsLeft =
averageCompletionTime * (totalUsers - currentIndex);
console.log(
`${currentIndex}/${totalUsers} users | estimated ${secondsToString(
estimateSecondsLeft,
true
)} left`
);
// console.log(userData);
// let newUser;
// try{
// let data = userDoc.data();
// data._id = userDoc.id;
// newUser = new User(data);
// newUser.uid = userDoc.id;
// newUser.globalStats = {
// started: userDoc.data().startedTests,
// completed: userDoc.data().completedTests,
// time: userDoc.data().timeTyping,
// };
// let tagIdDict = {};
// let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get();
// tagsSnapshot.forEach((tagDoc) => {
// let formattedTag = tagDoc.data();
// formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId
// tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags
// newUser.tags.push(formattedTag);
// console.log(`Tag ${tagDoc.id} saved for user ${userCount}`);
// });
// let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get();
// let resCount = 1;
// resultsSnapshot.forEach((result) => {
// let formattedResult = result.data();
// if(formattedResult.tags != undefined){
// formattedResult.tags.forEach((tag, index) => {
// if (tagIdDict[tag])
// formattedResult.tags[index] = tagIdDict[tag];
// });
// }
// newUser.results.push(formattedResult);
// console.log(`Result ${resCount} saved for user ${userCount}`);
// resCount++;
// });
// newUser.results.sort((a, b) => {
// return a.timestamp - b.timestamp;
// });
// let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get();
// presetsSnapshot.forEach((preset) => {
// newUser.presets.push(preset.data());
// });
// await newUser.save();
// console.log(`User ${userCount} (${newUser.uid}) saved`);
// userCount++;
// }catch(e){
// // throw e;
// console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`);
// userCount++;
// }
}
resolveUser();
// console.log(userData);
// let newUser;
// try{
// let data = userDoc.data();
// data._id = userDoc.id;
// newUser = new User(data);
// newUser.uid = userDoc.id;
// newUser.globalStats = {
// started: userDoc.data().startedTests,
// completed: userDoc.data().completedTests,
// time: userDoc.data().timeTyping,
// };
// let tagIdDict = {};
// let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get();
// tagsSnapshot.forEach((tagDoc) => {
// let formattedTag = tagDoc.data();
// formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId
// tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags
// newUser.tags.push(formattedTag);
// console.log(`Tag ${tagDoc.id} saved for user ${userCount}`);
// });
// let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get();
// let resCount = 1;
// resultsSnapshot.forEach((result) => {
// let formattedResult = result.data();
// if(formattedResult.tags != undefined){
// formattedResult.tags.forEach((tag, index) => {
// if (tagIdDict[tag])
// formattedResult.tags[index] = tagIdDict[tag];
// });
// }
// newUser.results.push(formattedResult);
// console.log(`Result ${resCount} saved for user ${userCount}`);
// resCount++;
// });
// newUser.results.sort((a, b) => {
// return a.timestamp - b.timestamp;
// });
// let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get();
// presetsSnapshot.forEach((preset) => {
// newUser.presets.push(preset.data());
// });
// await newUser.save();
// console.log(`User ${userCount} (${newUser.uid}) saved`);
// userCount++;
// }catch(e){
// // throw e;
// console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`);
// userCount++;
// }
}
} while (querySnapshot.docs.length > 0);
console.log("Migration complete");
// console.log('end of foreach');