monkeytypegame/monkeytype
1
1
Fork 0
mirror of https://github.com/monkeytypegame/monkeytype.git synced 2024-09-20 23:36:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Removed bot routes, mongo-todo, migrate; cleaned replay.js comments (#1823) by lukew3

Browse source 
* fix stop on word replay error

* Remove mongo-todo, migrate, and bot routes

* removed unecessary comments from replay.js
This commit is contained in:
Luke Weiler 2021-09-08 17:48:36 -04:00 committed by GitHub
parent 3545368ad6
commit 80908c0b6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 0 additions and 621 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
backend/api/routes/bot.js
View file

@ -1,110 +0,0 @@
async function botAuth(req, res, next) {
const authHeader = req.headers["authorization"];
const token = await admin
.auth()
.verifyIdToken(req.headers.authorization.split(" ")[1]);
if (token.isDiscordBot == null || token.isDiscordBot == false) {
return res.sendStatus(401);
} else {
next();
}
}
app.get("/getBananas/:discordId", botAuth, (req, res) => {
User.findOne({ discordId: req.params.discordId }, (err, user) => {
if (user) {
res.send({ t60bananas: user.bananas.t60bananas });
} else {
res.send({ t60bananas: 0, message: "User not found" });
}
});
});
app.get("/getUserDiscordData/:uid", botAuth, (req, res) => {
//for announceDailyLbResult
User.findOne({ uid: req.params.uid }, (err, user) => {
res.send({ name: user.name, discordId: user.discordId });
});
});
app.get("/getUserPbs/:discordId", botAuth, (req, res) => {
//for fix wpm role
User.findOne({ discordId: req.params.discordId }, (err, user) => {
if (user) {
res.send({ personalBests: user.personalBests });
} else {
res.send({ error: "No user found with that id" });
}
});
});
app.get("/getUserPbsByUid/:uid", botAuth, (req, res) => {
//for verify
User.findOne({ uid: req.params.uid }, (err, user) => {
if (user) {
res.send({ personalBests: user.personalBests });
} else {
res.send({ error: "No user found with that id" });
}
});
});
app.get("/getTimeLeaderboard/:mode2/:type", botAuth, (req, res) => {
//for lb
Leaderboard.findOne({
mode: "time",
mode2: req.params.mode2,
type: req.params.type,
}).then((err, lb) => {
//get top 10 leaderboard
lb.board.length = 10;
res.send({ board: lb.board });
});
});
app.get("/getUserByDiscordId/:discordId", botAuth, (req, res) => {
//for lb
User.findOne({ discordId: req.params.discordId }, (err, user) => {
if (user) {
res.send({ uid: user.uid });
} else {
res.send({ error: "No user found with that id" });
}
});
});
app.get("/getRecentScore/:discordId", botAuth, (req, res) => {
User.findOne({ discordId: req.params.discordId }, (err, user) => {
if (user) {
if (user.results.length == 0) {
res.send({ recentScore: -1 });
} else {
res.send({ recentScore: user.results[user.results.length - 1] });
}
} else {
res.send({ error: "No user found with that id" });
}
});
});
app.get("/getUserStats/:discordId", botAuth, (req, res) => {
//for stats
User.findOne({ discordId: req.params.discordId }, (err, user) => {
if (user) {
res.send({ stats: user.globalStats });
} else {
res.send({ error: "No user found with that id" });
}
});
});
app.post("/newBotCommand", botAuth, (req, res) => {
let newBotCommand = new BotCommand({
command: req.body.command, //is always "updateRole"
arguments: req.body.arguments,
executed: req.body.executed, //is always false
requestTimestamp: req.body.requestTimestamp,
});
newBotCommand.save();
res.status(200);
});

443
backend/migrate.js
View file

@ -1,443 +0,0 @@
const { config } = require("dotenv");
const path = require("path");
config({ path: path.join(__dirname, ".env") });
const { mongoDB } = require("./init/mongodb");
const { connectDB } = require("./init/mongodb");
const { ObjectID } = require("mongodb");
const { performance } = require("perf_hooks");
const fs = require("fs");
// const { QuerySnapshotData } = require("firebase-firestore");
console.log(config());
const admin = require("firebase-admin");
// const { User } = require("./models/user");
// const { Leaderboard } = require("./models/leaderboard");
// const { BotCommand } = require("./models/bot-command");
const serviceAccount = require("./credentials/serviceAccountKey.json");
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
});
var db = admin.firestore();
var auth = admin.auth();
process.on("SIGTERM", async () => {
console.info("SIGTERM signal received. Stopping after this user is done");
await currentUserPromise;
process.exit(1);
});
process.on("exit", async () => {
console.info("exit signal received. Stopping after this user is done");
await currentUserPromise;
process.exit(1);
});
process.on("SIGINT", async () => {
console.info("SIGINT signal received. Stopping after this user is done");
await currentUserPromise;
process.exit(1);
});
// Database should be completely clear before this is ran in order to prevent overlapping documents
// Migrate users
let currentUserPromise = null;
let resolveUser = null;
async function migrateUsers() {
// let UIDOVERRIDE = "ugbG1GiSHxVEYMDmMeLV9byeukl2";
let UIDOVERRIDE = undefined;
let lastId;
let usersSoFar = 0;
let totalUsers = 330000;
let totalCompletionTime = 0;
let averageCompletionTime = 0;
try {
let migrationStats = JSON.parse(
fs.readFileSync("./migrationStats.txt", "utf8")
);
lastId = migrationStats.uid;
usersSoFar = migrationStats.usersSoFar;
totalCompletionTime = migrationStats.totalCompletionTime;
averageCompletionTime = migrationStats.averageCompletionTime;
} catch (e) {}
let querySnapshot;
let limit = 1000;
do {
console.log("starting another loop, getting users");
if (lastId) {
let lastSnapshot = await db.collection("users").doc(lastId).get();
querySnapshot = await db
.collection("users")
.where("banned", "==", true)
.orderBy("name")
.startAfter(lastSnapshot)
.limit(limit)
.get();
} else {
querySnapshot = await db
.collection("users")
.where("banned", "==", true)
.orderBy("name")
.limit(limit)
.get();
}
// console.log('start of foreach');
console.log(`migrating ${querySnapshot.docs.length} users`);
let fulllog = false;
for (const userDoc of querySnapshot.docs) {
let userstart = performance.now();
currentUserPromise = null;
currentUserPromise = new Promise((resolve, reject) => {
resolveUser = resolve;
});
let userData = userDoc.data();
let uid = userDoc.id;
try {
let userAuth = await auth.getUser(uid);
let email = userAuth.email;
let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime();
let mongoUser = {
name: userData.name,
email: email,
addedAt: userCreatedAt,
uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
oldTypingStats: {},
};
if (userData.completedTests)
mongoUser.oldTypingStats.completedTests = userData.completedTests;
if (userData.discordId) mongoUser.discordId = userData.discordId;
if (userData.banned) mongoUser.banned = userData.banned;
if (userData.verified) mongoUser.verified = userData.verified;
//banned
//verified
if (userData.startedTests)
mongoUser.oldTypingStats.startedTests = userData.startedTests;
if (userData.timeTyping)
mongoUser.oldTypingStats.timeTyping = userData.timeTyping;
if (userData.personalBests)
mongoUser.personalBests = userData.personalBests;
let tagPairs = {};
let mongoUserTags = [];
if (fulllog) console.log(`${uid} migrating tags`);
let tagsSnapshot = await db.collection(`users/${uid}/tags`).get();
await tagsSnapshot.forEach(async (tagDoc) => {
let tagData = tagDoc.data();
let tagId = tagDoc.id;
let new_id = ObjectID();
tagPairs[tagId] = new_id;
let tagtopush = { _id: new_id, name: tagData.name };
if (tagData.personalBests)
tagtopush.personalBests = tagData.personalBests;
mongoUserTags.push(tagtopush);
});
mongoUser.tags = mongoUserTags;
// if (fulllog) console.log(`${uid} migrating config`);
// if (userData.config) {
// await mongoDB()
// .collection("configs")
// .updateOne(
// { uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
// {
// $set: {
// uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
// config: userData.config,
// },
// },
// { upsert: true }
// );
// }
// if (fulllog) console.log(`${uid} migrating presets`);
// let presetsSnapshot = await db.collection(`users/${uid}/presets`).get();
// await presetsSnapshot.forEach(async (presetDoc) => {
// let presetData = presetDoc.data();
// let newpreset = {
// uid: UIDOVERRIDE ? UIDOVERRIDE : uid,
// name: presetData.name,
// };
// if (presetData.config) newpreset.config = presetData.config;
// await mongoDB().collection("presets").insertOne(newpreset);
// });
// let lastcount = 0;
// let limit = 1000;
// let lastdoc = "start";
// let total = 0;
// let newStats = {
// completedTests: 0,
// startedTests: 0,
// timeTyping: 0,
// };
// if (fulllog) console.log(`${uid} migrating results`);
// do {
// if (fulllog) console.log(`${total} so far`);
// let resultsSnapshot;
// if (lastdoc === "start") {
// resultsSnapshot = await db
// .collection(`users/${uid}/results`)
// .orderBy("timestamp", "desc")
// .limit(limit)
// .get();
// } else {
// resultsSnapshot = await db
// .collection(`users/${uid}/results`)
// .orderBy("timestamp", "desc")
// .startAfter(lastdoc)
// .limit(limit)
// .get();
// }
// await resultsSnapshot.forEach(async (resultDoc) => {
// let resultData = resultDoc.data();
// resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid;
// if (resultData.tags && resultData.tags.length > 0) {
// resultData.tags = resultData.tags.map((tag) => tagPairs[tag]);
// }
// if (!resultData.charStats) {
// resultData.charStats = [
// resultData.correctChars,
// resultData.incorrectChars,
// ];
// }
// delete resultData.correctChars;
// delete resultData.incorrectChars;
// delete resultData.allChars;
// delete resultData.keySpacing;
// delete resultData.keyDuration;
// delete resultData.theme;
// delete resultData.name;
// //remove the default fields here
// if (resultData.bailedOut === false) delete resultData.bailedOut;
// if (resultData.blindMode === false) delete resultData.blindMode;
// if (resultData.difficulty === "normal")
// delete resultData.difficulty;
// if (resultData.funbox === "none") delete resultData.funbox;
// if (resultData.language === "english") delete resultData.language;
// if (resultData.numbers === false) delete resultData.numbers;
// if (resultData.punctuation === false) delete resultData.punctuation;
// if (resultData.mode !== "custom") delete resultData.customText;
// newStats.completedTests++;
// if (resultData.restartCount) {
// newStats.startedTests += resultData.restartCount + 1;
// } else {
// newStats.startedTests++;
// }
// if (resultData.testDuration) {
// newStats.timeTyping += parseFloat(resultData.testDuration);
// }
// if (resultData.incompleteTestSeconds) {
// newStats.timeTyping += resultData.incompleteTestSeconds;
// }
// await mongoDB().collection("results").insertOne(resultData);
// });
// lastcount = resultsSnapshot.docs.length;
// lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1];
// total += lastcount;
// } while (lastcount > 0);
// if (fulllog) console.log(`${uid} migrated ${total} results`);
// mongoUser.completedTests = newStats.completedTests;
// mongoUser.startedTests = newStats.startedTests;
// mongoUser.timeTyping = newStats.timeTyping;
if (fulllog) console.log(`${uid} migrating user doc`);
await mongoDB()
.collection("users")
.updateOne(
{ uid: UIDOVERRIDE ? UIDOVERRIDE : uid },
{
$set: mongoUser,
},
{ upsert: true }
);
console.log(`${uid} migrated \t\t ${userData.name} \t\t`);
fs.appendFileSync(
"log_success.txt",
`${uid}\t\t${userData.name}\t\t`,
"utf8"
);
} catch (err) {
console.log(`${uid} failed`);
console.log(err);
fs.appendFileSync(
"log_failed.txt",
`${uid}\t\t${err.message}\n`,
"utf8"
);
}
lastId = uid;
let userend = performance.now();
let time = (userend - userstart) / 1000;
totalCompletionTime += time;
// console.log(`${uid} took ${time} seconds`);
averageCompletionTime = totalCompletionTime / usersSoFar + 1;
usersSoFar++;
let estimateSecondsLeft =
averageCompletionTime * (totalUsers - usersSoFar);
console.log(
`${usersSoFar}/${totalUsers} users | estimated ${secondsToString(
estimateSecondsLeft,
true
)} left`
);
fs.writeFileSync(
"migrationStats.txt",
JSON.stringify({
uid,
usersSoFar,
totalCompletionTime,
averageCompletionTime,
}),
"utf8"
);
resolveUser();
// console.log(userData);
// let newUser;
// try{
// let data = userDoc.data();
// data._id = userDoc.id;
// newUser = new User(data);
// newUser.uid = userDoc.id;
// newUser.globalStats = {
// started: userDoc.data().startedTests,
// completed: userDoc.data().completedTests,
// time: userDoc.data().timeTyping,
// };
// let tagIdDict = {};
// let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get();
// tagsSnapshot.forEach((tagDoc) => {
// let formattedTag = tagDoc.data();
// formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId
// tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags
// newUser.tags.push(formattedTag);
// console.log(`Tag ${tagDoc.id} saved for user ${userCount}`);
// });
// let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get();
// let resCount = 1;
// resultsSnapshot.forEach((result) => {
// let formattedResult = result.data();
// if(formattedResult.tags != undefined){
// formattedResult.tags.forEach((tag, index) => {
// if (tagIdDict[tag])
// formattedResult.tags[index] = tagIdDict[tag];
// });
// }
// newUser.results.push(formattedResult);
// console.log(`Result ${resCount} saved for user ${userCount}`);
// resCount++;
// });
// newUser.results.sort((a, b) => {
// return a.timestamp - b.timestamp;
// });
// let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get();
// presetsSnapshot.forEach((preset) => {
// newUser.presets.push(preset.data());
// });
// await newUser.save();
// console.log(`User ${userCount} (${newUser.uid}) saved`);
// userCount++;
// }catch(e){
// // throw e;
// console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`);
// userCount++;
// }
}
} while (querySnapshot.docs.length > 0);
console.log("Migration complete");
// console.log('end of foreach');
}
// //not tested because I can't get leaderboards to work on my fork for some reason
// db.collection("leaderboards")
// .get()
// .then((leaderboardsSnapshot) => {
// leaderboardsSnapshot.forEach((lbDoc) => {
// let newLb = new Leaderboard(lbDoc.data());
// newLb.save();
// });
// });
// //migrate bot-commands
// db.collection("bot-commands")
// .get()
// .then((botCommandsSnapshot) => {
// botCommandsSnapshot.forEach((bcDoc) => {
// let newBotCommand = new BotCommand(bcDoc.data());
// newBotCommand.save();
// });
// });
//migrate public stat
async function migratePublicStats() {
db.collection("public")
.doc("stats")
.get()
.then((ret) => {
let stats = ret.data();
mongoDB()
.collection("public")
.updateOne(
{ type: "stats" },
{
$set: {
completedTests: stats.completedTests,
startedTests: stats.startedTests,
timeTyping: stats.timeTyping,
},
},
{ upsert: true }
);
});
}
async function init() {
await connectDB();
// await migratePublicStats();
await migrateUsers();
// process.exit(1);
}
function secondsToString(sec, full = false) {
const hours = Math.floor(sec / 3600);
const minutes = Math.floor((sec % 3600) / 60);
const seconds = Math.round((sec % 3600) % 60);
let hoursString;
let minutesString;
let secondsString;
hours < 10 ? (hoursString = "0" + hours) : (hoursString = hours);
minutes < 10 ? (minutesString = "0" + minutes) : (minutesString = minutes);
seconds < 10 && (minutes > 0 || hours > 0 || full)
? (secondsString = "0" + seconds)
: (secondsString = seconds);
let ret = "";
if (hours > 0 || full) ret += hoursString + ":";
if (minutes > 0 || hours > 0 || full) ret += minutesString + ":";
ret += secondsString;
return ret;
}
init();

57
backend/mongo-todo.md
View file

@ -1,57 +0,0 @@
# Mongo todo
## Todo
- Make sure that the branch is ready for deployment
- Make sure that the bot can interact with the data on the express server
- Would be optimal if the bot were to run on the same server as the express server, so that the bot wouldn't have to access data through api routes
- Determine if generatePairingCode should be removed or migrated
- This function was commented out in index.js but is used in frontend
## Bugs
- Make sure that the bot is able to interact with the mongo database
- If bot is on same server, it could work with mongo directly, otherwise, more api routes are needed
- Do names have to be made lowercase before checking if a duplicate name is found?(that is when a new user is created or username is changed)
### Minor/efficiency bugs
- Does clearDailyLeaderboards cause a memory leak?
- Is filteredResults.reverse(); in account.js going to cause efficiency issues?
- For loop in account could work backwards instead, but this would add complexity
- Why does `if (page == "account") pageTransition = false;` get rid of endless account loading bug when accessing via url
- Name is not passed in user token/auth().currentUser
- Account button sometimes shows loading infinitely after a test
- Can't navigate to user until page is refreshed
- After refresh, pr is not saved
- Can't induce this error and doesn't occur often so adding it as minor bug
- lbmemory undefined if page not refreshed after user sign up?
- If you are in first place and you place on the leaderboard but not above yourself, you may get glb undefined error
- Might also occur if you are simply on the leaderboard and make the leaderboard but not above your current position
- Doesn't happen all the time
- Hidden property of leaderboard is unused
- Verified property of user is unused, set at false by default
- Can't find where the property would be set in the code
- Is this discord verified, if so, why do you need discord verified to be on leaderboard?
- Temporarily removed from leaderboard requirements
### Functions not found anywhere except for index.js
Might need to be migrated, might not. I'm not sure why these are in the file if they are not being used.
- getAllNames
- getAllUsers
- getPatreons
- requestTest
- incrementStartedTestCounter
- incrementTestCounter
### Possibilities
- Might be worthwhile to use redis to store userdata up to a certain point
- Users who have been requested in the last hour will be stored in the redis database so that their data can be sent again without having to search a large database
- After an hour without a new request they can be removed from memory
- Create a backup system to prevent loss of data
- Users should be able to export their data themselves
- Pretty much is just the user snap but without uid
- Could split server.js into multiple files for easier code management

11
src/js/replay.js
View file

@ -1,14 +1,3 @@
/*
TODO:
Export replay as video
Export replay as typing test file?
.ttr file extension (stands for typing test record)
Should just be json, but fields should be specified by some format
metadata field with rules, website source, mode, name of typist
data field should be a list of objects, like monkeytype replay uses
signature or verfication field should be able to check file validity with server
And add ability to upload file to watch replay
*/
import config from "./config";
import * as Sound from "./sound";