diff --git a/backend/api/routes/bot.js b/backend/api/routes/bot.js deleted file mode 100644 index b7081b964..000000000 --- a/backend/api/routes/bot.js +++ /dev/null @@ -1,110 +0,0 @@ -async function botAuth(req, res, next) { - const authHeader = req.headers["authorization"]; - const token = await admin - .auth() - .verifyIdToken(req.headers.authorization.split(" ")[1]); - if (token.isDiscordBot == null || token.isDiscordBot == false) { - return res.sendStatus(401); - } else { - next(); - } -} - -app.get("/getBananas/:discordId", botAuth, (req, res) => { - User.findOne({ discordId: req.params.discordId }, (err, user) => { - if (user) { - res.send({ t60bananas: user.bananas.t60bananas }); - } else { - res.send({ t60bananas: 0, message: "User not found" }); - } - }); -}); - -app.get("/getUserDiscordData/:uid", botAuth, (req, res) => { - //for announceDailyLbResult - User.findOne({ uid: req.params.uid }, (err, user) => { - res.send({ name: user.name, discordId: user.discordId }); - }); -}); - -app.get("/getUserPbs/:discordId", botAuth, (req, res) => { - //for fix wpm role - User.findOne({ discordId: req.params.discordId }, (err, user) => { - if (user) { - res.send({ personalBests: user.personalBests }); - } else { - res.send({ error: "No user found with that id" }); - } - }); -}); - -app.get("/getUserPbsByUid/:uid", botAuth, (req, res) => { - //for verify - User.findOne({ uid: req.params.uid }, (err, user) => { - if (user) { - res.send({ personalBests: user.personalBests }); - } else { - res.send({ error: "No user found with that id" }); - } - }); -}); - -app.get("/getTimeLeaderboard/:mode2/:type", botAuth, (req, res) => { - //for lb - Leaderboard.findOne({ - mode: "time", - mode2: req.params.mode2, - type: req.params.type, - }).then((err, lb) => { - //get top 10 leaderboard - lb.board.length = 10; - res.send({ board: lb.board }); - }); -}); - -app.get("/getUserByDiscordId/:discordId", botAuth, (req, res) => { - //for lb - User.findOne({ discordId: req.params.discordId }, (err, user) => { - if (user) { - res.send({ uid: user.uid }); - } else { - res.send({ error: "No user found with that id" }); - } - }); -}); - -app.get("/getRecentScore/:discordId", botAuth, (req, res) => { - User.findOne({ discordId: req.params.discordId }, (err, user) => { - if (user) { - if (user.results.length == 0) { - res.send({ recentScore: -1 }); - } else { - res.send({ recentScore: user.results[user.results.length - 1] }); - } - } else { - res.send({ error: "No user found with that id" }); - } - }); -}); - -app.get("/getUserStats/:discordId", botAuth, (req, res) => { - //for stats - User.findOne({ discordId: req.params.discordId }, (err, user) => { - if (user) { - res.send({ stats: user.globalStats }); - } else { - res.send({ error: "No user found with that id" }); - } - }); -}); - -app.post("/newBotCommand", botAuth, (req, res) => { - let newBotCommand = new BotCommand({ - command: req.body.command, //is always "updateRole" - arguments: req.body.arguments, - executed: req.body.executed, //is always false - requestTimestamp: req.body.requestTimestamp, - }); - newBotCommand.save(); - res.status(200); -}); diff --git a/backend/migrate.js b/backend/migrate.js deleted file mode 100644 index 71a6fc83e..000000000 --- a/backend/migrate.js +++ /dev/null @@ -1,443 +0,0 @@ -const { config } = require("dotenv"); -const path = require("path"); -config({ path: path.join(__dirname, ".env") }); -const { mongoDB } = require("./init/mongodb"); -const { connectDB } = require("./init/mongodb"); -const { ObjectID } = require("mongodb"); -const { performance } = require("perf_hooks"); -const fs = require("fs"); - -// const { QuerySnapshotData } = require("firebase-firestore"); - -console.log(config()); - -const admin = require("firebase-admin"); - -// const { User } = require("./models/user"); -// const { Leaderboard } = require("./models/leaderboard"); -// const { BotCommand } = require("./models/bot-command"); - -const serviceAccount = require("./credentials/serviceAccountKey.json"); - -admin.initializeApp({ - credential: admin.credential.cert(serviceAccount), -}); - -var db = admin.firestore(); -var auth = admin.auth(); - -process.on("SIGTERM", async () => { - console.info("SIGTERM signal received. Stopping after this user is done"); - await currentUserPromise; - process.exit(1); -}); - -process.on("exit", async () => { - console.info("exit signal received. Stopping after this user is done"); - await currentUserPromise; - process.exit(1); -}); - -process.on("SIGINT", async () => { - console.info("SIGINT signal received. Stopping after this user is done"); - await currentUserPromise; - process.exit(1); -}); - -// Database should be completely clear before this is ran in order to prevent overlapping documents -// Migrate users - -let currentUserPromise = null; - -let resolveUser = null; - -async function migrateUsers() { - // let UIDOVERRIDE = "ugbG1GiSHxVEYMDmMeLV9byeukl2"; - let UIDOVERRIDE = undefined; - let lastId; - let usersSoFar = 0; - let totalUsers = 330000; - let totalCompletionTime = 0; - let averageCompletionTime = 0; - try { - let migrationStats = JSON.parse( - fs.readFileSync("./migrationStats.txt", "utf8") - ); - lastId = migrationStats.uid; - usersSoFar = migrationStats.usersSoFar; - totalCompletionTime = migrationStats.totalCompletionTime; - averageCompletionTime = migrationStats.averageCompletionTime; - } catch (e) {} - let querySnapshot; - let limit = 1000; - do { - console.log("starting another loop, getting users"); - if (lastId) { - let lastSnapshot = await db.collection("users").doc(lastId).get(); - querySnapshot = await db - .collection("users") - .where("banned", "==", true) - .orderBy("name") - .startAfter(lastSnapshot) - .limit(limit) - .get(); - } else { - querySnapshot = await db - .collection("users") - .where("banned", "==", true) - .orderBy("name") - .limit(limit) - .get(); - } - // console.log('start of foreach'); - console.log(`migrating ${querySnapshot.docs.length} users`); - let fulllog = false; - for (const userDoc of querySnapshot.docs) { - let userstart = performance.now(); - currentUserPromise = null; - currentUserPromise = new Promise((resolve, reject) => { - resolveUser = resolve; - }); - let userData = userDoc.data(); - let uid = userDoc.id; - try { - let userAuth = await auth.getUser(uid); - let email = userAuth.email; - let userCreatedAt = new Date(userAuth.metadata.creationTime).getTime(); - - let mongoUser = { - name: userData.name, - email: email, - addedAt: userCreatedAt, - uid: UIDOVERRIDE ? UIDOVERRIDE : uid, - oldTypingStats: {}, - }; - - if (userData.completedTests) - mongoUser.oldTypingStats.completedTests = userData.completedTests; - if (userData.discordId) mongoUser.discordId = userData.discordId; - if (userData.banned) mongoUser.banned = userData.banned; - if (userData.verified) mongoUser.verified = userData.verified; - //banned - //verified - if (userData.startedTests) - mongoUser.oldTypingStats.startedTests = userData.startedTests; - if (userData.timeTyping) - mongoUser.oldTypingStats.timeTyping = userData.timeTyping; - - if (userData.personalBests) - mongoUser.personalBests = userData.personalBests; - - let tagPairs = {}; - - let mongoUserTags = []; - - if (fulllog) console.log(`${uid} migrating tags`); - let tagsSnapshot = await db.collection(`users/${uid}/tags`).get(); - await tagsSnapshot.forEach(async (tagDoc) => { - let tagData = tagDoc.data(); - let tagId = tagDoc.id; - let new_id = ObjectID(); - tagPairs[tagId] = new_id; - let tagtopush = { _id: new_id, name: tagData.name }; - if (tagData.personalBests) - tagtopush.personalBests = tagData.personalBests; - mongoUserTags.push(tagtopush); - }); - - mongoUser.tags = mongoUserTags; - - // if (fulllog) console.log(`${uid} migrating config`); - // if (userData.config) { - // await mongoDB() - // .collection("configs") - // .updateOne( - // { uid: UIDOVERRIDE ? UIDOVERRIDE : uid }, - // { - // $set: { - // uid: UIDOVERRIDE ? UIDOVERRIDE : uid, - // config: userData.config, - // }, - // }, - // { upsert: true } - // ); - // } - - // if (fulllog) console.log(`${uid} migrating presets`); - // let presetsSnapshot = await db.collection(`users/${uid}/presets`).get(); - // await presetsSnapshot.forEach(async (presetDoc) => { - // let presetData = presetDoc.data(); - // let newpreset = { - // uid: UIDOVERRIDE ? UIDOVERRIDE : uid, - // name: presetData.name, - // }; - // if (presetData.config) newpreset.config = presetData.config; - // await mongoDB().collection("presets").insertOne(newpreset); - // }); - - // let lastcount = 0; - // let limit = 1000; - // let lastdoc = "start"; - // let total = 0; - // let newStats = { - // completedTests: 0, - // startedTests: 0, - // timeTyping: 0, - // }; - // if (fulllog) console.log(`${uid} migrating results`); - // do { - // if (fulllog) console.log(`${total} so far`); - // let resultsSnapshot; - // if (lastdoc === "start") { - // resultsSnapshot = await db - // .collection(`users/${uid}/results`) - // .orderBy("timestamp", "desc") - // .limit(limit) - // .get(); - // } else { - // resultsSnapshot = await db - // .collection(`users/${uid}/results`) - // .orderBy("timestamp", "desc") - // .startAfter(lastdoc) - // .limit(limit) - // .get(); - // } - // await resultsSnapshot.forEach(async (resultDoc) => { - // let resultData = resultDoc.data(); - // resultData.uid = UIDOVERRIDE ? UIDOVERRIDE : uid; - // if (resultData.tags && resultData.tags.length > 0) { - // resultData.tags = resultData.tags.map((tag) => tagPairs[tag]); - // } - // if (!resultData.charStats) { - // resultData.charStats = [ - // resultData.correctChars, - // resultData.incorrectChars, - // ]; - // } - // delete resultData.correctChars; - // delete resultData.incorrectChars; - // delete resultData.allChars; - // delete resultData.keySpacing; - // delete resultData.keyDuration; - // delete resultData.theme; - // delete resultData.name; - - // //remove the default fields here - // if (resultData.bailedOut === false) delete resultData.bailedOut; - // if (resultData.blindMode === false) delete resultData.blindMode; - // if (resultData.difficulty === "normal") - // delete resultData.difficulty; - // if (resultData.funbox === "none") delete resultData.funbox; - // if (resultData.language === "english") delete resultData.language; - // if (resultData.numbers === false) delete resultData.numbers; - // if (resultData.punctuation === false) delete resultData.punctuation; - - // if (resultData.mode !== "custom") delete resultData.customText; - - // newStats.completedTests++; - // if (resultData.restartCount) { - // newStats.startedTests += resultData.restartCount + 1; - // } else { - // newStats.startedTests++; - // } - // if (resultData.testDuration) { - // newStats.timeTyping += parseFloat(resultData.testDuration); - // } - // if (resultData.incompleteTestSeconds) { - // newStats.timeTyping += resultData.incompleteTestSeconds; - // } - // await mongoDB().collection("results").insertOne(resultData); - // }); - // lastcount = resultsSnapshot.docs.length; - // lastdoc = resultsSnapshot.docs[resultsSnapshot.docs.length - 1]; - // total += lastcount; - // } while (lastcount > 0); - - // if (fulllog) console.log(`${uid} migrated ${total} results`); - - // mongoUser.completedTests = newStats.completedTests; - // mongoUser.startedTests = newStats.startedTests; - // mongoUser.timeTyping = newStats.timeTyping; - - if (fulllog) console.log(`${uid} migrating user doc`); - await mongoDB() - .collection("users") - .updateOne( - { uid: UIDOVERRIDE ? UIDOVERRIDE : uid }, - { - $set: mongoUser, - }, - { upsert: true } - ); - - console.log(`${uid} migrated \t\t ${userData.name} \t\t`); - fs.appendFileSync( - "log_success.txt", - `${uid}\t\t${userData.name}\t\t`, - "utf8" - ); - } catch (err) { - console.log(`${uid} failed`); - console.log(err); - fs.appendFileSync( - "log_failed.txt", - `${uid}\t\t${err.message}\n`, - "utf8" - ); - } - lastId = uid; - let userend = performance.now(); - let time = (userend - userstart) / 1000; - totalCompletionTime += time; - // console.log(`${uid} took ${time} seconds`); - averageCompletionTime = totalCompletionTime / usersSoFar + 1; - usersSoFar++; - let estimateSecondsLeft = - averageCompletionTime * (totalUsers - usersSoFar); - console.log( - `${usersSoFar}/${totalUsers} users | estimated ${secondsToString( - estimateSecondsLeft, - true - )} left` - ); - - fs.writeFileSync( - "migrationStats.txt", - JSON.stringify({ - uid, - usersSoFar, - totalCompletionTime, - averageCompletionTime, - }), - "utf8" - ); - - resolveUser(); - - // console.log(userData); - // let newUser; - // try{ - // let data = userDoc.data(); - // data._id = userDoc.id; - // newUser = new User(data); - // newUser.uid = userDoc.id; - // newUser.globalStats = { - // started: userDoc.data().startedTests, - // completed: userDoc.data().completedTests, - // time: userDoc.data().timeTyping, - // }; - // let tagIdDict = {}; - // let tagsSnapshot = await db.collection(`users/${userDoc.id}/tags`).get(); - // tagsSnapshot.forEach((tagDoc) => { - // let formattedTag = tagDoc.data(); - // formattedTag._id = mongoose.Types.ObjectId(); //generate new objectId - // tagIdDict[tagDoc.id] = formattedTag._id; //save pair of ids in memory to determine what to set new id as in result tags - // newUser.tags.push(formattedTag); - // console.log(`Tag ${tagDoc.id} saved for user ${userCount}`); - // }); - // let resultsSnapshot = await db.collection(`users/${userDoc.id}/results`).get(); - // let resCount = 1; - // resultsSnapshot.forEach((result) => { - // let formattedResult = result.data(); - // if(formattedResult.tags != undefined){ - // formattedResult.tags.forEach((tag, index) => { - // if (tagIdDict[tag]) - // formattedResult.tags[index] = tagIdDict[tag]; - // }); - // } - // newUser.results.push(formattedResult); - // console.log(`Result ${resCount} saved for user ${userCount}`); - // resCount++; - // }); - // newUser.results.sort((a, b) => { - // return a.timestamp - b.timestamp; - // }); - // let presetsSnapshot = await db.collection(`users/${userDoc.id}/presets`).get(); - // presetsSnapshot.forEach((preset) => { - // newUser.presets.push(preset.data()); - // }); - // await newUser.save(); - // console.log(`User ${userCount} (${newUser.uid}) saved`); - // userCount++; - // }catch(e){ - // // throw e; - // console.log(`User ${userCount} (${newUser.uid}) failed: ${e.message}`); - // userCount++; - // } - } - } while (querySnapshot.docs.length > 0); - - console.log("Migration complete"); - // console.log('end of foreach'); -} -// //not tested because I can't get leaderboards to work on my fork for some reason -// db.collection("leaderboards") -// .get() -// .then((leaderboardsSnapshot) => { -// leaderboardsSnapshot.forEach((lbDoc) => { -// let newLb = new Leaderboard(lbDoc.data()); -// newLb.save(); -// }); -// }); - -// //migrate bot-commands -// db.collection("bot-commands") -// .get() -// .then((botCommandsSnapshot) => { -// botCommandsSnapshot.forEach((bcDoc) => { -// let newBotCommand = new BotCommand(bcDoc.data()); -// newBotCommand.save(); -// }); -// }); - -//migrate public stat -async function migratePublicStats() { - db.collection("public") - .doc("stats") - .get() - .then((ret) => { - let stats = ret.data(); - mongoDB() - .collection("public") - .updateOne( - { type: "stats" }, - { - $set: { - completedTests: stats.completedTests, - startedTests: stats.startedTests, - timeTyping: stats.timeTyping, - }, - }, - { upsert: true } - ); - }); -} - -async function init() { - await connectDB(); - // await migratePublicStats(); - await migrateUsers(); - // process.exit(1); -} - -function secondsToString(sec, full = false) { - const hours = Math.floor(sec / 3600); - const minutes = Math.floor((sec % 3600) / 60); - const seconds = Math.round((sec % 3600) % 60); - let hoursString; - let minutesString; - let secondsString; - hours < 10 ? (hoursString = "0" + hours) : (hoursString = hours); - minutes < 10 ? (minutesString = "0" + minutes) : (minutesString = minutes); - seconds < 10 && (minutes > 0 || hours > 0 || full) - ? (secondsString = "0" + seconds) - : (secondsString = seconds); - - let ret = ""; - if (hours > 0 || full) ret += hoursString + ":"; - if (minutes > 0 || hours > 0 || full) ret += minutesString + ":"; - ret += secondsString; - return ret; -} - -init(); diff --git a/backend/mongo-todo.md b/backend/mongo-todo.md deleted file mode 100644 index 0c5daef2c..000000000 --- a/backend/mongo-todo.md +++ /dev/null @@ -1,57 +0,0 @@ -# Mongo todo - -## Todo - -- Make sure that the branch is ready for deployment -- Make sure that the bot can interact with the data on the express server - - Would be optimal if the bot were to run on the same server as the express server, so that the bot wouldn't have to access data through api routes -- Determine if generatePairingCode should be removed or migrated - - This function was commented out in index.js but is used in frontend - -## Bugs - -- Make sure that the bot is able to interact with the mongo database - - If bot is on same server, it could work with mongo directly, otherwise, more api routes are needed -- Do names have to be made lowercase before checking if a duplicate name is found?(that is when a new user is created or username is changed) - -### Minor/efficiency bugs - -- Does clearDailyLeaderboards cause a memory leak? -- Is filteredResults.reverse(); in account.js going to cause efficiency issues? - - For loop in account could work backwards instead, but this would add complexity -- Why does `if (page == "account") pageTransition = false;` get rid of endless account loading bug when accessing via url -- Name is not passed in user token/auth().currentUser -- Account button sometimes shows loading infinitely after a test - - Can't navigate to user until page is refreshed - - After refresh, pr is not saved - - Can't induce this error and doesn't occur often so adding it as minor bug -- lbmemory undefined if page not refreshed after user sign up? -- If you are in first place and you place on the leaderboard but not above yourself, you may get glb undefined error - - Might also occur if you are simply on the leaderboard and make the leaderboard but not above your current position - - Doesn't happen all the time -- Hidden property of leaderboard is unused -- Verified property of user is unused, set at false by default - - Can't find where the property would be set in the code - - Is this discord verified, if so, why do you need discord verified to be on leaderboard? - - Temporarily removed from leaderboard requirements - -### Functions not found anywhere except for index.js - -Might need to be migrated, might not. I'm not sure why these are in the file if they are not being used. - -- getAllNames -- getAllUsers -- getPatreons -- requestTest -- incrementStartedTestCounter -- incrementTestCounter - -### Possibilities - -- Might be worthwhile to use redis to store userdata up to a certain point - - Users who have been requested in the last hour will be stored in the redis database so that their data can be sent again without having to search a large database - - After an hour without a new request they can be removed from memory -- Create a backup system to prevent loss of data - - Users should be able to export their data themselves - - Pretty much is just the user snap but without uid -- Could split server.js into multiple files for easier code management diff --git a/package-lock.json b/package-lock.json index 5dfca213f..feb9d49a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "GPL-3.0", "dependencies": { "@babel/runtime": "^7.12.5", - "axios": "^0.21.1", + "axios": "^0.21.2", "chart.js": "^2.9.4", "chartjs-plugin-annotation": "^0.5.7", "chartjs-plugin-trendline": "^0.2.2", @@ -2814,11 +2814,11 @@ } }, "node_modules/axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz", + "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==", "dependencies": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "node_modules/babel-eslint": { @@ -16580,11 +16580,11 @@ "dev": true }, "axios": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", - "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz", + "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==", "requires": { - "follow-redirects": "^1.10.0" + "follow-redirects": "^1.14.0" } }, "babel-eslint": { diff --git a/package.json b/package.json index 8c610deaf..c4df61eba 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ }, "dependencies": { "@babel/runtime": "^7.12.5", - "axios": "^0.21.1", + "axios": "^0.21.2", "chart.js": "^2.9.4", "chartjs-plugin-annotation": "^0.5.7", "chartjs-plugin-trendline": "^0.2.2", diff --git a/src/js/replay.js b/src/js/replay.js index e36047c39..9b2e96770 100644 --- a/src/js/replay.js +++ b/src/js/replay.js @@ -1,14 +1,3 @@ -/* -TODO: - Export replay as video - Export replay as typing test file? - .ttr file extension (stands for typing test record) - Should just be json, but fields should be specified by some format - metadata field with rules, website source, mode, name of typist - data field should be a list of objects, like monkeytype replay uses - signature or verfication field should be able to check file validity with server - And add ability to upload file to watch replay -*/ import config from "./config"; import * as Sound from "./sound"; diff --git a/static/privacy-policy.html b/static/privacy-policy.html index 212a7c60d..0af3cd057 100644 --- a/static/privacy-policy.html +++ b/static/privacy-policy.html @@ -111,7 +111,7 @@

-

Effective date: August 27, 2021

+

Effective date: September 8, 2021

Thanks for trusting Monkeytype ('Monkeytype', 'we', 'us', 'our') with your personal information, your code, and your projects! We take our @@ -188,7 +188,7 @@

  • Change settings on the website
    • -

    How do we use your data?

    +

    How will we use your data?

    Monkeytype collects your data so that we can:

    What types of cookies do we use?

    - There are a number of different types of cookies, however, our website + There are a number of different types of cookies; however, our website uses functionality cookies. Monkeytype uses these cookies so we recognize you on our website and remember your previously selected settings.

    -

    How to manage cookies

    +

    How to manage your cookies

    You can set your browser not to accept cookies, and the above website tells you how to remove cookies from your browser. However, in a few @@ -324,7 +323,7 @@

    How to contact us

    - If you have any questions about Monkeytypes’s privacy policy, the data + If you have any questions about Monkeytype’s privacy policy, the data we hold on you, or you would like to exercise one of your data protection rights, please do not hesitate to contact us.

    diff --git a/static/security-policy.html b/static/security-policy.html index 9c488a47a..32f4acdbc 100644 --- a/static/security-policy.html +++ b/static/security-policy.html @@ -129,19 +129,19 @@

    How to Disclose a Vulnerability

    - For vulnerabilities that impact the confidentiality, integrity and - availability of monkeytype services, please send your disclosure via + For vulnerabilities that impact the confidentiality, integrity, and + availability of Monkeytype services, please send your disclosure via (1) - mail - , or (2) private discord chat to - miodec - . For non-security related platform bugs, follow the bug submission + email, + or (2) private Discord chat to + Miodec. + For non-security related platform bugs, follow the bug submission guidelines - - .Include as much detail as possible to ensure reproducibility. At a + . + Include as much detail as possible to ensure reproducibility. At a minimum, vulnerability disclosures should include: