Refactor cron jobs (#2357) by Bruception

* Refactor cron job configuration

* Refactor leaderboard update logic

* Update naming

* Refactor to use leaderboard dao

* Deduplicate top 10 query

* Fix oopsy

* Fix query order

* Undo method signature update

* Update new records detection logic

* Fix lint

backend/dao/bot.js

@@ -10,6 +10,25 @@ async function addCommand(command, arguments) {
   });
 }
 
+async function addCommands(commands, arguments) {
+  if (commands.length === 0 || commands.length !== arguments.length) {
+    return [];
+  }
+
+  const normalizedCommands = commands.map((command, index) => {
+    return {
+      command,
+      arguments: arguments[index],
+      executed: false,
+      requestTimestamp: Date.now(),
+    };
+  });
+
+  return await mongoDB()
+    .collection("bot-commands")
+    .insertMany(normalizedCommands);
+}
+
 class BotDAO {
   static async updateDiscordRole(discordId, wpm) {
     return await addCommand("updateRole", [discordId, wpm]);
@@ -27,16 +46,25 @@ class BotDAO {
     return await addCommand("awardChallenge", [discordId, challengeName]);
   }
 
-  static async announceLbUpdate(discordId, pos, lb, wpm, raw, acc, con) {
-    return await addCommand("sayLbUpdate", [
-      discordId,
-      pos,
-      lb,
-      wpm,
-      raw,
-      acc,
-      con,
-    ]);
+  static async announceLbUpdate(newRecords, leaderboardId) {
+    if (newRecords.length === 0) {
+      return [];
+    }
+
+    const leaderboardCommands = Array(newRecords.length).fill("sayLbUpdate");
+    const leaderboardCommandsArguments = newRecords.map((newRecord) => {
+      return [
+        newRecord.discordId ?? newRecord.name,
+        newRecord.rank,
+        leaderboardId,
+        newRecord.wpm,
+        newRecord.raw,
+        newRecord.acc,
+        newRecord.consistency,
+      ];
+    });
+
+    return await addCommands(leaderboardCommands, leaderboardCommandsArguments);
   }
 }
 

backend/jobs/deleteOldLogs.js

@@ -0,0 +1,20 @@
+const { CronJob } = require("cron");
+const { mongoDB } = require("../init/mongodb");
+
+const CRON_SCHEDULE = "0 0 0 * * *";
+const LOG_MAX_AGE_DAYS = 7;
+const LOG_MAX_AGE_MILLISECONDS = LOG_MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
+
+async function deleteOldLogs() {
+  const data = await mongoDB()
+    .collection("logs")
+    .deleteMany({ timestamp: { $lt: Date.now() - LOG_MAX_AGE_MILLISECONDS } });
+
+  Logger.log(
+    "system_logs_deleted",
+    `${data.deletedCount} logs deleted older than ${LOG_MAX_AGE_DAYS} day(s)`,
+    undefined
+  );
+}
+
+module.exports = new CronJob(CRON_SCHEDULE, deleteOldLogs);

backend/jobs/index.js

@@ -0,0 +1,4 @@
+const updateLeaderboards = require("./updateLeaderboards");
+const deleteOldLogs = require("./deleteOldLogs");
+
+module.exports = [updateLeaderboards, deleteOldLogs];

backend/jobs/updateLeaderboards.js

@@ -0,0 +1,55 @@
+const { CronJob } = require("cron");
+const { mongoDB } = require("../init/mongodb");
+const BotDAO = require("../dao/bot");
+const LeaderboardsDAO = require("../dao/leaderboards");
+
+const CRON_SCHEDULE = "30 4/5 * * * *";
+const RECENT_AGE_MINUTES = 10;
+const RECENT_AGE_MILLISECONDS = RECENT_AGE_MINUTES * 60 * 1000;
+
+async function getTop10(leaderboardTime) {
+  return await LeaderboardsDAO.get("time", leaderboardTime, "english", 0, 10);
+}
+
+async function updateLeaderboardAndNotifyChanges(leaderboardTime) {
+  const top10BeforeUpdate = await getTop10(leaderboardTime);
+
+  const previousRecordsMap = Object.fromEntries(
+    top10BeforeUpdate.map((record) => {
+      return [record.uid, record];
+    })
+  );
+
+  await LeaderboardsDAO.update("time", leaderboardTime, "english");
+
+  const top10AfterUpdate = await getTop10(leaderboardTime);
+
+  const newRecords = top10AfterUpdate.filter((record) => {
+    const userId = record.uid;
+
+    const userImprovedRank =
+      userId in previousRecordsMap &&
+      previousRecordsMap[userId].rank > record.rank;
+
+    const newUserInTop10 = !(userId in previousRecordsMap);
+
+    const isRecentRecord =
+      record.timestamp > Date.now() - RECENT_AGE_MILLISECONDS;
+
+    return (userImprovedRank || newUserInTop10) && isRecentRecord;
+  });
+
+  if (newRecords.length > 0) {
+    await BotDAO.announceLbUpdate(
+      newRecords,
+      `time ${leaderboardTime} english`
+    );
+  }
+}
+
+async function updateLeaderboards() {
+  await updateLeaderboardAndNotifyChanges("15");
+  await updateLeaderboardAndNotifyChanges("60");
+}
+
+module.exports = new CronJob(CRON_SCHEDULE, updateLeaderboards);

backend/server.js

@@ -3,13 +3,12 @@ const { config } = require("dotenv");
 const path = require("path");
 const MonkeyError = require("./handlers/error");
 config({ path: path.join(__dirname, ".env") });
-const CronJob = require("cron").CronJob;
 const cors = require("cors");
 const admin = require("firebase-admin");
 const Logger = require("./handlers/logger.js");
 const serviceAccount = require("./credentials/serviceAccountKey.json");
 const { connectDB, mongoDB } = require("./init/mongodb");
-const BotDAO = require("./dao/bot");
+const jobs = require("./jobs");
 const addApiRoutes = require("./api/routes");
 
 const PORT = process.env.PORT || 5005;
@@ -71,8 +70,6 @@ app.use(function (e, req, res, next) {
   return res.status(monkeyError.status || 500).json(monkeyError);
 });
 
-const LeaderboardsDAO = require("./dao/leaderboards");
-
 console.log("Starting server...");
 app.listen(PORT, async () => {
   console.log(`Listening on port ${PORT}`);
@@ -83,95 +80,6 @@ app.listen(PORT, async () => {
     credential: admin.credential.cert(serviceAccount),
   });
 
-  let lbjob = new CronJob("30 4/5 * * * *", async () => {
-    let before15 = await mongoDB()
-      .collection("leaderboards.english.time.15")
-      .find()
-      .limit(10)
-      .toArray();
-    LeaderboardsDAO.update("time", "15", "english").then(async () => {
-      let after15 = await mongoDB()
-        .collection("leaderboards.english.time.15")
-        .find()
-        .limit(10)
-        .toArray();
-
-      let changed;
-      let recent = false;
-      for (let index in before15) {
-        if (before15[index].uid !== after15[index].uid) {
-          //something changed at this index
-          if (after15[index].timestamp > Date.now() - 1000 * 60 * 10) {
-            //checking if test is within 10 minutes
-            recent = true;
-          }
-          changed = after15[index];
-          break;
-        }
-      }
-      if (changed && recent) {
-        let name = changed.discordId ?? changed.name;
-        BotDAO.announceLbUpdate(
-          name,
-          changed.rank,
-          "time 15 english",
-          changed.wpm,
-          changed.raw,
-          changed.acc,
-          changed.consistency
-        );
-      }
-    });
-
-    let before60 = await mongoDB()
-      .collection("leaderboards.english.time.60")
-      .find()
-      .limit(10)
-      .toArray();
-    LeaderboardsDAO.update("time", "60", "english").then(async () => {
-      let after60 = await mongoDB()
-        .collection("leaderboards.english.time.60")
-        .find()
-        .limit(10)
-        .toArray();
-      let changed;
-      let recent = false;
-      for (let index in before60) {
-        if (before60[index].uid !== after60[index].uid) {
-          //something changed at this index
-          if (after60[index].timestamp > Date.now() - 1000 * 60 * 10) {
-            //checking if test is within 10 minutes
-            recent = true;
-          }
-          changed = after60[index];
-          break;
-        }
-      }
-      if (changed && recent) {
-        let name = changed.discordId ?? changed.name;
-        BotDAO.announceLbUpdate(
-          name,
-          changed.rank,
-          "time 60 english",
-          changed.wpm,
-          changed.raw,
-          changed.acc,
-          changed.consistency
-        );
-      }
-    });
-  });
-  lbjob.start();
-
-  let logjob = new CronJob("0 0 0 * * *", async () => {
-    let data = await mongoDB()
-      .collection("logs")
-      .deleteMany({ timestamp: { $lt: Date.now() - 604800000 } });
-    Logger.log(
-      "system_logs_deleted",
-      `${data.deletedCount} logs deleted older than 7 days`,
-      undefined
-    );
-  });
-  logjob.start();
+  console.log("Starting cron jobs...");
+  jobs.forEach((job) => job.start());
 });