Migrate deletion to asynchronous job

backend/src/api/controllers/user.ts

@@ -38,7 +38,7 @@ import {
   TestActivity,
   UserProfileDetails,
 } from "@monkeytype/contracts/schemas/users";
-import { addImportantLog, addLog, deleteUserLogs } from "../../dal/logs";
+import { addImportantLog, addLog } from "../../dal/logs";
 import { sendForgotPasswordEmail as authSendForgotPasswordEmail } from "../../utils/auth";
 import {
   AddCustomThemeRequest,
@@ -231,39 +231,7 @@ export async function deleteUser(
 ): Promise<MonkeyResponse2> {
   const { uid } = req.ctx.decodedToken;
 
-  const userInfo = await UserDAL.getPartialUser(uid, "delete user", [
-    "banned",
-    "name",
-    "email",
-    "discordId",
-  ]);
-
-  if (userInfo.banned === true) {
-    await BlocklistDal.add(userInfo);
-  }
-
-  //cleanup database
-  await Promise.all([
-    UserDAL.deleteUser(uid),
-    deleteUserLogs(uid),
-    deleteAllApeKeys(uid),
-    deleteAllPresets(uid),
-    deleteConfig(uid),
-    deleteAllResults(uid),
-    purgeUserFromDailyLeaderboards(
-      uid,
-      req.ctx.configuration.dailyLeaderboards
-    ),
-  ]);
-
-  //delete user from
-  await AuthUtil.deleteUser(uid);
-
-  void addImportantLog(
-    "user_deleted",
-    `${userInfo.email} ${userInfo.name}`,
-    uid
-  );
+  await UserDAL.softDeleteUser(uid);
 
   return new MonkeyResponse2("User deleted", null);
 }

backend/src/dal/user.ts

@@ -76,6 +76,23 @@ export async function deleteUser(uid: string): Promise<void> {
   await getUsersCollection().deleteOne({ uid });
 }
 
+export async function softDeleteUser(uid: string): Promise<void> {
+  await getUsersCollection().updateOne(
+    { uid },
+    {
+      $set: {
+        deleted: true,
+      },
+    }
+  );
+}
+
+export async function getSoftDeletedUsers(
+  limit: number
+): Promise<MonkeyTypes.DBUser[]> {
+  return getUsersCollection().find({ deleted: true }, { limit }).toArray();
+}
+
 export async function resetUser(uid: string): Promise<void> {
   await getUsersCollection().updateOne(
     { uid },

backend/src/jobs/delete-users.ts

@@ -0,0 +1,67 @@
+import { CronJob } from "cron";
+
+import { deleteAllApeKeys } from "../dal/ape-keys";
+import * as BlocklistDal from "../dal/blocklist";
+import { deleteConfig } from "../dal/config";
+import { addImportantLog, deleteUserLogs } from "../dal/logs";
+import { deleteAllPresets } from "../dal/preset";
+import { deleteAll as deleteAllResults } from "../dal/result";
+import * as UserDAL from "../dal/user";
+import { getCachedConfiguration } from "../init/configuration";
+import * as AuthUtil from "../utils/auth";
+import { purgeUserFromDailyLeaderboards } from "../utils/daily-leaderboards";
+import { mapLimit } from "../utils/misc";
+
+const CRON_SCHEDULE = "*/10 * * * *"; // every 10 minutes
+const DELETE_BATCH_SIZE = 50;
+const CONCURRENT_DELETIONS = 5;
+
+async function deleteUser(uid: string): Promise<void> {
+  const config = await getCachedConfiguration();
+
+  const userInfo = await UserDAL.getPartialUser(uid, "delete user", [
+    "banned",
+    "name",
+    "email",
+    "discordId",
+  ]);
+
+  if (userInfo.banned === true) {
+    await BlocklistDal.add(userInfo);
+  }
+
+  // cleanup database
+  await Promise.all([
+    deleteUserLogs(uid),
+    deleteAllApeKeys(uid),
+    deleteAllPresets(uid),
+    deleteConfig(uid),
+    deleteAllResults(uid),
+    purgeUserFromDailyLeaderboards(uid, config.dailyLeaderboards),
+  ]);
+
+  // delete user from auth
+  await AuthUtil.deleteUser(uid);
+
+  void addImportantLog(
+    "user_deleted",
+    `${userInfo.email} ${userInfo.name}`,
+    uid
+  );
+
+  await UserDAL.deleteUser(uid);
+}
+
+async function deleteUsers(): Promise<void> {
+  const softDeletedUsers = await UserDAL.getSoftDeletedUsers(DELETE_BATCH_SIZE);
+
+  if (softDeletedUsers.length === 0) {
+    return;
+  }
+
+  await mapLimit(softDeletedUsers, CONCURRENT_DELETIONS, async (user) => {
+    await deleteUser(user.uid);
+  });
+}
+
+export default new CronJob(CRON_SCHEDULE, deleteUsers);

backend/src/jobs/index.ts

@@ -2,10 +2,12 @@ import updateLeaderboards from "./update-leaderboards";
 import deleteOldLogs from "./delete-old-logs";
 import logCollectionSizes from "./log-collection-sizes";
 import logQueueSizes from "./log-queue-sizes";
+import deleteUsers from "./delete-users";
 
 export default [
   updateLeaderboards,
   deleteOldLogs,
   logCollectionSizes,
   logQueueSizes,
+  deleteUsers,
 ];

backend/src/utils/misc.ts

@@ -343,3 +343,35 @@ export function replaceObjectIds<T extends { _id: ObjectId }>(
   if (data === undefined) return data;
   return data.map((it) => replaceObjectId(it));
 }
+
+type MapLimitIteratee<T, V> = (element: T, index: number) => V;
+
+export async function mapLimit<T, V>(
+  input: T[],
+  limit: number,
+  iteratee: MapLimitIteratee<T, V>
+): Promise<V[]> {
+  const size = input.length;
+
+  const allElements = new Array(size);
+  const results = new Array(size);
+
+  for (let i = 0; i < size; ++i) {
+    allElements[size - 1 - i] = [input[i], i];
+  }
+
+  const execute = async () => {
+    while (allElements.length > 0) {
+      const [element, index] = allElements.pop();
+      results[index] = await iteratee(element, index);
+    }
+  };
+
+  const allExecutors: Promise<void>[] = [];
+  for (let i = 0; i < limit; ++i) {
+    allExecutors.push(execute());
+  }
+  await Promise.all(allExecutors);
+
+  return results;
+}