diff --git a/backend/api/controllers/result.ts b/backend/api/controllers/result.ts index a171360d6..936abc362 100644 --- a/backend/api/controllers/result.ts +++ b/backend/api/controllers/result.ts @@ -16,6 +16,7 @@ import { } from "../../anticheat/index"; import MonkeyStatusCodes from "../../constants/monkey-status-codes"; import { incrementResult } from "../../utils/prometheus"; +import George from "../../tasks/george"; const objecthash = node_object_hash().hash; @@ -61,6 +62,10 @@ class ResultController { static async addResult(req: MonkeyTypes.Request): Promise { const { uid } = req.ctx.decodedToken; + + const useRedisForBotTasks = + req.ctx.configuration.useRedisForBotTasks.enabled; + const result = Object.assign({}, req.body.result); result.uid = uid; if (result.wpm === result.raw && result.acc !== 100) { @@ -246,11 +251,17 @@ class ResultController { if (result.mode === "time" && String(result.mode2) === "60") { UserDAO.incrementBananas(uid, result.wpm); if (isPb && user.discordId) { + if (useRedisForBotTasks) { + George.updateDiscordRole(user.discordId, result.wpm); + } BotDAO.updateDiscordRole(user.discordId, result.wpm); } } if (result.challenge && user.discordId) { + if (useRedisForBotTasks) { + George.awardChallenge(user.discordId, result.challenge); + } BotDAO.awardChallenge(user.discordId, result.challenge); } else { delete result.challenge; diff --git a/backend/api/controllers/user.ts b/backend/api/controllers/user.ts index add843e76..3c4e3fc66 100644 --- a/backend/api/controllers/user.ts +++ b/backend/api/controllers/user.ts @@ -5,6 +5,7 @@ import Logger from "../../utils/logger"; import { MonkeyResponse } from "../../utils/monkey-response"; import { linkAccount } from "../../utils/discord"; import { buildAgentLog } from "../../utils/misc"; +import George from "../../tasks/george"; class UserController { static async createNewUser( @@ -110,6 +111,9 @@ class UserController { data: { tokenType, accessToken }, } = req.body; + const useRedisForBotTasks = + req.ctx.configuration.useRedisForBotTasks.enabled; + const userInfo = await UsersDAO.getUser(uid); if (userInfo.banned) { throw new MonkeyError(403, "Banned accounts cannot link with Discord"); @@ -134,6 +138,10 @@ class UserController { } await UsersDAO.linkDiscord(uid, discordId); + + if (useRedisForBotTasks) { + George.linkDiscord(discordId, uid); + } await BotDAO.linkDiscord(uid, discordId); Logger.log("user_discord_link", `linked to ${discordId}`, uid); @@ -145,12 +153,19 @@ class UserController { ): Promise { const { uid } = req.ctx.decodedToken; + const useRedisForBotTasks = + req.ctx.configuration.useRedisForBotTasks.enabled; + const userInfo = await UsersDAO.getUser(uid); if (!userInfo.discordId) { throw new MonkeyError(404, "User does not have a linked Discord account"); } + if (useRedisForBotTasks) { + George.unlinkDiscord(userInfo.discordId, uid); + } await BotDAO.unlinkDiscord(uid, userInfo.discordId); + await UsersDAO.unlinkDiscord(uid); Logger.log("user_discord_unlinked", userInfo.discordId, uid); diff --git a/backend/constants/base-configuration.ts b/backend/constants/base-configuration.ts index c3f99d68d..fd286715f 100644 --- a/backend/constants/base-configuration.ts +++ b/backend/constants/base-configuration.ts @@ -26,6 +26,9 @@ const BASE_CONFIGURATION: MonkeyTypes.Configuration = { enableSavingResults: { enabled: false, }, + useRedisForBotTasks: { + enabled: false, + }, }; export default BASE_CONFIGURATION; diff --git a/backend/example.env b/backend/example.env index b3cd2a4ef..ca3acace1 100644 --- a/backend/example.env +++ b/backend/example.env @@ -1,5 +1,6 @@ DB_NAME=monkeytype DB_URI=mongodb://localhost:27017 +REDIS_URI=redis://localhost:6379 MODE=dev # You can also use the format mongodb://username:password@host:port or # uncomment the following lines if you want to define them separately diff --git a/backend/init/redis.ts b/backend/init/redis.ts new file mode 100644 index 000000000..b473f9f5c --- /dev/null +++ b/backend/init/redis.ts @@ -0,0 +1,41 @@ +import IORedis from "ioredis"; + +class RedisClient { + static connection: IORedis.Redis; + static connected = false; + + static async connect(): Promise { + if (this.connected) { + return; + } + + const { REDIS_URI } = process.env; + + if (!REDIS_URI) { + throw new Error("No redis configuration provided"); + } + + this.connection = new IORedis(REDIS_URI, { + maxRetriesPerRequest: null, // These options are required for BullMQ + enableReadyCheck: false, + lazyConnect: true, + }); + + try { + await this.connection.connect(); + this.connected = true; + } catch (error) { + console.error(error.message); + console.error( + "Failed to connect to redis. Exiting with exit status code 1." + ); + process.exit(1); + } + } + + static getConnection(): IORedis.Redis { + return this.connection; + } +} + +export default RedisClient; diff --git a/backend/jobs/update-leaderboards.ts b/backend/jobs/update-leaderboards.ts index f3a64cd13..c3aa884ac 100644 --- a/backend/jobs/update-leaderboards.ts +++ b/backend/jobs/update-leaderboards.ts @@ -1,7 +1,9 @@ import { CronJob } from "cron"; import BotDAO from "../dao/bot"; +import George from "../tasks/george"; import { Document, WithId } from "mongodb"; import LeaderboardsDAO from "../dao/leaderboards"; +import ConfigurationClient from "../init/configuration"; const CRON_SCHEDULE = "30 14/15 * * * *"; const RECENT_AGE_MINUTES = 10; @@ -48,10 +50,15 @@ async function updateLeaderboardAndNotifyChanges( }); if (newRecords.length > 0) { - await BotDAO.announceLbUpdate( - newRecords, - `time ${leaderboardTime} english` - ); + const cachedConfig = await ConfigurationClient.getCachedConfiguration(); + + const leaderboardId = `time ${leaderboardTime} english`; + + if (cachedConfig.useRedisForBotTasks.enabled) { + await George.announceLbUpdate(newRecords, leaderboardId); + } + + await BotDAO.announceLbUpdate(newRecords, leaderboardId); } } diff --git a/backend/package-lock.json b/backend/package-lock.json index 88f8b27ff..9475555d7 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,6 +10,7 @@ "license": "GPL-3.0", "dependencies": { "bcrypt": "5.0.1", + "bullmq": "1.78.1", "cors": "2.8.5", "cron": "1.8.2", "dotenv": "10.0.0", @@ -17,6 +18,7 @@ "express-rate-limit": "6.2.1", "firebase-admin": "10.0.2", "helmet": "4.6.0", + "ioredis": "4.28.5", "joi": "17.6.0", "lodash": "4.17.21", "mongodb": "4.4.0", @@ -36,6 +38,7 @@ "@types/bcrypt": "5.0.0", "@types/cors": "2.8.12", "@types/cron": "1.7.3", + "@types/ioredis": "4.28.10", "@types/lodash": "4.14.178", "@types/node": "17.0.18", "@types/node-fetch": "2.6.1", @@ -714,6 +717,15 @@ "@types/node": "*" } }, + "node_modules/@types/ioredis": { + "version": "4.28.10", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", + "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/lodash": { "version": "4.14.178", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.178.tgz", @@ -1260,6 +1272,35 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "node_modules/bullmq": { + "version": "1.78.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.78.1.tgz", + "integrity": "sha512-er45mM8nGhgA83EVCJ4PNxPyDSzakvoxeFGU4vdSgYeB+SbeFQAlJYmAC50Ms7YFPstm1LeinbVZ+oX/BmBzOg==", + "dependencies": { + "cron-parser": "^4.2.1", + "get-port": "^5.1.1", + "glob": "^7.2.0", + "ioredis": "^4.28.2", + "lodash": "^4.17.21", + "msgpackr": "^1.4.6", + "semver": "^6.3.0", + "tslib": "^1.14.1", + "uuid": "^8.3.2" + } + }, + "node_modules/bullmq/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/bullmq/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -1432,6 +1473,14 @@ "mimic-response": "^1.0.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1582,6 +1631,17 @@ "moment-timezone": "^0.5.x" } }, + "node_modules/cron-parser": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.2.1.tgz", + "integrity": "sha512-5sJBwDYyCp+0vU5b7POl8zLWfgV5fOHxlc45FWoWdHecGC7MQHCjx0CHivCMRnGFovghKhhyYM+Zm9DcY5qcHg==", + "dependencies": { + "luxon": "^1.28.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/crypto-random-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", @@ -2225,6 +2285,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -2659,6 +2730,60 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==" }, + "node_modules/ioredis": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "dependencies": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/denque": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", + "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -3062,11 +3187,26 @@ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -3145,6 +3285,14 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" }, + "node_modules/luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==", + "engines": { + "node": "*" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -3244,9 +3392,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "node_modules/minipass": { "version": "3.1.6", @@ -3363,6 +3511,31 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "node_modules/msgpackr": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.5.5.tgz", + "integrity": "sha512-JG0V47xRIQ9pyUnx6Hb4+3TrQoia2nA3UIdmyTldhxaxtKFkekkKpUW/N6fwHwod9o4BGuJGtouxOk+yCP5PEA==", + "optionalDependencies": { + "msgpackr-extract": "^1.0.14" + } + }, + "node_modules/msgpackr-extract": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-1.0.16.tgz", + "integrity": "sha512-fxdRfQUxPrL/TizyfYfMn09dK58e+d65bRD/fcaVH4052vj30QOzzqxcQIS7B0NsqlypEQ/6Du3QmP2DhWFfCA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "nan": "^2.14.2", + "node-gyp-build": "^4.2.3" + } + }, + "node_modules/nan": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.15.0.tgz", + "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", + "optional": true + }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -3396,13 +3569,24 @@ } }, "node_modules/node-forge": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", - "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz", + "integrity": "sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA==", "engines": { "node": ">= 6.13.0" } }, + "node_modules/node-gyp-build": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.3.0.tgz", + "integrity": "sha512-iWjXZvmboq0ja1pUGULQBexmxq8CV4xBhX7VDOTbL7ZR4FOowwY/VOtRxBN/yKxmdGoIp4j5ysNT4u3S2pDQ3Q==", + "optional": true, + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-object-hash": { "version": "2.3.10", "resolved": "https://registry.npmjs.org/node-object-hash/-/node-object-hash-2.3.10.tgz", @@ -3567,6 +3751,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "engines": { + "node": ">=6" + } + }, "node_modules/package-json": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", @@ -3887,6 +4079,30 @@ "node": ">=8.10.0" } }, + "node_modules/redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/registry-auth-token": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.1.tgz", @@ -4315,6 +4531,11 @@ "node": ">=0.10.0" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -5554,6 +5775,15 @@ "@types/node": "*" } }, + "@types/ioredis": { + "version": "4.28.10", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", + "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/lodash": { "version": "4.14.178", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.178.tgz", @@ -5991,6 +6221,34 @@ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, + "bullmq": { + "version": "1.78.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.78.1.tgz", + "integrity": "sha512-er45mM8nGhgA83EVCJ4PNxPyDSzakvoxeFGU4vdSgYeB+SbeFQAlJYmAC50Ms7YFPstm1LeinbVZ+oX/BmBzOg==", + "requires": { + "cron-parser": "^4.2.1", + "get-port": "^5.1.1", + "glob": "^7.2.0", + "ioredis": "^4.28.2", + "lodash": "^4.17.21", + "msgpackr": "^1.4.6", + "semver": "^6.3.0", + "tslib": "^1.14.1", + "uuid": "^8.3.2" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + }, + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + } + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -6117,6 +6375,11 @@ "mimic-response": "^1.0.0" } }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, "color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -6236,6 +6499,14 @@ "moment-timezone": "^0.5.x" } }, + "cron-parser": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.2.1.tgz", + "integrity": "sha512-5sJBwDYyCp+0vU5b7POl8zLWfgV5fOHxlc45FWoWdHecGC7MQHCjx0CHivCMRnGFovghKhhyYM+Zm9DcY5qcHg==", + "requires": { + "luxon": "^1.28.0" + } + }, "crypto-random-string": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", @@ -6750,6 +7021,11 @@ "has-symbols": "^1.0.1" } }, + "get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -7068,6 +7344,44 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==" }, + "ioredis": { + "version": "4.28.5", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", + "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "denque": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", + "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -7401,11 +7715,26 @@ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=" }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + }, "lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -7480,6 +7809,11 @@ } } }, + "luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -7548,9 +7882,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, "minipass": { "version": "3.1.6", @@ -7637,6 +7971,30 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "msgpackr": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.5.5.tgz", + "integrity": "sha512-JG0V47xRIQ9pyUnx6Hb4+3TrQoia2nA3UIdmyTldhxaxtKFkekkKpUW/N6fwHwod9o4BGuJGtouxOk+yCP5PEA==", + "requires": { + "msgpackr-extract": "^1.0.14" + } + }, + "msgpackr-extract": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-1.0.16.tgz", + "integrity": "sha512-fxdRfQUxPrL/TizyfYfMn09dK58e+d65bRD/fcaVH4052vj30QOzzqxcQIS7B0NsqlypEQ/6Du3QmP2DhWFfCA==", + "optional": true, + "requires": { + "nan": "^2.14.2", + "node-gyp-build": "^4.2.3" + } + }, + "nan": { + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.15.0.tgz", + "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", + "optional": true + }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -7656,9 +8014,15 @@ } }, "node-forge": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", - "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz", + "integrity": "sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA==" + }, + "node-gyp-build": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.3.0.tgz", + "integrity": "sha512-iWjXZvmboq0ja1pUGULQBexmxq8CV4xBhX7VDOTbL7ZR4FOowwY/VOtRxBN/yKxmdGoIp4j5ysNT4u3S2pDQ3Q==", + "optional": true }, "node-object-hash": { "version": "2.3.10", @@ -7775,6 +8139,11 @@ "yocto-queue": "^0.1.0" } }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + }, "package-json": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", @@ -8021,6 +8390,24 @@ "picomatch": "^2.2.1" } }, + "redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "registry-auth-token": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.1.tgz", @@ -8364,6 +8751,11 @@ "tweetnacl": "~0.14.0" } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", diff --git a/backend/package.json b/backend/package.json index a0bf2b957..4d8f62e7b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "bcrypt": "5.0.1", + "bullmq": "1.78.1", "cors": "2.8.5", "cron": "1.8.2", "dotenv": "10.0.0", @@ -23,6 +24,7 @@ "express-rate-limit": "6.2.1", "firebase-admin": "10.0.2", "helmet": "4.6.0", + "ioredis": "4.28.5", "joi": "17.6.0", "lodash": "4.17.21", "mongodb": "4.4.0", @@ -42,6 +44,7 @@ "@types/bcrypt": "5.0.0", "@types/cors": "2.8.12", "@types/cron": "1.7.3", + "@types/ioredis": "4.28.10", "@types/lodash": "4.14.178", "@types/node": "17.0.18", "@types/node-fetch": "2.6.1", diff --git a/backend/server.ts b/backend/server.ts index 79f8b0a22..c93250a45 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -9,6 +9,8 @@ import app from "./app"; import { Server } from "http"; import { version } from "./version"; import { recordServerVersion } from "./utils/prometheus"; +import RedisClient from "./init/redis"; +import George from "./tasks/george"; async function bootServer(port: number): Promise { try { @@ -28,6 +30,14 @@ async function bootServer(port: number): Promise { await ConfigurationClient.getLiveConfiguration(); console.log("Live configuration fetched"); + console.log("Connecting to redis..."); + await RedisClient.connect(); + console.log("Connected to redis"); + + console.log("Initializing task queues..."); + George.initJobQueue(RedisClient.getConnection()); + console.log("Task queues initialized"); + console.log("Starting cron jobs..."); jobs.forEach((job) => job.start()); console.log("Cron jobs started"); diff --git a/backend/tasks/george.ts b/backend/tasks/george.ts new file mode 100644 index 000000000..fd2b29e33 --- /dev/null +++ b/backend/tasks/george.ts @@ -0,0 +1,98 @@ +import { Queue, QueueScheduler } from "bullmq"; +import type IORedis from "ioredis"; + +const QUEUE_NAME = "george-tasks"; + +interface GeorgeTask { + command: string; + arguments: any[]; +} + +function buildGeorgeTask(command: string, taskArguments: any[]): GeorgeTask { + return { + command, + arguments: taskArguments, + }; +} + +class George { + jobQueue: Queue; + jobQueueScheduler: QueueScheduler; + + initJobQueue(redisConnection: IORedis.Redis): void { + this.jobQueue = new Queue(QUEUE_NAME, { + connection: redisConnection, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + attempts: 3, + backoff: { + type: "exponential", + delay: 2000, + }, + }, + }); + + this.jobQueueScheduler = new QueueScheduler(QUEUE_NAME, { + connection: redisConnection, + }); + } + + async updateDiscordRole(discordId: string, wpm: number): Promise { + const command = "updateRole"; + const updateDiscordRoleTask = buildGeorgeTask(command, [discordId, wpm]); + await this.jobQueue.add(command, updateDiscordRoleTask); + } + + async linkDiscord(discordId: string, uid: string): Promise { + const command = "linkDiscord"; + const linkDiscordTask = buildGeorgeTask(command, [discordId, uid]); + await this.jobQueue.add(command, linkDiscordTask); + } + + async unlinkDiscord(discordId: string, uid: string): Promise { + const command = "unlinkDiscord"; + const unlinkDiscordTask = buildGeorgeTask(command, [discordId, uid]); + await this.jobQueue.add(command, unlinkDiscordTask); + } + + async awardChallenge( + discordId: string, + challengeName: string + ): Promise { + const command = "awardChallenge"; + const awardChallengeTask = buildGeorgeTask(command, [ + discordId, + challengeName, + ]); + await this.jobQueue.add(command, awardChallengeTask); + } + + async announceLbUpdate( + newRecords: any[], + leaderboardId: string + ): Promise { + const command = "announceLbUpdate"; + + const leaderboardUpdateTasks = newRecords.map((record) => { + const taskData = buildGeorgeTask(command, [ + record.discordId ?? record.name, + record.rank, + leaderboardId, + record.wpm, + record.raw, + record.acc, + record.consistency, + ]); + + return { + name: command, + data: taskData, + }; + }); + + await this.jobQueue.addBulk(leaderboardUpdateTasks); + } +} + +export default new George(); diff --git a/backend/types/types.d.ts b/backend/types/types.d.ts index 4a5e02d53..fae6562ab 100644 --- a/backend/types/types.d.ts +++ b/backend/types/types.d.ts @@ -24,6 +24,9 @@ declare namespace MonkeyTypes { enableSavingResults: { enabled: boolean; }; + useRedisForBotTasks: { + enabled: boolean; + }; } interface DecodedToken {