From e5e03b603f0b456cde146d45233baecae9e9a4b5 Mon Sep 17 00:00:00 2001 From: Christian Fehmer Date: Fri, 10 Nov 2023 14:55:35 +0100 Subject: [PATCH] feat: Allow more results for premium users (fehmer) (#4767) * wip: More results and filters for premium users * use offset+limit instead of beforeTimestamp, add configuration for max results for regular/premium users * add isPremium to /users response * cleanup * review comments * review comments * update base config * update premium data type * add undefined check * interface name * add start timestamp * refactor * move function to util, rename params, use isFinite * merge fix * fixed tests --------- Co-authored-by: Miodec --- .../__tests__/api/controllers/result.spec.ts | 179 ++++++++++++++++++ backend/__tests__/dal/result.spec.ts | 132 +++++++++++++ backend/src/api/controllers/result.ts | 19 ++ backend/src/api/routes/results.ts | 2 + backend/src/constants/base-configuration.ts | 20 ++ backend/src/dal/result.ts | 22 ++- backend/src/dal/user.ts | 9 + backend/src/documentation/public-swagger.json | 19 +- backend/src/middlewares/rate-limit.ts | 5 +- backend/src/types/shared.ts | 4 + backend/src/types/types.d.ts | 6 + backend/src/utils/misc.ts | 10 + 12 files changed, 416 insertions(+), 11 deletions(-) create mode 100644 backend/__tests__/api/controllers/result.spec.ts create mode 100644 backend/__tests__/dal/result.spec.ts diff --git a/backend/__tests__/api/controllers/result.spec.ts b/backend/__tests__/api/controllers/result.spec.ts new file mode 100644 index 000000000..f08957408 --- /dev/null +++ b/backend/__tests__/api/controllers/result.spec.ts @@ -0,0 +1,179 @@ +import request from "supertest"; +import app from "../../../src/app"; +import * as Configuration from "../../../src/init/configuration"; +import * as ResultDal from "../../../src/dal/result"; +import * as UserDal from "../../../src/dal/user"; +import * as AuthUtils from "../../../src/utils/auth"; +import { DecodedIdToken } from "firebase-admin/lib/auth/token-verifier"; +import { messaging } from "firebase-admin"; +const uid = "123456"; + +const mockDecodedToken: DecodedIdToken = { + uid, + email: "newuser@mail.com", + iat: 0, +} as DecodedIdToken; + +jest.spyOn(AuthUtils, "verifyIdToken").mockResolvedValue(mockDecodedToken); + +const resultMock = jest.spyOn(ResultDal, "getResults"); + +const mockApp = request(app); + +const configuration = Configuration.getCachedConfiguration(); + +describe("result controller test", () => { + describe("getResults", () => { + beforeEach(() => { + resultMock.mockResolvedValue([]); + }); + afterEach(() => { + resultMock.mockReset(); + }); + it("should get latest 1000 results for regular user", async () => { + //GIVEN + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false); + //WHEN + await mockApp + .get("/results") + .set("Authorization", "Bearer 123456789") + .send() + .expect(200); + + //THEN + expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, { + limit: 1000, + offset: 0, + onOrAfterTimestamp: NaN, + }); + }); + it("should get results filter by onOrAfterTimestamp", async () => { + //GIVEN + const now = Date.now(); + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false); + //WHEN + await mockApp + .get("/results") + .query({ onOrAfterTimestamp: now }) + .set("Authorization", "Bearer 123456789") + .send() + .expect(200); + + //THEN + + expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, { + limit: 1000, + offset: 0, + onOrAfterTimestamp: now, + }); + }); + it("should get with limit and offset", async () => { + //GIVEN + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false); + + //WHEN + await mockApp + .get("/results") + .query({ offset: 500, limit: 250 }) + .set("Authorization", "Bearer 123456789") + .send() + .expect(200); + + //THEN + expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, { + limit: 250, + offset: 500, + onOrAfterTimestamp: NaN, + }); + }); + it("should fail exceeding max limit for regular user", async () => { + //GIVEN + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false); + + //WHEN + await mockApp + .get("/results") + .query({ limit: 600, offset: 800 }) + .set("Authorization", "Bearer 123456789") + .send() + .expect(422) + .expect( + expectErrorMessage( + `Max results limit of ${ + ( + await configuration + ).results.limits.regularUser + } exceeded.` + ) + ); + + //THEN + }); + it("should get with higher max limit for premium user", async () => { + //GIVEN + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(true); + + //WHEN + await mockApp + .get("/results") + .query({ offset: 600, limit: 800 }) + .set("Authorization", "Bearer 123456789") + .send() + .expect(200); + + //THEN + + expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, { + limit: 800, + offset: 600, + onOrAfterTimestamp: NaN, + }); + }); + it("should fail exceeding 1k limit", async () => { + //GIVEN + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false); + + //WHEN + await mockApp + .get("/results") + .query({ limit: 2000 }) + .set("Authorization", "Bearer 123456789") + .send() + .expect(422) + .expect( + expectErrorMessage( + '"limit" must be less than or equal to 1000 (2000)' + ) + ); + + //THEN + }); + it("should fail exceeding maxlimit for premium user", async () => { + //GIVEN + jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(true); + + //WHEN + await mockApp + .get("/results") + .query({ limit: 1000, offset: 24900 }) + .set("Authorization", "Bearer 123456789") + .send() + .expect(422) + .expect( + expectErrorMessage( + `Max results limit of ${ + ( + await configuration + ).results.limits.premiumUser + } exceeded.` + ) + ); + + //THEN + }); + }); +}); + +function expectErrorMessage(message: string): (res: request.Response) => void { + return (res) => expect(res.body).toHaveProperty("message", message); +} diff --git a/backend/__tests__/dal/result.spec.ts b/backend/__tests__/dal/result.spec.ts new file mode 100644 index 000000000..a0914cc9b --- /dev/null +++ b/backend/__tests__/dal/result.spec.ts @@ -0,0 +1,132 @@ +import * as ResultDal from "../../src/dal/result"; +import { ObjectId } from "mongodb"; +import * as UserDal from "../../src/dal/user"; + +type MonkeyTypesResult = MonkeyTypes.Result; + +let uid: string = ""; +const timestamp = Date.now() - 60000; + +async function createDummyData( + uid: string, + count: number, + timestamp: number, + tag?: string +): Promise { + const dummyUser: MonkeyTypes.User = { + uid, + addedAt: 0, + email: "test@example.com", + name: "Bob", + personalBests: { + time: {}, + words: {}, + quote: {}, + custom: {}, + zen: {}, + }, + }; + + jest.spyOn(UserDal, "getUser").mockResolvedValue(dummyUser); + const tags: string[] = []; + if (tag !== undefined) tags.push(tag); + for (let i = 0; i < count; i++) { + await ResultDal.addResult(uid, { + _id: new ObjectId(), + wpm: i, + rawWpm: i, + charStats: [], + acc: 0, + mode: "time", + mode2: "10" as never, + quoteLength: 1, + timestamp, + restartCount: 0, + incompleteTestSeconds: 0, + incompleteTests: [], + testDuration: 10, + afkDuration: 0, + tags, + consistency: 100, + keyConsistency: 100, + chartData: { wpm: [], raw: [], err: [] }, + uid, + keySpacingStats: { average: 0, sd: 0 }, + keyDurationStats: { average: 0, sd: 0 }, + difficulty: "normal", + language: "english", + } as MonkeyTypesResult); + } +} +describe("ResultDal", () => { + describe("getResults", () => { + beforeEach(() => { + uid = new ObjectId().toHexString(); + }); + afterEach(async () => { + if (uid) await ResultDal.deleteAll(uid); + }); + + it("should read lastest 10 results ordered by timestamp", async () => { + //GIVEN + await createDummyData(uid, 10, timestamp - 2000, "old"); + await createDummyData(uid, 20, timestamp, "current"); + + //WHEN + const results = await ResultDal.getResults(uid, { limit: 10 }); + + //THEN + expect(results).toHaveLength(10); + let last = results[0].timestamp; + results.forEach((it) => { + expect(it.tags).toContain("current"); + expect(it.timestamp).toBeGreaterThanOrEqual(last); + last = it.timestamp; + }); + }); + it("should read all if not limited", async () => { + //GIVEN + await createDummyData(uid, 10, timestamp - 2000, "old"); + await createDummyData(uid, 20, timestamp, "current"); + + //WHEN + const results = await ResultDal.getResults(uid, {}); + + //THEN + expect(results).toHaveLength(30); + }); + it("should read results onOrAfterTimestamp", async () => { + //GIVEN + await createDummyData(uid, 10, timestamp - 2000, "old"); + await createDummyData(uid, 20, timestamp, "current"); + + //WHEN + const results = await ResultDal.getResults(uid, { + onOrAfterTimestamp: timestamp, + }); + + //THEN + expect(results).toHaveLength(20); + results.forEach((it) => { + expect(it.tags).toContain("current"); + }); + }); + it("should read next 10 results", async () => { + //GIVEN + await createDummyData(uid, 10, timestamp - 2000, "old"); + await createDummyData(uid, 20, timestamp, "current"); + + //WHEN + const results = await ResultDal.getResults(uid, { + limit: 10, + offset: 20, + }); + + //THEN + expect(results).toHaveLength(10); + results.forEach((it) => { + expect(it.tags).toContain("old"); + }); + }); + }); +}); diff --git a/backend/src/api/controllers/result.ts b/backend/src/api/controllers/result.ts index eba6ef48e..36a31876e 100644 --- a/backend/src/api/controllers/result.ts +++ b/backend/src/api/controllers/result.ts @@ -14,6 +14,7 @@ import { mapRange, roundTo2, stdDev, + stringToNumberOrDefault, } from "../../utils/misc"; import objectHash from "object-hash"; import Logger from "../../utils/logger"; @@ -63,12 +64,30 @@ export async function getResults( req: MonkeyTypes.Request ): Promise { const { uid } = req.ctx.decodedToken; + const isPremium = await UserDAL.checkIfUserIsPremium(uid); + + const maxLimit = isPremium + ? req.ctx.configuration.results.limits.premiumUser + : req.ctx.configuration.results.limits.regularUser; + const onOrAfterTimestamp = parseInt( req.query.onOrAfterTimestamp as string, 10 ); + const limit = stringToNumberOrDefault( + req.query.limit as string, + Math.min(1000, maxLimit) + ); + const offset = stringToNumberOrDefault(req.query.offset as string, 0); + + if (limit + offset > maxLimit) { + throw new MonkeyError(422, `Max results limit of ${maxLimit} exceeded.`); + } + const results = await ResultDAL.getResults(uid, { onOrAfterTimestamp, + limit, + offset, }); return new MonkeyResponse("Results retrieved", results); } diff --git a/backend/src/api/routes/results.ts b/backend/src/api/routes/results.ts index fb413600c..1d147b7ad 100644 --- a/backend/src/api/routes/results.ts +++ b/backend/src/api/routes/results.ts @@ -22,6 +22,8 @@ router.get( validateRequest({ query: { onOrAfterTimestamp: joi.number().integer().min(1589428800000), + limit: joi.number().integer().min(0).max(1000), + offset: joi.number().integer().min(0), }, }), asyncHandler(ResultController.getResults) diff --git a/backend/src/constants/base-configuration.ts b/backend/src/constants/base-configuration.ts index 7f305aec5..d1302bf92 100644 --- a/backend/src/constants/base-configuration.ts +++ b/backend/src/constants/base-configuration.ts @@ -14,6 +14,10 @@ export const BASE_CONFIGURATION: Configuration = { enabled: false, maxPresetsPerUser: 0, }, + limits: { + regularUser: 1000, + premiumUser: 10000, + }, }, quotes: { reporting: { @@ -170,6 +174,22 @@ export const CONFIGURATION_FORM_SCHEMA: ObjectSchema = { }, }, }, + limits: { + type: "object", + label: "maximum results", + fields: { + regularUser: { + type: "number", + label: "for regular users", + min: 0, + }, + premiumUser: { + type: "number", + label: "for premium users", + min: 0, + }, + }, + }, }, }, quotes: { diff --git a/backend/src/dal/result.ts b/backend/src/dal/result.ts index 19d2df4e5..cf13943b4 100644 --- a/backend/src/dal/result.ts +++ b/backend/src/dal/result.ts @@ -90,16 +90,16 @@ export async function getResultByTimestamp( interface GetResultsOpts { onOrAfterTimestamp?: number; - start?: number; - end?: number; + limit?: number; + offset?: number; } export async function getResults( uid: string, opts?: GetResultsOpts ): Promise { - const { onOrAfterTimestamp, start, end } = opts ?? {}; - const results = await db + const { onOrAfterTimestamp, offset, limit } = opts ?? {}; + let query = db .collection("results") .find({ uid, @@ -108,10 +108,16 @@ export async function getResults( timestamp: { $gte: onOrAfterTimestamp }, }), }) - .sort({ timestamp: -1 }) - .skip(start ?? 0) - .limit(end ?? 1000) - .toArray(); // this needs to be changed to later take patreon into consideration + .sort({ timestamp: -1 }); + + if (limit !== undefined) { + query = query.limit(limit); + } + if (offset !== undefined) { + query = query.skip(offset); + } + + const results = await query.toArray(); if (!results) throw new MonkeyError(404, "Result not found"); return results; } diff --git a/backend/src/dal/user.ts b/backend/src/dal/user.ts index ce6fd6454..122b9df8b 100644 --- a/backend/src/dal/user.ts +++ b/backend/src/dal/user.ts @@ -1036,3 +1036,12 @@ export async function setBanned(uid: string, banned: boolean): Promise { await getUsersCollection().updateOne({ uid }, { $unset: { banned: "" } }); } } + +export async function checkIfUserIsPremium(uid: string): Promise { + const user = await getUser(uid, "checkIfUserIsPremium"); + const expirationDate = user.premium?.expirationTimestamp; + + if (expirationDate === undefined) return false; + if (expirationDate === -1) return true; //lifetime + return expirationDate > Date.now(); +} diff --git a/backend/src/documentation/public-swagger.json b/backend/src/documentation/public-swagger.json index 1331058d9..7fdd3d195 100644 --- a/backend/src/documentation/public-swagger.json +++ b/backend/src/documentation/public-swagger.json @@ -123,7 +123,7 @@ "/results": { "get": { "tags": ["results"], - "summary": "Gets up to 1000 results (endpoint limited to 1 request per hour)", + "summary": "Gets up to 1000 results (endpoint limited to 30 requests per day)", "parameters": [ { "name": "onOrAfterTimestamp", @@ -131,6 +131,23 @@ "description": "Get results on or after a Unix timestamp in milliseconds. Must be no earlier than Thu May 14 2020 04:00:00 GMT+0000 (i.e., 1670454228000). If omitted, defaults to the most recent results.", "required": false, "type": "number" + }, + { + "name": "limit", + "in": "query", + "description": "The maximum number of items to return per page.", + "required": false, + "type": "number", + "minimum": 0, + "maximum": 1000 + }, + { + "name": "offset", + "in": "query", + "description": "The offset of the item at which to begin the response.", + "required": false, + "type": "number", + "minimum": 0 } ], "responses": { diff --git a/backend/src/middlewares/rate-limit.ts b/backend/src/middlewares/rate-limit.ts index beb921d8a..27706827a 100644 --- a/backend/src/middlewares/rate-limit.ts +++ b/backend/src/middlewares/rate-limit.ts @@ -31,6 +31,7 @@ export const customHandler = ( const ONE_HOUR_SECONDS = 60 * 60; const ONE_HOUR_MS = 1000 * ONE_HOUR_SECONDS; +const ONE_DAY_MS = 24 * ONE_HOUR_MS; // Root Rate Limit export const rootRateLimiter = rateLimit({ @@ -257,8 +258,8 @@ export const resultsGet = rateLimit({ // Results Routing export const resultsGetApe = rateLimit({ - windowMs: ONE_HOUR_MS, - max: 1 * REQUEST_MULTIPLIER, + windowMs: ONE_DAY_MS, + max: 30 * REQUEST_MULTIPLIER, keyGenerator: getKeyWithUid, handler: customHandler, }); diff --git a/backend/src/types/shared.ts b/backend/src/types/shared.ts index ff8148750..d3152f654 100644 --- a/backend/src/types/shared.ts +++ b/backend/src/types/shared.ts @@ -29,6 +29,10 @@ export interface Configuration { enabled: boolean; maxPresetsPerUser: number; }; + limits: { + regularUser: number; + premiumUser: number; + }; }; users: { signUp: boolean; diff --git a/backend/src/types/types.d.ts b/backend/src/types/types.d.ts index 45bb9e46a..fb17ebd67 100644 --- a/backend/src/types/types.d.ts +++ b/backend/src/types/types.d.ts @@ -92,6 +92,7 @@ declare namespace MonkeyTypes { streak?: UserStreak; lastReultHashes?: string[]; lbOptOut?: boolean; + premium?: PremiumInfo; } interface UserStreak { @@ -383,4 +384,9 @@ declare namespace MonkeyTypes { frontendForcedConfig?: Record; frontendFunctions?: string[]; } + + interface PremiumInfo { + startTimestamp: number; + expirationTimestamp: number; + } } diff --git a/backend/src/utils/misc.ts b/backend/src/utils/misc.ts index af30ea0d0..d838ed3ab 100644 --- a/backend/src/utils/misc.ts +++ b/backend/src/utils/misc.ts @@ -288,3 +288,13 @@ export function intersect(a: T[], b: T[], removeDuplicates = false): T[] { }); return removeDuplicates ? [...new Set(filtered)] : filtered; } + +export function stringToNumberOrDefault( + string: string, + defaultValue: number +): number { + if (string === undefined) return defaultValue; + const value = parseInt(string, 10); + if (!Number.isFinite(value)) return defaultValue; + return value; +}