feat: Allow more results for premium users (fehmer) (#4767)

* wip: More results and filters for premium users

* use offset+limit instead of beforeTimestamp, add configuration for max results for regular/premium users

* add isPremium to /users response

* cleanup

* review comments

* review comments

* update base config

* update premium data type

* add undefined check

* interface name

* add start timestamp

* refactor

* move function to util, rename params, use isFinite

* merge fix

* fixed tests

---------

Co-authored-by: Miodec <jack@monkeytype.com>

backend/__tests__/api/controllers/result.spec.ts

@@ -0,0 +1,179 @@
+import request from "supertest";
+import app from "../../../src/app";
+import * as Configuration from "../../../src/init/configuration";
+import * as ResultDal from "../../../src/dal/result";
+import * as UserDal from "../../../src/dal/user";
+import * as AuthUtils from "../../../src/utils/auth";
+import { DecodedIdToken } from "firebase-admin/lib/auth/token-verifier";
+import { messaging } from "firebase-admin";
+const uid = "123456";
+
+const mockDecodedToken: DecodedIdToken = {
+  uid,
+  email: "newuser@mail.com",
+  iat: 0,
+} as DecodedIdToken;
+
+jest.spyOn(AuthUtils, "verifyIdToken").mockResolvedValue(mockDecodedToken);
+
+const resultMock = jest.spyOn(ResultDal, "getResults");
+
+const mockApp = request(app);
+
+const configuration = Configuration.getCachedConfiguration();
+
+describe("result controller test", () => {
+  describe("getResults", () => {
+    beforeEach(() => {
+      resultMock.mockResolvedValue([]);
+    });
+    afterEach(() => {
+      resultMock.mockReset();
+    });
+    it("should get latest 1000 results for regular user", async () => {
+      //GIVEN
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false);
+      //WHEN
+      await mockApp
+        .get("/results")
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(200);
+
+      //THEN
+      expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, {
+        limit: 1000,
+        offset: 0,
+        onOrAfterTimestamp: NaN,
+      });
+    });
+    it("should get results filter by onOrAfterTimestamp", async () => {
+      //GIVEN
+      const now = Date.now();
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false);
+      //WHEN
+      await mockApp
+        .get("/results")
+        .query({ onOrAfterTimestamp: now })
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(200);
+
+      //THEN
+
+      expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, {
+        limit: 1000,
+        offset: 0,
+        onOrAfterTimestamp: now,
+      });
+    });
+    it("should get with limit and offset", async () => {
+      //GIVEN
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false);
+
+      //WHEN
+      await mockApp
+        .get("/results")
+        .query({ offset: 500, limit: 250 })
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(200);
+
+      //THEN
+      expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, {
+        limit: 250,
+        offset: 500,
+        onOrAfterTimestamp: NaN,
+      });
+    });
+    it("should fail exceeding max limit for regular user", async () => {
+      //GIVEN
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false);
+
+      //WHEN
+      await mockApp
+        .get("/results")
+        .query({ limit: 600, offset: 800 })
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(422)
+        .expect(
+          expectErrorMessage(
+            `Max results limit of ${
+              (
+                await configuration
+              ).results.limits.regularUser
+            } exceeded.`
+          )
+        );
+
+      //THEN
+    });
+    it("should get with higher max limit for premium user", async () => {
+      //GIVEN
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(true);
+
+      //WHEN
+      await mockApp
+        .get("/results")
+        .query({ offset: 600, limit: 800 })
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(200);
+
+      //THEN
+
+      expect(resultMock).toHaveBeenCalledWith(mockDecodedToken.uid, {
+        limit: 800,
+        offset: 600,
+        onOrAfterTimestamp: NaN,
+      });
+    });
+    it("should fail exceeding 1k limit", async () => {
+      //GIVEN
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(false);
+
+      //WHEN
+      await mockApp
+        .get("/results")
+        .query({ limit: 2000 })
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(422)
+        .expect(
+          expectErrorMessage(
+            '"limit" must be less than or equal to 1000 (2000)'
+          )
+        );
+
+      //THEN
+    });
+    it("should fail exceeding maxlimit for premium user", async () => {
+      //GIVEN
+      jest.spyOn(UserDal, "checkIfUserIsPremium").mockResolvedValue(true);
+
+      //WHEN
+      await mockApp
+        .get("/results")
+        .query({ limit: 1000, offset: 24900 })
+        .set("Authorization", "Bearer 123456789")
+        .send()
+        .expect(422)
+        .expect(
+          expectErrorMessage(
+            `Max results limit of ${
+              (
+                await configuration
+              ).results.limits.premiumUser
+            } exceeded.`
+          )
+        );
+
+      //THEN
+    });
+  });
+});
+
+function expectErrorMessage(message: string): (res: request.Response) => void {
+  return (res) => expect(res.body).toHaveProperty("message", message);
+}

backend/__tests__/dal/result.spec.ts

@@ -0,0 +1,132 @@
+import * as ResultDal from "../../src/dal/result";
+import { ObjectId } from "mongodb";
+import * as UserDal from "../../src/dal/user";
+
+type MonkeyTypesResult = MonkeyTypes.Result<MonkeyTypes.Mode>;
+
+let uid: string = "";
+const timestamp = Date.now() - 60000;
+
+async function createDummyData(
+  uid: string,
+  count: number,
+  timestamp: number,
+  tag?: string
+): Promise<void> {
+  const dummyUser: MonkeyTypes.User = {
+    uid,
+    addedAt: 0,
+    email: "test@example.com",
+    name: "Bob",
+    personalBests: {
+      time: {},
+      words: {},
+      quote: {},
+      custom: {},
+      zen: {},
+    },
+  };
+
+  jest.spyOn(UserDal, "getUser").mockResolvedValue(dummyUser);
+  const tags: string[] = [];
+  if (tag !== undefined) tags.push(tag);
+  for (let i = 0; i < count; i++) {
+    await ResultDal.addResult(uid, {
+      _id: new ObjectId(),
+      wpm: i,
+      rawWpm: i,
+      charStats: [],
+      acc: 0,
+      mode: "time",
+      mode2: "10" as never,
+      quoteLength: 1,
+      timestamp,
+      restartCount: 0,
+      incompleteTestSeconds: 0,
+      incompleteTests: [],
+      testDuration: 10,
+      afkDuration: 0,
+      tags,
+      consistency: 100,
+      keyConsistency: 100,
+      chartData: { wpm: [], raw: [], err: [] },
+      uid,
+      keySpacingStats: { average: 0, sd: 0 },
+      keyDurationStats: { average: 0, sd: 0 },
+      difficulty: "normal",
+      language: "english",
+    } as MonkeyTypesResult);
+  }
+}
+describe("ResultDal", () => {
+  describe("getResults", () => {
+    beforeEach(() => {
+      uid = new ObjectId().toHexString();
+    });
+    afterEach(async () => {
+      if (uid) await ResultDal.deleteAll(uid);
+    });
+
+    it("should read lastest 10 results ordered by timestamp", async () => {
+      //GIVEN
+      await createDummyData(uid, 10, timestamp - 2000, "old");
+      await createDummyData(uid, 20, timestamp, "current");
+
+      //WHEN
+      const results = await ResultDal.getResults(uid, { limit: 10 });
+
+      //THEN
+      expect(results).toHaveLength(10);
+      let last = results[0].timestamp;
+      results.forEach((it) => {
+        expect(it.tags).toContain("current");
+        expect(it.timestamp).toBeGreaterThanOrEqual(last);
+        last = it.timestamp;
+      });
+    });
+    it("should read all if not limited", async () => {
+      //GIVEN
+      await createDummyData(uid, 10, timestamp - 2000, "old");
+      await createDummyData(uid, 20, timestamp, "current");
+
+      //WHEN
+      const results = await ResultDal.getResults(uid, {});
+
+      //THEN
+      expect(results).toHaveLength(30);
+    });
+    it("should read results onOrAfterTimestamp", async () => {
+      //GIVEN
+      await createDummyData(uid, 10, timestamp - 2000, "old");
+      await createDummyData(uid, 20, timestamp, "current");
+
+      //WHEN
+      const results = await ResultDal.getResults(uid, {
+        onOrAfterTimestamp: timestamp,
+      });
+
+      //THEN
+      expect(results).toHaveLength(20);
+      results.forEach((it) => {
+        expect(it.tags).toContain("current");
+      });
+    });
+    it("should read next 10 results", async () => {
+      //GIVEN
+      await createDummyData(uid, 10, timestamp - 2000, "old");
+      await createDummyData(uid, 20, timestamp, "current");
+
+      //WHEN
+      const results = await ResultDal.getResults(uid, {
+        limit: 10,
+        offset: 20,
+      });
+
+      //THEN
+      expect(results).toHaveLength(10);
+      results.forEach((it) => {
+        expect(it.tags).toContain("old");
+      });
+    });
+  });
+});

backend/src/api/controllers/result.ts

@@ -14,6 +14,7 @@ import {
   mapRange,
   roundTo2,
   stdDev,
+  stringToNumberOrDefault,
 } from "../../utils/misc";
 import objectHash from "object-hash";
 import Logger from "../../utils/logger";
@@ -63,12 +64,30 @@ export async function getResults(
   req: MonkeyTypes.Request
 ): Promise<MonkeyResponse> {
   const { uid } = req.ctx.decodedToken;
+  const isPremium = await UserDAL.checkIfUserIsPremium(uid);
+
+  const maxLimit = isPremium
+    ? req.ctx.configuration.results.limits.premiumUser
+    : req.ctx.configuration.results.limits.regularUser;
+
   const onOrAfterTimestamp = parseInt(
     req.query.onOrAfterTimestamp as string,
     10
   );
+  const limit = stringToNumberOrDefault(
+    req.query.limit as string,
+    Math.min(1000, maxLimit)
+  );
+  const offset = stringToNumberOrDefault(req.query.offset as string, 0);
+
+  if (limit + offset > maxLimit) {
+    throw new MonkeyError(422, `Max results limit of ${maxLimit} exceeded.`);
+  }
+
   const results = await ResultDAL.getResults(uid, {
     onOrAfterTimestamp,
+    limit,
+    offset,
   });
   return new MonkeyResponse("Results retrieved", results);
 }

backend/src/api/routes/results.ts

@@ -22,6 +22,8 @@ router.get(
   validateRequest({
     query: {
       onOrAfterTimestamp: joi.number().integer().min(1589428800000),
+      limit: joi.number().integer().min(0).max(1000),
+      offset: joi.number().integer().min(0),
     },
   }),
   asyncHandler(ResultController.getResults)

backend/src/constants/base-configuration.ts

@@ -14,6 +14,10 @@ export const BASE_CONFIGURATION: Configuration = {
       enabled: false,
       maxPresetsPerUser: 0,
     },
+    limits: {
+      regularUser: 1000,
+      premiumUser: 10000,
+    },
   },
   quotes: {
     reporting: {
@@ -170,6 +174,22 @@ export const CONFIGURATION_FORM_SCHEMA: ObjectSchema<Configuration> = {
             },
           },
         },
+        limits: {
+          type: "object",
+          label: "maximum results",
+          fields: {
+            regularUser: {
+              type: "number",
+              label: "for regular users",
+              min: 0,
+            },
+            premiumUser: {
+              type: "number",
+              label: "for premium users",
+              min: 0,
+            },
+          },
+        },
       },
     },
     quotes: {

backend/src/dal/result.ts

@@ -90,16 +90,16 @@ export async function getResultByTimestamp(
 
 interface GetResultsOpts {
   onOrAfterTimestamp?: number;
-  start?: number;
-  end?: number;
+  limit?: number;
+  offset?: number;
 }
 
 export async function getResults(
   uid: string,
   opts?: GetResultsOpts
 ): Promise<MonkeyTypesResult[]> {
-  const { onOrAfterTimestamp, start, end } = opts ?? {};
-  const results = await db
+  const { onOrAfterTimestamp, offset, limit } = opts ?? {};
+  let query = db
     .collection<MonkeyTypesResult>("results")
     .find({
       uid,
@@ -108,10 +108,16 @@ export async function getResults(
           timestamp: { $gte: onOrAfterTimestamp },
         }),
     })
-    .sort({ timestamp: -1 })
-    .skip(start ?? 0)
-    .limit(end ?? 1000)
-    .toArray(); // this needs to be changed to later take patreon into consideration
+    .sort({ timestamp: -1 });
+
+  if (limit !== undefined) {
+    query = query.limit(limit);
+  }
+  if (offset !== undefined) {
+    query = query.skip(offset);
+  }
+
+  const results = await query.toArray();
   if (!results) throw new MonkeyError(404, "Result not found");
   return results;
 }

backend/src/dal/user.ts

@@ -1036,3 +1036,12 @@ export async function setBanned(uid: string, banned: boolean): Promise<void> {
     await getUsersCollection().updateOne({ uid }, { $unset: { banned: "" } });
   }
 }
+
+export async function checkIfUserIsPremium(uid: string): Promise<boolean> {
+  const user = await getUser(uid, "checkIfUserIsPremium");
+  const expirationDate = user.premium?.expirationTimestamp;
+
+  if (expirationDate === undefined) return false;
+  if (expirationDate === -1) return true; //lifetime
+  return expirationDate > Date.now();
+}

backend/src/documentation/public-swagger.json

@@ -123,7 +123,7 @@
     "/results": {
       "get": {
         "tags": ["results"],
-        "summary": "Gets up to 1000 results (endpoint limited to 1 request per hour)",
+        "summary": "Gets up to 1000 results (endpoint limited to 30 requests per day)",
         "parameters": [
           {
             "name": "onOrAfterTimestamp",
@@ -131,6 +131,23 @@
             "description": "Get results on or after a Unix timestamp in milliseconds. Must be no earlier than Thu May 14 2020 04:00:00 GMT+0000 (i.e., 1670454228000). If omitted, defaults to the most recent results.",
             "required": false,
             "type": "number"
+          },
+          {
+            "name": "limit",
+            "in": "query",
+            "description": "The maximum number of items to return per page.",
+            "required": false,
+            "type": "number",
+            "minimum": 0,
+            "maximum": 1000
+          },
+          {
+            "name": "offset",
+            "in": "query",
+            "description": "The offset of the item at which to begin the response.",
+            "required": false,
+            "type": "number",
+            "minimum": 0
           }
         ],
         "responses": {

backend/src/middlewares/rate-limit.ts

@@ -31,6 +31,7 @@ export const customHandler = (
 
 const ONE_HOUR_SECONDS = 60 * 60;
 const ONE_HOUR_MS = 1000 * ONE_HOUR_SECONDS;
+const ONE_DAY_MS = 24 * ONE_HOUR_MS;
 
 // Root Rate Limit
 export const rootRateLimiter = rateLimit({
@@ -257,8 +258,8 @@ export const resultsGet = rateLimit({
 
 // Results Routing
 export const resultsGetApe = rateLimit({
-  windowMs: ONE_HOUR_MS,
-  max: 1 * REQUEST_MULTIPLIER,
+  windowMs: ONE_DAY_MS,
+  max: 30 * REQUEST_MULTIPLIER,
   keyGenerator: getKeyWithUid,
   handler: customHandler,
 });

backend/src/types/shared.ts

@@ -29,6 +29,10 @@ export interface Configuration {
       enabled: boolean;
       maxPresetsPerUser: number;
     };
+    limits: {
+      regularUser: number;
+      premiumUser: number;
+    };
   };
   users: {
     signUp: boolean;

backend/src/types/types.d.ts

@@ -92,6 +92,7 @@ declare namespace MonkeyTypes {
     streak?: UserStreak;
     lastReultHashes?: string[];
     lbOptOut?: boolean;
+    premium?: PremiumInfo;
   }
 
   interface UserStreak {
@@ -383,4 +384,9 @@ declare namespace MonkeyTypes {
     frontendForcedConfig?: Record<string, string[] | boolean[]>;
     frontendFunctions?: string[];
   }
+
+  interface PremiumInfo {
+    startTimestamp: number;
+    expirationTimestamp: number;
+  }
 }

backend/src/utils/misc.ts

@@ -288,3 +288,13 @@ export function intersect<T>(a: T[], b: T[], removeDuplicates = false): T[] {
   });
   return removeDuplicates ? [...new Set(filtered)] : filtered;
 }
+
+export function stringToNumberOrDefault(
+  string: string,
+  defaultValue: number
+): number {
+  if (string === undefined) return defaultValue;
+  const value = parseInt(string, 10);
+  if (!Number.isFinite(value)) return defaultValue;
+  return value;
+}