Save speed stats in leaderboard update (#3652) mrbrianevans

* Save speed stats in leaderboard update

Saves a histogram data structure of speeds for buckets rounded to the nearest 10.

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* API endpoint to get public speed stats

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Add APE class for public stats (WIP)

I created an APE class for accessing public stats. Also stubbed getting and showing the public speed stats on the about page. Haven't implemented the histogram yet though.

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Draw histogram for global speed stats

On about page

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Update histogram colors on theme change

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Fixed out-of-order data in speed histogram

 Data was not sorted correctly, which resulted in an incorrect histogram being drawn.

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Public speed stats PR fixes

Small fixes based on PR feedback:
 - changed _req to req
 - removed unnecessary client version header

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Add endpoint for typing stats

New endpoint to retrieve the public typing stats such as global count of tests completed.

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Renamed public-stats to public

Except in cases where it would cause an identifier named `public` as this is forbidden in strict mode.

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Add stats section to about page

In this commit:
 - add a section above about called stats
 - display typing stats in three columns
 - underneath show the histogram of speeds on english time 60
 - make chart responsive

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* Add unit test for Public DAL

Signed-off-by: Brian Evans <ebrian101@gmail.com>

* updated styling

* only requesting data once per session

* going one column on narrow screens

* added option to specify number of decimal poitns

* just showing million instead of abbreviating
updated structure
updated styling

Signed-off-by: Brian Evans <ebrian101@gmail.com>
Co-authored-by: Miodec <jack@monkeytype.com>

backend/__tests__/dal/public.spec.ts

@@ -0,0 +1,58 @@
+import * as PublicDAL from "../../src/dal/public";
+import * as db from "../../src/init/db";
+import { ObjectId } from "mongodb";
+
+const mockSpeedHistogram = {
+  _id: new ObjectId(),
+  type: "speedStats",
+  english_time_15: {
+    "70": 2761,
+    "80": 2520,
+    "90": 2391,
+    "100": 2317,
+  },
+  english_time_60: {
+    "50": 8781,
+    "60": 2978,
+    "70": 2786,
+    "80": 2572,
+    "90": 2399,
+  },
+};
+
+describe("PublicDAL", function () {
+  it("should be able to update stats", async function () {
+    // checks it doesn't throw an error. the actual values are checked in another test.
+    await PublicDAL.updateStats(1, 15);
+  });
+
+  it("should be able to get typing stats", async function () {
+    const typingStats = await PublicDAL.getTypingStats();
+    expect(typingStats).toHaveProperty("testsCompleted");
+    expect(typingStats).toHaveProperty("testsStarted");
+    expect(typingStats).toHaveProperty("timeTyping");
+  });
+
+  it("should increment stats on update", async function () {
+    // checks that both functions are working on the same data in mongo
+    const priorStats = await PublicDAL.getTypingStats();
+    await PublicDAL.updateStats(1, 60);
+    const afterStats = await PublicDAL.getTypingStats();
+    expect(afterStats.testsCompleted).toBe(priorStats.testsCompleted + 1);
+    expect(afterStats.testsStarted).toBe(priorStats.testsStarted + 2);
+    expect(afterStats.timeTyping).toBe(priorStats.timeTyping + 60);
+  });
+
+  it("should be able to get speed histogram", async function () {
+    // this test ensures that the property access is correct
+    await db
+      .collection("public")
+      .replaceOne({ type: "speedStats" }, mockSpeedHistogram, { upsert: true });
+    const speedHistogram = await PublicDAL.getSpeedHistogram(
+      "english",
+      "time",
+      "60"
+    );
+    expect(speedHistogram["50"]).toBe(8781); // check a value in the histogram that has been set
+  });
+});

backend/src/api/controllers/public.ts

@@ -0,0 +1,17 @@
+import * as PublicDAL from "../../dal/public";
+import { MonkeyResponse } from "../../utils/monkey-response";
+
+export async function getPublicSpeedHistogram(
+  req: MonkeyTypes.Request
+): Promise<MonkeyResponse> {
+  const { language, mode, mode2 } = req.query;
+  const data = await PublicDAL.getSpeedHistogram(language, mode, mode2);
+  return new MonkeyResponse("Public speed histogram retrieved", data);
+}
+
+export async function getPublicTypingStats(
+  _req: MonkeyTypes.Request
+): Promise<MonkeyResponse> {
+  const data = await PublicDAL.getTypingStats();
+  return new MonkeyResponse("Public typing stats retrieved", data);
+}

backend/src/api/controllers/result.ts

@@ -7,7 +7,7 @@ import {
   updateTypingStats,
   recordAutoBanEvent,
 } from "../../dal/user";
-import * as PublicStatsDAL from "../../dal/public-stats";
+import * as PublicDAL from "../../dal/public";
 import {
   getCurrentDayTimestamp,
   getStartOfDayTimestamp,
@@ -351,7 +351,7 @@ export async function addResult(
   }
   tt = result.testDuration + result.incompleteTestSeconds - afk;
   updateTypingStats(uid, result.restartCount, tt);
-  PublicStatsDAL.updateStats(result.restartCount, tt);
+  PublicDAL.updateStats(result.restartCount, tt);
 
   const dailyLeaderboardsConfig = req.ctx.configuration.dailyLeaderboards;
   const dailyLeaderboard = getDailyLeaderboard(

backend/src/api/routes/index.ts

@@ -1,5 +1,6 @@
 import _ from "lodash";
 import psas from "./psas";
+import publicStats from "./public";
 import users from "./users";
 import { join } from "path";
 import quotes from "./quotes";
@@ -32,6 +33,7 @@ const API_ROUTE_MAP = {
   "/results": results,
   "/presets": presets,
   "/psas": psas,
+  "/public": publicStats,
   "/leaderboards": leaderboards,
   "/quotes": quotes,
   "/ape-keys": apeKeys,

backend/src/api/routes/public.ts

@@ -0,0 +1,30 @@
+import { Router } from "express";
+import * as PublicController from "../controllers/public";
+import * as RateLimit from "../../middlewares/rate-limit";
+import { asyncHandler, validateRequest } from "../../middlewares/api-utils";
+import joi from "joi";
+
+const GET_MODE_STATS_VALIDATION_SCHEMA = {
+  language: joi.string().required(),
+  mode: joi.string().required(),
+  mode2: joi.string().required(),
+};
+
+const router = Router();
+
+router.get(
+  "/speedHistogram",
+  RateLimit.publicStatsGet,
+  validateRequest({
+    query: GET_MODE_STATS_VALIDATION_SCHEMA,
+  }),
+  asyncHandler(PublicController.getPublicSpeedHistogram)
+);
+
+router.get(
+  "/typingStats",
+  RateLimit.publicStatsGet,
+  asyncHandler(PublicController.getPublicTypingStats)
+);
+
+export default router;

backend/src/dal/leaderboards.ts

@@ -153,14 +153,31 @@ export async function update(
   leaderboardUpdating[`${language}_${mode}_${mode2}`] = false;
   const end4 = performance.now();
 
+  const start5 = performance.now();
+  const buckets = {}; // { "70": count, "80": count }
+  for (const lbEntry of lb) {
+    const bucket = Math.floor(lbEntry.wpm / 10).toString() + "0";
+    if (bucket in buckets) buckets[bucket]++;
+    else buckets[bucket] = 1;
+  }
+  await db
+    .collection("public")
+    .updateOne(
+      { type: "speedStats" },
+      { $set: { [`${language}_${mode}_${mode2}`]: buckets } },
+      { upsert: true }
+    );
+  const end5 = performance.now();
+
   const timeToRunAggregate = (end1 - start1) / 1000;
   const timeToRunLoop = (end2 - start2) / 1000;
   const timeToRunInsert = (end3 - start3) / 1000;
   const timeToRunIndex = (end4 - start4) / 1000;
+  const timeToSaveHistogram = (end5 - start5) / 1000; // not sent to prometheus yet
 
   Logger.logToDb(
     `system_lb_update_${language}_${mode}_${mode2}`,
-    `Aggregate ${timeToRunAggregate}s, loop ${timeToRunLoop}s, insert ${timeToRunInsert}s, index ${timeToRunIndex}s`,
+    `Aggregate ${timeToRunAggregate}s, loop ${timeToRunLoop}s, insert ${timeToRunInsert}s, index ${timeToRunIndex}s, histogram ${timeToSaveHistogram}`,
     uid
   );
 

backend/src/dal/public-stats.ts

@@ -1,20 +0,0 @@
-import * as db from "../init/db";
-import { roundTo2 } from "../utils/misc";
-
-export async function updateStats(
-  restartCount: number,
-  time: number
-): Promise<boolean> {
-  await db.collection<MonkeyTypes.PublicStats>("public").updateOne(
-    { type: "stats" },
-    {
-      $inc: {
-        testsCompleted: 1,
-        testsStarted: restartCount + 1,
-        timeTyping: roundTo2(time),
-      },
-    },
-    { upsert: true }
-  );
-  return true;
-}

backend/src/dal/public.ts

@@ -0,0 +1,51 @@
+import * as db from "../init/db";
+import { roundTo2 } from "../utils/misc";
+import MonkeyError from "../utils/error";
+
+export async function updateStats(
+  restartCount: number,
+  time: number
+): Promise<boolean> {
+  await db.collection<MonkeyTypes.PublicStats>("public").updateOne(
+    { type: "stats" },
+    {
+      $inc: {
+        testsCompleted: 1,
+        testsStarted: restartCount + 1,
+        timeTyping: roundTo2(time),
+      },
+    },
+    { upsert: true }
+  );
+  return true;
+}
+
+/** Get the histogram stats of speed buckets for all users.
+ * @returns an object mapping wpm => count, eg { '80': 4388, '90': 2149}
+ */
+export async function getSpeedHistogram(
+  language,
+  mode,
+  mode2
+): Promise<Record<string, number>> {
+  const key = `${language}_${mode}_${mode2}`;
+  const stats = await db
+    .collection<MonkeyTypes.PublicSpeedStats>("public")
+    .findOne({ type: "speedStats" }, { projection: { [key]: 1 } });
+  return stats?.[key] ?? {};
+}
+
+/** Get typing stats such as total number of tests completed on site */
+export async function getTypingStats(): Promise<MonkeyTypes.PublicStats> {
+  const stats = await db
+    .collection<MonkeyTypes.PublicStats>("public")
+    .findOne({ type: "stats" }, { projection: { _id: 0 } });
+  if (!stats) {
+    throw new MonkeyError(
+      404,
+      "Public typing stats not found",
+      "get typing stats"
+    );
+  }
+  return stats;
+}

backend/src/middlewares/rate-limit.ts

@@ -225,6 +225,14 @@ export const psaGet = rateLimit({
   handler: customHandler,
 });
 
+// get public speed stats
+export const publicStatsGet = rateLimit({
+  windowMs: 60 * 1000,
+  max: 60 * REQUEST_MULTIPLIER,
+  keyGenerator: getKeyWithUid,
+  handler: customHandler,
+});
+
 // Results Routing
 export const resultsGet = rateLimit({
   windowMs: ONE_HOUR_MS,

backend/src/types/types.d.ts

@@ -447,6 +447,12 @@ declare namespace MonkeyTypes {
     type: string;
   }
 
+  interface PublicSpeedStats {
+    _id: string;
+    type: "speedStats";
+    [language_mode_mode2: string]: Record<string, number>;
+  }
+
   interface QuoteRating {
     _id: string;
     average: number;

frontend/src/styles/about.scss

@@ -62,6 +62,44 @@
       margin: 0;
       padding: 0;
       color: var(--text-color);
+      &.small {
+        font-size: 0.75em;
+        color: var(--sub-color);
+        text-align: right;
+      }
     }
   }
+
+  .triplegroup {
+    display: grid;
+    grid-template-columns: 1fr 1fr 1fr;
+    gap: 1rem;
+    justify-items: center;
+    margin-top: 1rem;
+  }
+  .group {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    text-align: center;
+    .label {
+      color: var(--sub-color);
+    }
+    .val {
+      font-size: 3rem;
+      line-height: 3.5rem;
+    }
+    .valSmall {
+      font-size: 1.5rem;
+      line-height: 1.5rem;
+    }
+  }
+
+  .chart canvas {
+    width: 100% !important;
+  }
+  .chart {
+    margin-top: 1rem;
+    position: relative;
+  }
 }

frontend/src/styles/z_media-queries.scss

@@ -413,14 +413,16 @@
     grid-template-columns: 1fr;
   }
 
-  .pageAbout .section {
-    .contributors,
-    .supporters {
-      grid-template-columns: 1fr 1fr 1fr;
-    }
-    .contactButtons,
-    .supportButtons {
-      grid-template-columns: 1fr 1fr;
+  .pageAbout {
+    .section {
+      .contributors,
+      .supporters {
+        grid-template-columns: 1fr 1fr 1fr;
+      }
+      .contactButtons,
+      .supportButtons {
+        grid-template-columns: 1fr 1fr;
+      }
     }
   }
 
@@ -452,6 +454,20 @@
       }
     }
   }
+  .pageAbout {
+    .triplegroup {
+      grid-template-columns: 1fr;
+      .group {
+        display: grid;
+        grid-template-columns: 1fr 1fr;
+        align-items: center;
+        gap: 0rem 1rem;
+        .label {
+          grid-column: span 2;
+        }
+      }
+    }
+  }
 }
 
 @media only screen and (max-width: 700px) {

frontend/src/ts/ape/endpoints/index.ts

@@ -6,12 +6,14 @@ import Quotes from "./quotes";
 import Results from "./results";
 import Users from "./users";
 import ApeKeys from "./ape-keys";
+import Public from "./public";
 
 export default {
   Configs,
   Leaderboards,
   Presets,
   Psas,
+  Public,
   Quotes,
   Results,
   Users,

frontend/src/ts/ape/endpoints/public.ts

@@ -0,0 +1,23 @@
+const BASE_PATH = "/public";
+
+interface SpeedStatsQuery {
+  language: string;
+  mode: string;
+  mode2: string;
+}
+
+export default class Public {
+  constructor(private httpClient: Ape.HttpClient) {
+    this.httpClient = httpClient;
+  }
+
+  async getSpeedHistogram(searchQuery: SpeedStatsQuery): Ape.EndpointData {
+    return await this.httpClient.get(`${BASE_PATH}/speedHistogram`, {
+      searchQuery,
+    });
+  }
+
+  async getTypingStats(): Ape.EndpointData {
+    return await this.httpClient.get(`${BASE_PATH}/typingStats`);
+  }
+}

frontend/src/ts/ape/index.ts

@@ -20,6 +20,7 @@ const Ape = {
   quotes: new endpoints.Quotes(httpClient),
   leaderboards: new endpoints.Leaderboards(httpClient),
   presets: new endpoints.Presets(httpClient),
+  publicStats: new endpoints.Public(httpClient),
   apeKeys: new endpoints.ApeKeys(httpClient),
 };
 

frontend/src/ts/controllers/chart-controller.ts

@@ -640,6 +640,69 @@ export const accountHistogram: ChartWithUpdateColors<
   },
 });
 
+export const globalSpeedHistogram: ChartWithUpdateColors<
+  "bar",
+  MonkeyTypes.ActivityChartDataPoint[],
+  string
+> = new ChartWithUpdateColors($(".pageAbout #publicStatsHistogramChart"), {
+  type: "bar",
+  data: {
+    labels: [],
+    datasets: [
+      {
+        yAxisID: "count",
+        label: "Users",
+        data: [],
+      },
+    ],
+  },
+  options: {
+    responsive: true,
+    maintainAspectRatio: false,
+    hover: {
+      mode: "nearest",
+      intersect: false,
+    },
+    scales: {
+      x: {
+        axis: "x",
+        bounds: "ticks",
+        display: true,
+        title: {
+          display: false,
+          text: "Bucket",
+        },
+        offset: true,
+      },
+      count: {
+        axis: "y",
+        beginAtZero: true,
+        min: 0,
+        ticks: {
+          autoSkip: true,
+          autoSkipPadding: 20,
+          stepSize: 10,
+        },
+        display: true,
+        title: {
+          display: true,
+          text: "Users",
+        },
+      },
+    },
+    plugins: {
+      annotation: {
+        annotations: [],
+      },
+      tooltip: {
+        animation: { duration: 250 },
+        intersect: false,
+        mode: "index",
+      },
+    },
+  },
+});
+
 export const miniResult: ChartWithUpdateColors<
   "line" | "scatter",
   number[],
@@ -910,6 +973,7 @@ export function setDefaultFontFamily(font: string): void {
 export function updateAllChartColors(): void {
   ThemeColors.update();
   accountHistory.updateColors();
+  globalSpeedHistogram.updateColors();
   result.updateColors();
   accountActivity.updateColors();
   miniResult.updateColors();

frontend/src/ts/pages/about.ts

@@ -1,14 +1,107 @@
 import * as Misc from "../utils/misc";
 import Page from "./page";
+import Ape from "../ape";
+import * as Notifications from "../elements/notifications";
+import * as ChartController from "../controllers/chart-controller";
+import * as ConnectionState from "../states/connection";
+import intervalToDuration from "date-fns/intervalToDuration";
 
 function reset(): void {
   $(".pageAbout .contributors").empty();
   $(".pageAbout .supporters").empty();
+  ChartController.globalSpeedHistogram.data.datasets[0].data = [];
+  ChartController.globalSpeedHistogram.updateColors();
+}
+
+let speedStatsResponseData: any | undefined;
+let typingStatsResponseData: any | undefined;
+
+function updateStatsAndHistogram(): void {
+  if (!speedStatsResponseData && !typingStatsResponseData) {
+    return;
+  }
+  ChartController.globalSpeedHistogram.updateColors();
+  const bucketedSpeedStats = getHistogramDataBucketed(speedStatsResponseData);
+  ChartController.globalSpeedHistogram.data.labels = bucketedSpeedStats.labels;
+  ChartController.globalSpeedHistogram.data.datasets[0].data =
+    bucketedSpeedStats.data;
+
+  const secondsRounded = Math.round(typingStatsResponseData.timeTyping);
+
+  const timeTypingDuration = intervalToDuration({
+    start: 0,
+    end: secondsRounded * 1000,
+  });
+
+  $(".pageAbout #totalTimeTypingStat .val").text(
+    timeTypingDuration.years?.toString() ?? ""
+  );
+  $(".pageAbout #totalTimeTypingStat .valSmall").text("years");
+  $(".pageAbout #totalTimeTypingStat").attr(
+    "aria-label",
+    Math.round(secondsRounded / 3600) + " hours"
+  );
+
+  $(".pageAbout #totalStartedTestsStat .val").text(
+    Math.round(typingStatsResponseData.testsStarted / 1000000)
+  );
+  $(".pageAbout #totalStartedTestsStat .valSmall").text("million");
+  $(".pageAbout #totalStartedTestsStat").attr(
+    "aria-label",
+    typingStatsResponseData.testsStarted + " tests"
+  );
+
+  $(".pageAbout #totalCompletedTestsStat .val").text(
+    Math.round(typingStatsResponseData.testsCompleted / 1000000)
+  );
+  $(".pageAbout #totalCompletedTestsStat .valSmall").text("million");
+  $(".pageAbout #totalCompletedTestsStat").attr(
+    "aria-label",
+    typingStatsResponseData.testsCompleted + " tests"
+  );
+}
+
+async function getStatsAndHistogramData(): Promise<void> {
+  if (speedStatsResponseData && typingStatsResponseData) {
+    return;
+  }
+
+  if (!ConnectionState.get()) {
+    Notifications.add("Cannot update all time stats - offline", 0);
+    return;
+  }
+
+  const speedStats = await Ape.publicStats.getSpeedHistogram({
+    language: "english",
+    mode: "time",
+    mode2: "60",
+  });
+  if (speedStats.status >= 200 && speedStats.status < 300) {
+    speedStatsResponseData = speedStats.data;
+  } else {
+    Notifications.add(
+      `Failed to get global speed stats for histogram: ${speedStats.message}`,
+      -1
+    );
+  }
+  const typingStats = await Ape.publicStats.getTypingStats();
+  if (typingStats.status >= 200 && typingStats.status < 300) {
+    typingStatsResponseData = typingStats.data;
+  } else {
+    Notifications.add(
+      `Failed to get global typing stats: ${speedStats.message}`,
+      -1
+    );
+  }
 }
 
 async function fill(): Promise<void> {
   const supporters = await Misc.getSupportersList();
   const contributors = await Misc.getContributorsList();
+
+  await getStatsAndHistogramData();
+  updateStatsAndHistogram();
+
   supporters.forEach((supporter) => {
     $(".pageAbout .supporters").append(`
       <div>${supporter}</div>
@@ -38,3 +131,31 @@ export const page = new Page(
     //
   }
 );
+
+/** Convert histogram data to the format required to draw a bar chart. */
+function getHistogramDataBucketed(data: Record<string, number>): {
+  data: { x: number; y: number }[];
+  labels: string[];
+} {
+  const histogramChartDataBucketed: { x: number; y: number }[] = [];
+  const labels: string[] = [];
+
+  const keys = Object.keys(data).sort(
+    (a, b) => parseInt(a, 10) - parseInt(b, 10)
+  );
+  for (let i = 0; i < keys.length; i++) {
+    const bucket = parseInt(keys[i], 10);
+    histogramChartDataBucketed.push({
+      x: bucket,
+      y: data[bucket],
+    });
+    labels.push(`${bucket} - ${bucket + 9}`);
+    if (bucket + 10 != parseInt(keys[i + 1], 10)) {
+      for (let j = bucket + 10; j < parseInt(keys[i + 1], 10); j += 10) {
+        histogramChartDataBucketed.push({ x: j, y: 0 });
+        labels.push(`${j} - ${j + 9}`);
+      }
+    }
+  }
+  return { data: histogramChartDataBucketed, labels };
+}

frontend/src/ts/utils/misc.ts

@@ -1230,14 +1230,14 @@ export async function promiseAnimation(
 }
 
 //abbreviateNumber
-export function abbreviateNumber(num: number): string {
+export function abbreviateNumber(num: number, decimalPoints = 1): string {
   if (num < 1000) {
     return num.toString();
   }
 
   const exp = Math.floor(Math.log(num) / Math.log(1000));
   const pre = "kmbtqQsSond".charAt(exp - 1);
-  return (num / Math.pow(1000, exp)).toFixed(1) + pre;
+  return (num / Math.pow(1000, exp)).toFixed(decimalPoints) + pre;
 }
 
 export async function sleep(ms: number): Promise<void> {

frontend/static/html/pages/about.html

@@ -12,6 +12,44 @@
     <br />
     Launched on 15th of May, 2020.
   </div>
+  <div class="section histogramChart">
+    <div class="triplegroup">
+      <div
+        class="group"
+        id="totalStartedTestsStat"
+        aria-label=""
+        data-balloon-pos="up"
+      >
+        <div class="label">total started tests</div>
+        <div class="val">-</div>
+        <div class="valSmall">-</div>
+      </div>
+      <div
+        class="group"
+        id="totalTimeTypingStat"
+        aria-label=""
+        data-balloon-pos="up"
+      >
+        <div class="label">total time typing</div>
+        <div class="val">-</div>
+        <div class="valSmall">-</div>
+      </div>
+      <div
+        class="group"
+        id="totalCompletedTestsStat"
+        aria-label=""
+        data-balloon-pos="up"
+      >
+        <div class="label">total completed tests</div>
+        <div class="val">-</div>
+        <div class="valSmall">-</div>
+      </div>
+    </div>
+    <div class="chart" style="height: 200px">
+      <canvas id="publicStatsHistogramChart"></canvas>
+    </div>
+    <p class="small">distribution of time 60 leaderbord results</p>
+  </div>
   <div class="section">
     <div class="title">about</div>
     <!-- <h1>about</h1> -->