From ff0ee93fe4faf08485ec0207820cd88a8a81eab5 Mon Sep 17 00:00:00 2001
From: Bruce Berrios <58147810+Bruception@users.noreply.github.com>
Date: Mon, 7 Mar 2022 09:32:59 -0500
Subject: [PATCH] Fix tokenizer (#2652)

---
 frontend/src/scripts/popups/quote-search-popup.ts | 2 +-
 frontend/src/scripts/utils/search-service.ts      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/frontend/src/scripts/popups/quote-search-popup.ts b/frontend/src/scripts/popups/quote-search-popup.ts
index ed98c73ae..cbdeb7a6f 100644
--- a/frontend/src/scripts/popups/quote-search-popup.ts
+++ b/frontend/src/scripts/popups/quote-search-popup.ts
@@ -42,7 +42,7 @@ function highlightMatches(text: string, matchedText: string[]): string {
     return text;
   }
   const words = text.split(
-    /(?=[.,'"/#!$%^&*;:{}=\-_`~()\s])|(?<=[.,'"/#!$%^&*;:{}=\-_`~()\s])/g
+    /(?=[.,"/#!$%^&*;:{}=\-_`~()\s])|(?<=[.,"/#!$%^&*;:{}=\-_`~()\s])/g
   );
 
   const normalizedWords = words.map((word) => {
diff --git a/frontend/src/scripts/utils/search-service.ts b/frontend/src/scripts/utils/search-service.ts
index 83218396f..8ab0d9440 100644
--- a/frontend/src/scripts/utils/search-service.ts
+++ b/frontend/src/scripts/utils/search-service.ts
@@ -48,7 +48,7 @@ function inverseDocumentFrequency(
 }
 
 function tokenize(text: string): string[] {
-  return text.match(/[a-zA-Z0-9]+/g) || [];
+  return text.match(/[^\\\][.,"/#!?$%^&*;:{}=\-_`~()\s]+/g) || [];
 }
 
 export const buildSearchService = <T>(