fix(search): Prevent from adding duplicate threads to the search index

- We have to manually check if we are inserting a thread to the index
that already exists because the virtual table does not support unique
indexes
- Add versioning to the index to be able to rebuild it for the next
update

internal_packages/thread-search-index/lib/search-index-store.es6

@@ -13,7 +13,7 @@ const MAX_INDEX_SIZE = 30000
 const CHUNKS_PER_ACCOUNT = 10
 const INDEXING_WAIT = 1000
 const MESSAGE_BODY_LENGTH = 50000
-
+const INDEX_VERSION = 1
 
 class SearchIndexStore {
 
@@ -28,6 +28,10 @@ class SearchIndexStore {
 
       this.accountIds = _.pluck(AccountStore.accounts(), 'id')
       this.initializeIndex()
+      .then(() => {
+        NylasEnv.config.set('threadSearchIndexVersion', INDEX_VERSION)
+        return Promise.resolve()
+      })
       .then(() => {
         console.log(`Thread Search: Index built successfully in ${((Date.now() - date) / 1000)}s`)
         this.unsubscribers = [
@@ -48,6 +52,11 @@ class SearchIndexStore {
    * before sync completes
    */
   initializeIndex() {
+    if (NylasEnv.config.get('threadSearchIndexVersion') !== INDEX_VERSION) {
+      return this.clearIndex()
+      .then(() => this.buildIndex(this.accountIds))
+    }
+
     return DatabaseStore.searchIndexSize(Thread)
     .then((size) => {
       console.log(`Thread Search: Current index size is ${(size || 0)} threads`)
@@ -56,12 +65,7 @@ class SearchIndexStore {
       }
       return this.getUnindexedAccounts()
     })
-    .then((accountIds) => {
-      if (accountIds.length > 0) {
-        return this.buildIndex(accountIds)
-      }
-      return Promise.resolve()
-    })
+    .then((accountIds) => this.buildIndex(accountIds))
   }
 
   /**
@@ -137,6 +141,7 @@ class SearchIndexStore {
   }
 
   buildIndex = (accountIds) => {
+    if (!accountIds || accountIds.length === 0) { return Promise.resolve() }
     const sizePerAccount = Math.floor(INDEX_SIZE / accountIds.length)
     return Promise.resolve(accountIds)
     .each((accountId) => (

src/flux/attributes/matcher.coffee

@@ -75,11 +75,11 @@ class Matcher
       when '>=' then return modelValue >= matcherValue
       when 'in' then return modelValue in matcherValue
       when 'contains'
-        !!modelArrayContainsValue(modelValue, matcherValue)
+        modelArrayContainsValue(modelValue, matcherValue)
 
       when 'containsAny'
         _.any matcherValue, (submatcherValue) ->
-          !!modelArrayContainsValue(modelValue, submatcherValue)
+          modelArrayContainsValue(modelValue, submatcherValue)
 
       when 'startsWith' then return modelValue.startsWith(matcherValue)
       when 'like' then modelValue.search(new RegExp(".*#{matcherValue}.*", "gi")) >= 0

src/flux/stores/database-store.coffee

@@ -574,27 +574,39 @@ class DatabaseStore extends NylasStore
     sql = "DROP TABLE IF EXISTS `#{searchTableName}`"
     @_query(sql)
 
-  indexModel: (model, indexData) =>
-    searchTableName = "#{model.constructor.name}Search"
-    indexFields = Object.keys(indexData)
-    keysSql = 'content_id, ' + indexFields.join(", ")
-    valsSql = '?, ' + indexFields.map(=> '?').join(", ")
-    values = [model.id].concat(indexFields.map((k) => indexData[k]))
-    sql = (
-      "INSERT INTO `#{searchTableName}`(#{keysSql}) VALUES (#{valsSql})"
-    )
-    return @_query(sql, values)
-
-  updateModelIndex: (model, indexData) =>
+  isModelIndexed: (model, isIndexed) =>
+    return Promise.resolve(true) if isIndexed is true
     searchTableName = "#{model.constructor.name}Search"
     exists = (
       "SELECT rowid FROM `#{searchTableName}` WHERE `#{searchTableName}`.`content_id` = ?"
     )
-    return @_query(exists, [model.id])
-    .then((results) =>
-      isIndexed = results.length > 0
+    return @_query(exists, [model.id]).then((results) =>
+      return Promise.resolve(results.length > 0)
+    )
+
+  indexModel: (model, indexData, isModelIndexed) =>
+    searchTableName = "#{model.constructor.name}Search"
+    @isModelIndexed(model, isModelIndexed)
+    .then((isIndexed) =>
+      if (isIndexed)
+        return @updateModelIndex(model, indexData, isIndexed)
+
+      indexFields = Object.keys(indexData)
+      keysSql = 'content_id, ' + indexFields.join(", ")
+      valsSql = '?, ' + indexFields.map(=> '?').join(", ")
+      values = [model.id].concat(indexFields.map((k) => indexData[k]))
+      sql = (
+        "INSERT INTO `#{searchTableName}`(#{keysSql}) VALUES (#{valsSql})"
+      )
+      return @_query(sql, values)
+    )
+
+  updateModelIndex: (model, indexData, isModelIndexed) =>
+    searchTableName = "#{model.constructor.name}Search"
+    @isModelIndexed(model, isModelIndexed)
+    .then((isIndexed) =>
       if (not isIndexed)
-        return @indexModel(model, indexData)
+        return @indexModel(model, indexData, isIndexed)
 
       indexFields = Object.keys(indexData)
       values = indexFields.map((key) => indexData[key]).concat([model.id])