Mailspring/internal_packages/thread-search-index/lib/search-index-store.es6

import _ from 'underscore'
import {
  Utils,
  Thread,
  AccountStore,
  DatabaseStore,
  NylasSyncStatusStore,
  QuotedHTMLTransformer,
} from 'nylas-exports'

const INDEX_SIZE = 10000
const MAX_INDEX_SIZE = 25000
const CHUNKS_PER_ACCOUNT = 10
const INDEXING_WAIT = 1000
const MESSAGE_BODY_LENGTH = 50000


class SearchIndexStore {

  constuctor() {
    this.accountIds = _.pluck(AccountStore.accounts(), 'id')
    this.unsubscribers = []
  }

  activate() {
    NylasSyncStatusStore.whenSyncComplete().then(() => {
      const date = Date.now()
      console.log('ThreadSearch: Initializing thread search index...')
      this.initializeIndex(this.accountIds)
      .then(() => {
        console.log('ThreadSearch: Index built successfully in ' + ((Date.now() - date) / 1000) + 's')
        this.unsubscribers = [
          DatabaseStore.listen(::this.onDataChanged),
          AccountStore.listen(::this.onAccountsChanged),
        ]
      })
    })
  }

  initializeIndex(accountIds) {
    return DatabaseStore.searchIndexSize(Thread)
    .then((size) => {
      console.log('ThreadSearch: Current index size is ' + (size || 0) + ' threads')
      if (!size || size >= MAX_INDEX_SIZE || size === 0) {
        return this.clearIndex().thenReturn(true)
      }
      return Promise.resolve(false)
    })
    .then((shouldRebuild) => {
      if (shouldRebuild) {
        return this.buildIndex(accountIds)
      }
      return Promise.resolve()
    })
  }

  onAccountsChanged() {
    const date = Date.now()
    const newIds = _.pluck(AccountStore.accounts(), 'id')
    if (newIds.length === this.accountIds.length) {
      return;
    }

    this.accountIds = newIds
    this.clearIndex()
    .then(() => this.buildIndex(this.accountIds))
    .then(() => {
      console.log('ThreadSearch: Index rebuilt successfully in ' + ((Date.now() - date) / 1000) + 's')
    })
  }

  onDataChanged(change) {
    if (change.objectClass !== Thread.name) {
      return;
    }
    const {objects, type} = change
    let promises = []
    if (type === 'persist') {
      promises = objects.map(thread => this.updateThreadIndex(thread))
    } else if (type === 'unpersist') {
      promises = objects.map(thread => DatabaseStore.unindexModel(thread))
    }
    Promise.all(promises)
  }

  clearIndex() {
    return (
      DatabaseStore.dropSearchIndex(Thread)
      .then(() => DatabaseStore.createSearchIndex(Thread))
    )
  }

  buildIndex(accountIds) {
    const numAccounts = accountIds.length
    return Promise.resolve(accountIds)
    .each((accountId) => (
      this.indexThreadsForAccount(accountId, Math.floor(INDEX_SIZE / numAccounts))
    ))
  }

  indexThreadsForAccount(accountId, indexSize) {
    const chunkSize = Math.floor(indexSize / CHUNKS_PER_ACCOUNT)
    const chunks = Promise.resolve(_.times(CHUNKS_PER_ACCOUNT, () => chunkSize))

    return chunks.each((size, idx) => {
      return DatabaseStore.findAll(Thread)
      .where({accountId})
      .limit(size)
      .offset(size * idx)
      .order(Thread.attributes.lastMessageReceivedTimestamp.descending())
      .then((threads) => {
        return Promise.all(
          threads.map(thread => this.indexThread(thread))
        ).then(() => {
          return new Promise((resolve) => setTimeout(resolve, INDEXING_WAIT))
        })
      })
    })
  }

  indexThread(thread) {
    return (
      this.getIndexData(thread)
      .then((indexData) => (
        DatabaseStore.indexModel(thread, indexData)
      ))
    )
  }

  updateThreadIndex(thread) {
    return (
      this.getIndexData(thread)
      .then((indexData) => (
        DatabaseStore.updateModelIndex(thread, indexData)
      ))
    )
  }

  getIndexData(thread) {
    const messageBodies = (
      thread.messages()
      .then((messages) => (
        Promise.resolve(
          messages
          .map(({body, snippet}) => (
            !_.isString(body) ?
              {snippet} :
              {body: QuotedHTMLTransformer.removeQuotedHTML(body)}
          ))
          .map(({body, snippet}) => (
            snippet ?
              snippet :
              Utils.extractTextFromHtml(body, {maxLength: MESSAGE_BODY_LENGTH}).replace(/(\s)+/g, ' ')
          ))
          .join(' ')
        )
      ))
    )
    const participants = (
      thread.participants
      .map(({name, email}) => `${name} ${email}`)
      .join(" ")
    )

    return Promise.props({
      participants,
      body: messageBodies,
      subject: thread.subject,
    })
  }

  deactivate() {
    this.unsubscribers.forEach(unsub => unsub())
  }
}

export default new SearchIndexStore();