[local-sync] Skip unecessary folder syncs

Summary:
Before trying to sync a folder, check if we actually need to do so. This will prevent us from doing unnecessary work that slows down the sync loop (like performing SELECT commands)

We will perform a folder sync if any of the following are true
- The folder hasn't been completely synced
- There are new messages (using imap STATUS command)
- There are attribute changes indicated via highestmodseq (using imap STATUS command)
- If server doesn't support highestmodseq, it has passed enough time since we last ran an attribute scan on the folder.

Addresses T7513

Test Plan: manual

Reviewers: evan, halla, spang

Reviewed By: halla, spang

Differential Revision: https://phab.nylas.com/D3675

packages/isomorphic-core/src/imap-connection.js

@@ -193,6 +193,9 @@ class IMAPConnection extends EventEmitter {
    * @return {Promise} that resolves to instance of IMAPBox
    */
   openBox(folderName, {readOnly = false} = {}) {
+    if (!folderName) {
+      throw new Error('IMAPConnection::openBox - You must provide a folder name')
+    }
     if (!this._imap) {
       throw new IMAPConnectionNotReadyError(`IMAPConnection::openBox`)
     }
@@ -210,6 +213,21 @@ class IMAPConnection extends EventEmitter {
     })
   }
 
+  getBoxStatus(folderName) {
+    if (!folderName) {
+      throw new Error('IMAPConnection::getBoxStatus - You must provide a folder name')
+    }
+    const openBoxName = this.getOpenBoxName()
+    if (openBoxName === folderName) {
+      return this._imap._box
+    }
+    return this.createConnectionPromise((resolve, reject) => {
+      return this._imap.statusAsync(folderName)
+      .then((...args) => resolve(...args))
+      .catch((...args) => reject(...args))
+    })
+  }
+
   getBoxes() {
     if (!this._imap) {
       throw new IMAPConnectionNotReadyError(`IMAPConnection::getBoxes`)

packages/local-sync/src/local-sync-worker/sync-tasks/fetch-messages-in-folder.imap.es6

@@ -8,6 +8,7 @@ const MessageFlagAttributes = ['id', 'threadId', 'folderImapUID', 'unread', 'sta
 const FETCH_ATTRIBUTES_BATCH_SIZE = 1000;
 const FETCH_MESSAGES_FIRST_COUNT = 100;
 const FETCH_MESSAGES_COUNT = 200;
+const ATTRIBUTE_SCAN_INTERVAL_MS = 10 * 60 * 1000;
 
 
 class FetchMessagesInFolderIMAP extends SyncTask {
@@ -259,7 +260,6 @@ class FetchMessagesInFolderIMAP extends SyncTask {
       fetchedmax: fetchedmax ? Math.max(fetchedmax, max) : max,
       uidnext: boxUidnext,
       uidvalidity: boxUidvalidity,
-      timeFetchedUnseen: Date.now(),
     });
   }
 
@@ -450,21 +450,52 @@ class FetchMessagesInFolderIMAP extends SyncTask {
     // this._logger.info(`FetchMessagesInFolder: Deep scan finished.`);
     await this._folder.updateSyncState({
       attributeFetchedMax: (from <= 1 ? recentStart : from),
+      lastAttributeScanTime: Date.now(),
     });
   }
 
+  * _shouldSyncFolder() {
+    if (!this._folder.isSyncComplete()) {
+      return true
+    }
+
+    const boxStatus = yield this._imap.getBoxStatus(this._folder.name)
+    const {syncState} = this._folder
+    const hasNewMessages = boxStatus.uidnext > syncState.fetchedmax
+
+    if (hasNewMessages) {
+      return true
+    }
+
+    if (this._supportsChangesSince()) {
+      const hasAttributeUpdates = syncState.highestmodseq !== boxStatus.highestmodseq
+      if (hasAttributeUpdates) {
+        return true
+      }
+    } else {
+      const {lastAttributeScanTime} = syncState
+      const shouldScanForAttributeChanges = (
+        !lastAttributeScanTime ||
+        Date.now() - lastAttributeScanTime >= ATTRIBUTE_SCAN_INTERVAL_MS
+      )
+      if (shouldScanForAttributeChanges) {
+        return true
+      }
+    }
+
+    return false
+  }
+
   /**
    * Note: This function is an ES6 generator so we can `yield` at points
    * we want to interrupt sync. This is enabled by `SyncOperation` and
    * `Interruptible`
    */
-  * runTask(db, imap) {
+  async * runTask(db, imap) {
     console.log(`🔜 📂 ${this._folder.name}`)
     this._db = db;
     this._imap = imap;
 
-    this._box = yield this._openMailboxAndEnsureValidity();
-
     // If we haven't set any syncState at all, let's set it for the first time
     // to generate a delta for N1
     if (_.isEmpty(this._folder.syncState)) {
@@ -476,6 +507,14 @@ class FetchMessagesInFolderIMAP extends SyncTask {
         failedUIDs: [],
       })
     }
+
+    const shouldSyncFolder = yield this._shouldSyncFolder()
+    if (!shouldSyncFolder) {
+      console.log(`🔚 📂 ${this._folder.name} has no updates - skipping sync`)
+      return;
+    }
+
+    this._box = yield this._openMailboxAndEnsureValidity();
     yield this._fetchUnsyncedMessages();
     yield this._fetchMessageAttributeChanges();
     console.log(`🔚 📂 ${this._folder.name} done`)

packages/local-sync/src/models/folder.js

@@ -29,8 +29,13 @@ module.exports = (sequelize, Sequelize) => {
      *   // we need resync whole folder
      *   uidvalidity,
      *
-     *   // Timestamp when we last fetched unseen messages
-     *   timeFetchedUnseen,
+     *   // Keeps track of the last uid we've scanned for attribtue changes when
+     *   // the server doesn't support CONDSTORE
+     *   attributeFetchedMax
+     *
+     *   // Timestamp when we last scanned attribute changes inside this folder
+     *   // This is only applicable when the server doesn't support CONDSTORE
+     *   lastAttributeScanTime,
      *
      *   // UIDs that failed to be fetched
      *   failedUIDs,