[client-app] More defensive error handling to prevent sync from halting

Summary:
This commit adds error handling to the sync-loop's `onSyncError` and
`scheduleNextSync`.

These functions generally don't fail, as they are in the `catch` and
`finally` blocks respectively of the sync loop. But as we've seen in
D4152, the datbase can sometime error if it's in a bad state. If it
errors inside these functions, we will never schedule the next run of
the sync loop.

Depends on D4152

Test Plan: manual

Reviewers: evan, spang, mark, halla

Reviewed By: halla

Differential Revision: https://phab.nylas.com/D4153

packages/client-sync/src/local-sync-worker/sync-worker.es6

@@ -25,7 +25,6 @@ const {SYNC_STATE_RUNNING, SYNC_STATE_AUTH_FAILED, SYNC_STATE_ERROR} = Account
 const AC_SYNC_LOOP_INTERVAL_MS = 10 * 1000            // 10 sec
 const BATTERY_SYNC_LOOP_INTERVAL_MS = 5 * 60 * 1000   //  5 min
 const MAX_SYNC_BACKOFF_MS = 5 * 60 * 1000 // 5 min
-const PERMANENT_ERROR_RETRY_BACKOFF_MS = 60 * 1000 // 1 min
 
 class SyncWorker {
   constructor(account, db, parentManager) {
@@ -294,6 +293,7 @@ class SyncWorker {
   }
 
   async _onSyncError(error) {
+    try {
       this._clearConnections();
       this._logger.error(`🔃  SyncWorker: Errored while syncing account`, error)
 
@@ -344,6 +344,11 @@ class SyncWorker {
 
       this._account.syncError = errorJSON
       await this._account.save()
+    } catch (err) {
+      this._logger.error(`🔃  SyncWorker: Errored while handling error`, error)
+      err.message = `Error while handling sync loop error: ${err.message}`
+      NylasEnv.reportError(err)
+    }
   }
 
   async _onSyncDidComplete() {
@@ -372,18 +377,20 @@ class SyncWorker {
   }
 
   async _scheduleNextSync(error) {
+    let reason;
+    let interval;
+    try {
       if (this._stopped) { return; }
       const {Folder} = this._db;
 
       const folders = await Folder.findAll();
       const moreToSync = folders.some((f) => !f.isSyncComplete())
 
-    let interval;
       if (error != null) {
         if (error instanceof IMAPErrors.RetryableError) {
           interval = this._retryScheduler.currentDelay();
         } else {
-        interval = PERMANENT_ERROR_RETRY_BACKOFF_MS;
+          interval = AC_SYNC_LOOP_INTERVAL_MS;
         }
       } else {
         const shouldSyncImmediately = (
@@ -400,16 +407,21 @@ class SyncWorker {
         }
       }
 
-
+      reason = 'Normal schedule'
-    let reason = "Scheduled"
       if (error != null) {
         reason = `Sync errored: ${error.message}`
       } else if (this._interrupted) {
         reason = `Sync interrupted and restarted. Interrupt reason: ${reason}`
       } else if (moreToSync) {
-      reason = "More to sync"
+        reason = `More to sync`
       }
-
+    } catch (err) {
+      this._logger.error(`🔃  SyncWorker: Errored while scheduling next sync`, err)
+      err.message = `Error while scheduling next sync: ${err.message}`
+      NylasEnv.reportError(err)
+      interval = AC_SYNC_LOOP_INTERVAL_MS
+      reason = 'Errored while while scheduling next sync'
+    } finally {
       const nextSyncIn = Math.max(1, this._lastSyncTime + interval - Date.now())
       this._logger.log(`🔃 🔜 in ${nextSyncIn}ms - Reason: ${reason}`)
 
@@ -417,6 +429,7 @@ class SyncWorker {
         this.syncNow({reason});
       }, nextSyncIn);
     }
+  }
 
   async _runTask(task) {
     this._currentTask = task