From c6f371aa0fbe77dad107d1e3ec94894828485f77 Mon Sep 17 00:00:00 2001
From: Christine Spang <spang@nylas.com>
Date: Tue, 31 Jan 2017 16:13:50 -0800
Subject: [PATCH] [local-sync] Serialize category sync progress to edgehill
 rather than syncState

Summary:
syncState on folders may contain arbitrarily long arrays of UIDs
(particularly, failedUIDs). If we serialize this JSON column to
edgehill.db, we can end up serializing very large objects when
persisting the local task queue. When the queue contains many tasks,
this can balloon the JSON blob to megabytes, causing the main window and
the worker window to become unresponsive.

The UI doesn't need to know about IMAP bookkeeping internals, so
serialize the sync progress instead of the sync state. This has the
advantages that (1) we don't need to worry about future keys added
to the syncState being large and (2) when we add Exchange support
we already have an abstraction for sync progress.

Test Plan: manual

Reviewers: juan, mark, halla

Reviewed By: halla

Differential Revision: https://phab.nylas.com/D3817
---
 .../src/models/{folder.js => folder.es6}      | 36 +++++++++++++++----
 1 file changed, 30 insertions(+), 6 deletions(-)
 rename packages/local-sync/src/models/{folder.js => folder.es6} (70%)

diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.es6
similarity index 70%
rename from packages/local-sync/src/models/folder.js
rename to packages/local-sync/src/models/folder.es6
index a5c1dcd5a..36183838b 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.es6
@@ -1,9 +1,11 @@
-const _ = require('underscore')
-const crypto = require('crypto')
-const {DatabaseTypes: {JSONColumn}} = require('isomorphic-core');
-const {formatImapPath} = require('../shared/imap-paths-utils');
+import _ from 'underscore'
+import crypto from 'crypto'
+import {DatabaseTypes} from 'isomorphic-core'
+import {formatImapPath} from '../shared/imap-paths-utils'
 
-module.exports = (sequelize, Sequelize) => {
+const {JSONColumn} = DatabaseTypes
+
+export default (sequelize, Sequelize) => {
   return sequelize.define('folder', {
     id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
@@ -74,6 +76,24 @@ module.exports = (sequelize, Sequelize) => {
         return this.save();
       },
 
+      syncProgress() {
+        if (!this.syncState) {
+          return {
+            approxPercentComplete: 0,
+            approxTotal: 0,
+            oldestProcessedDate: new Date(),
+          }
+        }
+        const {fetchedmax, fetchedmin, uidnext, minUID, oldestProcessedDate} = this.syncState;
+        return {
+          // based on % of uid space scanned, but space may be sparse
+          approxPercentComplete: (+fetchedmax - +fetchedmin + 1) /
+                                 (uidnext - Math.min(minUID, fetchedmin) + 1),
+          approxTotal: uidnext,
+          oldestProcessedDate: oldestProcessedDate,
+        }
+      },
+
       toJSON() {
         return {
           id: `${this.id}`,
@@ -81,7 +101,11 @@ module.exports = (sequelize, Sequelize) => {
           object: 'folder',
           name: this.role,
           display_name: formatImapPath(this.name),
-          sync_state: this.syncState,
+          sync_progress: this.syncProgress(),
+          // intentionally overwrite any sync states stored in edgehill.db,
+          // since it may contain long arrays and cause perf degredation
+          // when serialized repeatedly
+          sync_state: null,
         };
       },
     },