[local-sync] Check for missing Category roles by localized display names

Sometimes imap mailboxes aren't properly flagged with their roles, so we
check the display names against known variations to see if we can find any
missing roles.

packages/local-sync/nylas-sync/imap/fetch-folder-list.js

@@ -1,6 +1,8 @@
 const {Provider, PromiseUtils} = require('nylas-core');
+const {localizedCategoryNames} = require('../sync-utils')
 
-const GMAIL_ROLES_WITH_FOLDERS = ['all', 'trash', 'junk'];
+const BASE_ROLES = ['inbox', 'sent', 'trash', 'spam'];
+const GMAIL_ROLES_WITH_FOLDERS = ['all', 'trash', 'spam'];
 
 class FetchFolderList {
   constructor(provider, logger) {
@@ -15,6 +17,12 @@ class FetchFolderList {
     return `FetchFolderList`;
   }
 
+  _getMissingRoles(categories) {
+    const currentRoles = new Set(categories.map(cat => cat.role));
+    const missingRoles = BASE_ROLES.filter(role => !currentRoles.has(role));
+    return missingRoles;
+  }
+
   _classForMailboxWithRole(role, {Folder, Label}) {
     if (this._provider === Provider.Gmail) {
       return GMAIL_ROLES_WITH_FOLDERS.includes(role) ? Folder : Label;
@@ -22,24 +30,32 @@ class FetchFolderList {
     return Folder;
   }
 
-  _roleForMailbox(boxName, box) {
+  _roleByName(boxName) {
+    for (const role of Object.keys(localizedCategoryNames)) {
+      if (localizedCategoryNames[role].includes(boxName.toLowerCase().trim())) {
+        return role;
+      }
+    }
+    return null;
+  }
+
+  _roleByAttr(box) {
     for (const attrib of (box.attribs || [])) {
       const role = {
         '\\Sent': 'sent',
         '\\Drafts': 'drafts',
-        '\\Junk': 'junk',
+        '\\Junk': 'spam',
+        '\\Spam': 'spam',
         '\\Trash': 'trash',
         '\\All': 'all',
         '\\Important': 'important',
         '\\Flagged': 'flagged',
+        '\\Inbox': 'inbox',
       }[attrib];
       if (role) {
         return role;
       }
     }
-    if (boxName.toLowerCase().trim() === 'inbox') {
-      return 'inbox';
-    }
     return null;
   }
 
@@ -74,7 +90,7 @@ class FetchFolderList {
 
       let category = categories.find((cat) => cat.name === boxName);
       if (!category) {
-        const role = this._roleForMailbox(boxName, box);
+        const role = this._roleByAttr(box);
         const Klass = this._classForMailboxWithRole(role, this._db);
         category = Klass.build({
           name: boxName,
@@ -104,7 +120,23 @@ class FetchFolderList {
           labels: Label.findAll({transaction}),
         }).then(({folders, labels}) => {
           const all = [].concat(folders, labels);
-          const {created, deleted} = this._updateCategoriesWithBoxes(all, boxes);
+          const {next, created, deleted} = this._updateCategoriesWithBoxes(all, boxes);
+
+          const categoriesByRoles = next.reduce((obj, cat) => {
+            const role = this._roleByName(cat.name);
+            if (role in obj) {
+              obj[role].push(cat);
+            } else {
+              obj[role] = [cat];
+            }
+            return obj;
+          }, {})
+
+          this._getMissingRoles(next).forEach((role) => {
+            if (categoriesByRoles[role] && categoriesByRoles[role].length === 1) {
+              categoriesByRoles[role][0].role = role;
+            }
+          })
 
           let promises = [Promise.resolve()]
           promises = promises.concat(created.map(cat => cat.save({transaction})))

packages/local-sync/nylas-sync/sync-utils.js

@@ -8,4 +8,34 @@ function jsonError(error) {
 
 module.exports = {
   jsonError,
+
+  // This folder list was generated by aggregating examples of user folders
+  // that were not properly labeled as trash, sent, or spam.
+  // This list was constructed semi automatically, and manuallly verified.
+  // Should we encounter problems with account folders in the future, add them
+  // below to test for them.
+  //
+  // Make sure these are lower case! (for comparison purposes)
+  localizedCategoryNames: {
+    trash: ['gel\xc3\xb6scht', 'papierkorb',
+              '\xd0\x9a\xd0\xbe\xd1\x80\xd0\xb7\xd0\xb8\xd0\xbd\xd0\xb0',
+              '[imap]/trash', 'papelera', 'borradores',
+              '[imap]/\xd0\x9a\xd0\xbe\xd1\x80',
+              '\xd0\xb7\xd0\xb8\xd0\xbd\xd0\xb0', 'deleted items',
+              '\xd0\xa1\xd0\xbc\xd1\x96\xd1\x82\xd1\x82\xd1\x8f',
+              'papierkorb/trash', 'gel\xc3\xb6schte elemente',
+              'deleted messages', '[gmail]/trash', 'inbox/trash', 'trash',
+              'mail/trash', 'inbox.trash'],
+    spam: ['roskaposti', 'inbox.spam', 'inbox.spam', 'skr\xc3\xa4ppost',
+             'spamverdacht', 'spam', 'spam', '[gmail]/spam', '[imap]/spam',
+             '\xe5\x9e\x83\xe5\x9c\xbe\xe9\x82\xae\xe4\xbb\xb6', 'junk',
+             'junk mail', 'junk e-mail'],
+    inbox: ['inbox'],
+    sent: ['postausgang', 'inbox.gesendet', '[gmail]/sent mail',
+             '\xeb\xb3\xb4\xeb\x82\xbc\xed\x8e\xb8\xec\xa7\x80\xed\x95\xa8',
+             'elementos enviados', 'sent', 'sent items', 'sent messages',
+             'inbox.papierkorb', 'odeslan\xc3\xa9', 'mail/sent-mail',
+             'ko\xc5\xa1', 'outbox', 'outbox', 'inbox.sentmail', 'gesendet',
+             'ko\xc5\xa1/sent items', 'gesendete elemente'],
+  },
 }

packages/local-sync/spec/sync/fetch-folder-list-spec.js

@@ -0,0 +1,115 @@
+const {PromiseUtils} = require('nylas-core');
+const mockDatabase = require('./mock-database');
+const FetchFolderList = require('../../nylas-sync/imap/fetch-folder-list')
+
+const testCategoryRoles = (db, mailboxes) => {
+  const mockLogger = {
+    info: () => {},
+    debug: () => {},
+    error: () => {},
+  }
+  const mockImap = {
+    getBoxes: () => {
+      return Promise.resolve(mailboxes)
+    },
+  }
+  return new FetchFolderList('fakeProvider', mockLogger).run(db, mockImap).then(() => {
+    const {Folder, Label} = db;
+    return PromiseUtils.props({
+      folders: Folder.findAll(),
+      labels: Label.findAll(),
+    }).then(({folders, labels}) => {
+      const all = [].concat(folders, labels);
+      for (const category of all) {
+        expect(category.role).toEqual(mailboxes[category.name].role);
+      }
+    })
+  })
+};
+
+describe("FetchFolderList", () => {
+  beforeEach((done) => {
+    mockDatabase().then((db) => {
+      this.db = db;
+      done();
+    })
+  })
+
+  it("assigns roles when given a role attribute/flag", (done) => {
+    const mailboxes = {
+      'Sent': {attribs: ['\\Sent'], role: 'sent'},
+      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
+      'Spam': {attribs: ['\\Spam'], role: 'spam'},
+      'Trash': {attribs: ['\\Trash'], role: 'trash'},
+      'All Mail': {attribs: ['\\All'], role: 'all'},
+      'Important': {attribs: ['\\Important'], role: 'important'},
+      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
+      'Inbox': {attribs: ['\\Inbox'], role: 'inbox'},
+      'TestFolder': {attribs: [], role: null},
+      'Receipts': {attribs: [], role: null},
+    }
+
+    testCategoryRoles(this.db, mailboxes).then(done, done.fail);
+  })
+
+  it("assigns missing roles by localized display names", (done) => {
+    const mailboxes = {
+      'Sent': {attribs: [], role: 'sent'},
+      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
+      'Spam': {attribs: ['\\Spam'], role: 'spam'},
+      'Trash': {attribs: ['\\Trash'], role: 'trash'},
+      'All Mail': {attribs: ['\\All'], role: 'all'},
+      'Important': {attribs: ['\\Important'], role: 'important'},
+      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
+      'Inbox': {attribs: [], role: 'inbox'},
+    }
+
+    testCategoryRoles(this.db, mailboxes).then(done, done.fail);
+  })
+
+  it("doesn't assign a role more than once", (done) => {
+    const mailboxes = {
+      'Sent': {attribs: [], role: null},
+      'Sent Items': {attribs: [], role: null},
+      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
+      'Spam': {attribs: ['\\Spam'], role: 'spam'},
+      'Trash': {attribs: ['\\Trash'], role: 'trash'},
+      'All Mail': {attribs: ['\\All'], role: 'all'},
+      'Important': {attribs: ['\\Important'], role: 'important'},
+      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
+      'Mail': {attribs: ['\\Inbox'], role: 'inbox'},
+      'inbox': {attribs: [], role: null},
+    }
+
+    testCategoryRoles(this.db, mailboxes).then(done, done.fail);
+  })
+
+  it("updates role assignments if an assigned category is deleted", (done) => {
+    let mailboxes = {
+      'Sent': {attribs: [], role: null},
+      'Sent Items': {attribs: [], role: null},
+      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
+      'Spam': {attribs: ['\\Spam'], role: 'spam'},
+      'Trash': {attribs: ['\\Trash'], role: 'trash'},
+      'All Mail': {attribs: ['\\All'], role: 'all'},
+      'Important': {attribs: ['\\Important'], role: 'important'},
+      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
+      'Mail': {attribs: ['\\Inbox'], role: 'inbox'},
+    }
+
+    testCategoryRoles(this.db, mailboxes).then(() => {
+      mailboxes = {
+        'Sent Items': {attribs: [], role: 'sent'},
+        'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
+        'Spam': {attribs: ['\\Spam'], role: 'spam'},
+        'Trash': {attribs: ['\\Trash'], role: 'trash'},
+        'All Mail': {attribs: ['\\All'], role: 'all'},
+        'Important': {attribs: ['\\Important'], role: 'important'},
+        'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
+        'Mail': {attribs: ['\\Inbox'], role: 'inbox'},
+      }
+
+      return testCategoryRoles(this.db, mailboxes).then(done, done.fail);
+    }, done.fail);
+  })
+});

packages/local-sync/spec/sync/mock-database.js

@@ -0,0 +1,55 @@
+const {DatabaseConnector} = require('nylas-core');
+
+/*
+ * Mocks out various Model and Instance methods to prevent actually saving data
+ * to the sequelize database. Note that with the current implementation, only
+ * instances created with Model.build() are mocked out.
+ *
+ * Currently mocks out the following:
+ *   Model
+ *     .build()
+ *     .findAll()
+ *   Instance
+ *     .destroy()
+ *     .save()
+ *
+ */
+function mockDatabase() {
+  return DatabaseConnector.forAccount(-1).then((db) => {
+    const data = {};
+    for (const modelName of Object.keys(db.sequelize.models)) {
+      const model = db.sequelize.models[modelName];
+      data[modelName] = {};
+
+      spyOn(model, 'findAll').and.callFake(() => {
+        return Promise.resolve(
+          Object.keys(data[modelName]).map(key => data[modelName][key])
+        );
+      });
+
+      const origBuild = model.build;
+      spyOn(model, 'build').and.callFake((...args) => {
+        const instance = origBuild.apply(model, args);
+
+        spyOn(instance, 'save').and.callFake(() => {
+          if (instance.id == null) {
+            const sortedIds = Object.keys(data[modelName]).sort();
+            const len = sortedIds.length;
+            instance.id = len ? +sortedIds[len - 1] + 1 : 0;
+          }
+          data[modelName][instance.id] = instance;
+        });
+
+        spyOn(instance, 'destroy').and.callFake(() => {
+          delete data[modelName][instance.id]
+        });
+
+        return instance;
+      })
+    }
+
+    return Promise.resolve(db);
+  });
+}
+
+module.exports = mockDatabase;