First step to getting persistent id in K2

Summary:
This diff adds persistent unique ids for messages and contacts. For messages, we just take a hash of the headers. For contacts, we hash the contact's email address.

This diff bundles a couple of tiny fixes too, like always trying to restart an account's sync, even after an exception.

Note that since there's no reliable way to have persistent ids for threads, we'll have to change our code to use message ids instead. Alas, that's a story for another (massive) diff.

Test Plan: Tested manually.

Reviewers: bengotow

Reviewed By: bengotow

Differential Revision: https://phab.nylas.com/D3468

packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js

@@ -242,7 +242,7 @@ class FetchMessagesInFolder {
         accountId: this._db.accountId,
         folderId: this._folder.id,
       });
-      const existingMessage = await Message.find({where: {hash: messageValues.hash}});
+      const existingMessage = await Message.find({where: {id: messageValues.id}});
 
       if (existingMessage) {
         await existingMessage.update(messageValues)

packages/local-sync/src/local-sync-worker/sync-worker.js

@@ -146,9 +146,6 @@ class SyncWorker {
     }
 
     console.log(this._account)
-    if (this._account.errored()) {
-      this._logger.error(`SyncWorker: Account is in error state - Retrying sync\n${this._account.syncError.message}`, this._account.syncError.stack.join('\n'))
-    }
     this._logger.info({reason}, `SyncWorker: Account sync started`)
 
     try {

packages/local-sync/src/models/contact.js

@@ -1,5 +1,6 @@
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('contact', {
+    id: {type: Sequelize.STRING(65), primaryKey: true},
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     name: Sequelize.STRING,
@@ -8,13 +9,13 @@ module.exports = (sequelize, Sequelize) => {
     indexes: [
       {
         unique: true,
-        fields: ['email'],
+        fields: ['id'],
       },
     ],
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: `${this.id}`,
+          id: `${this.publicId}`,
           account_id: this.accountId,
           object: 'contact',
           email: this.email,

packages/local-sync/src/models/file.js

@@ -2,6 +2,7 @@ const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('file', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     filename: Sequelize.STRING(500),
@@ -14,6 +15,12 @@ module.exports = (sequelize, Sequelize) => {
         File.belongsTo(Message)
       },
     },
+    indexes: [
+      {
+        unique: true,
+        fields: ['id'],
+      },
+    ],
     instanceMethods: {
       fetch: function fetch({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
@@ -39,7 +46,7 @@ module.exports = (sequelize, Sequelize) => {
       },
       toJSON: function toJSON() {
         return {
-          id: `${this.id}`,
+          id: this.id,
           object: 'file',
           account_id: this.accountId,
           message_id: this.messageId,

packages/local-sync/src/models/message.js

@@ -1,10 +1,11 @@
-const crypto = require('crypto');
+const cryptography = require('crypto');
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 const {DatabaseTypes: {JSONType, JSONARRAYType}} = require('isomorphic-core');
 
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('message', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     headerMessageId: Sequelize.STRING,
@@ -12,7 +13,6 @@ module.exports = (sequelize, Sequelize) => {
     headers: JSONType('headers'),
     subject: Sequelize.STRING(500),
     snippet: Sequelize.STRING(255),
-    hash: Sequelize.STRING(65),
     date: Sequelize.DATE,
     unread: Sequelize.BOOLEAN,
     starred: Sequelize.BOOLEAN,
@@ -28,7 +28,7 @@ module.exports = (sequelize, Sequelize) => {
     indexes: [
       {
         unique: true,
-        fields: ['hash'],
+        fields: ['id'],
       },
     ],
     classMethods: {
@@ -40,7 +40,7 @@ module.exports = (sequelize, Sequelize) => {
       },
 
       hashForHeaders(headers) {
-        return crypto.createHash('sha256').update(headers, 'utf8').digest('hex');
+        return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
       },
     },
     instanceMethods: {
@@ -79,7 +79,7 @@ module.exports = (sequelize, Sequelize) => {
         // Message though and need to protect `this.date` from null
         // errors.
         return {
-          id: `${this.id}`,
+          id: this.id,
           account_id: this.accountId,
           object: 'message',
           body: this.body,

packages/local-sync/src/new-message-processor/extract-contacts.js

@@ -1,3 +1,4 @@
+const cryptography = require('crypto');
 
 function isContactVerified(contact) {
   // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
@@ -21,13 +22,13 @@ function extractContacts({db, message}) {
   })
 
   const verifiedContacts = allContacts.filter(c => isContactVerified(c));
-
   return db.sequelize.transaction((transaction) => {
     return Promise.all(verifiedContacts.map((contact) =>
       Contact.upsert({
         name: contact.name,
         email: contact.email,
         accountId: message.accountId,
+        id: cryptography.createHash('sha256').update(contact.email, 'utf8').digest('hex'),
       }, {
         transaction,
       })

packages/local-sync/src/shared/local-database-connector.js

@@ -32,7 +32,6 @@ class LocalDatabaseConnector {
     const newSequelize = this._sequelizePoolForDatabase(`a-${accountId}`);
     const db = loadModels(Sequelize, newSequelize, {
       modelDirs: [path.resolve(__dirname, '..', 'models')],
-      schema: `a${accountId}`,
     })
 
     HookTransactionLog(db, newSequelize);

packages/local-sync/src/shared/message-factory.js

@@ -38,12 +38,12 @@ function parseFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
   }
 
   const values = {
+    id: Message.hashForHeaders(headers),
     to: extractContacts(parsedHeaders.to),
     cc: extractContacts(parsedHeaders.cc),
     bcc: extractContacts(parsedHeaders.bcc),
     from: extractContacts(parsedHeaders.from),
     replyTo: extractContacts(parsedHeaders['reply-to']),
-    hash: Message.hashForHeaders(headers),
     accountId: accountId,
     body: body['text/html'] || body['text/plain'] || body['application/pgp-encrypted'] || '',
     snippet: body['text/plain'] ? body['text/plain'].substr(0, 255) : null,

test_accounts.txt

@@ -14,3 +14,6 @@ Others:
 curl -k -X POST -H "Content-Type: application/json" -d '{"email":"nylastest@runbox.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"nylastest","imap_host":"mail.runbox.com","imap_port":993,"smtp_host":"mail.runbox.com","smtp_port":0,"smtp_username":"nylastest", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
 curl -k -X POST -H "Content-Type: application/json" -d '{"email":"securemail@defendcyber.space", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"securemail@defendcyber.space","imap_host":"imap.secureserver.net","imap_port":143,"smtp_host":"smtpout.secureserver.net","smtp_port":25,"smtp_username":"securemail@defendcyber.space", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":false}}' "http://localhost:5100/auth?client_id=123"
 curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest4@gmail.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest4@gmail.com","imap_host":"imap.gmail.com","imap_port":993,"smtp_host":"smtp.gmail.com","smtp_port":465,"smtp_username":"inboxapptest4@gmail.com", "smtp_password":"ihategmail","imap_password":"ihategmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
+
+Fastmail:
+curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"imap.fastmail.com","imap_port":993,"smtp_host":"smtp.fastmail.com","smtp_port":465,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"6e7bucyuxffmg2vj","imap_password":"6e7bucyuxffmg2vj","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"