[local-sync] Fix file/contact creation

We were getting sql unique constraint violation errors for ids because
we were attempting to create objects with the same id within the same
transaction.

This commit ensures that only attempt to write a contact with the same
id once, and that we check for all exsiting contacts before hand.

For file, we ensure that we don't attempt to write 2 files with the same
id more than once

packages/local-sync/src/models/contact.js

@@ -1,3 +1,5 @@
+const crypto = require('crypto')
+
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('contact', {
     id: {type: Sequelize.STRING(65), primaryKey: true},
@@ -12,8 +14,13 @@ module.exports = (sequelize, Sequelize) => {
         fields: ['id'],
       },
     ],
+    classMethods: {
+      hash({email}) {
+        return crypto.createHash('sha256').update(email, 'utf8').digest('hex');
+      },
+    },
     instanceMethods: {
-      toJSON: function toJSON() {
+      toJSON() {
         return {
           id: `${this.publicId}`,
           account_id: this.accountId,

packages/local-sync/src/new-message-processor/extract-contacts.js

@@ -1,4 +1,3 @@
-const cryptography = require('crypto');
 
 function isContactMeaningful(contact) {
   // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
@@ -14,32 +13,40 @@ function isContactMeaningful(contact) {
 }
 
 async function extractContacts({db, message}) {
+  const {Contact} = db
   let allContacts = [];
   ['to', 'from', 'bcc', 'cc'].forEach((field) => {
     allContacts = allContacts.concat(message[field])
   })
 
   const meaningfulContacts = allContacts.filter(c => isContactMeaningful(c));
+  const contactsDataById = new Map()
+  meaningfulContacts.forEach(c => {
+    const id = Contact.hash(c)
+    const cdata = {
+      id,
+      name: c.name,
+      email: c.email,
+      accountId: message.accountId,
+    }
+    contactsDataById.set(id, cdata)
+  })
+  const existingContacts = await Contact.findAll({
+    where: {
+      id: Array.from(contactsDataById.keys()),
+    },
+  })
 
   await db.sequelize.transaction(async (transaction) => {
-    const promises = [];
-
-    for (const c of meaningfulContacts) {
-      const id = cryptography.createHash('sha256').update(c.email, 'utf8').digest('hex');
-      const existing = await db.Contact.findById(id);
-      const cdata = {
-        id,
-        name: c.name,
-        email: c.email,
-        accountId: message.accountId,
-      };
-
+    const promises = []
+    for (const c of contactsDataById.values()) {
+      const existing = existingContacts.find(({id}) => id === c.id)
       if (!existing) {
-        promises.push(db.Contact.create(cdata, {transaction}));
+        promises.push(Contact.create(c, {transaction}));
       } else {
-        const updateRequired = (cdata.name !== existing.name);
+        const updateRequired = (c.name !== existing.name);
         if (updateRequired) {
-          promises.push(existing.update(cdata, {transaction}));
+          promises.push(existing.update(c, {transaction}));
         }
       }
     }

packages/local-sync/src/new-message-processor/extract-files.js

@@ -1,24 +1,28 @@
 
-function collectFilesFromStruct({db, message, struct}) {
+function collectFilesFromStruct({db, message, struct, fileIds = new Set()}) {
   const {File} = db;
   let collected = [];
 
   for (const part of struct) {
     if (part.constructor === Array) {
-      collected = collected.concat(collectFilesFromStruct({db, message, struct: part}));
+      collected = collected.concat(collectFilesFromStruct({db, message, struct: part, fileIds}));
     } else if (part.type !== 'text' && part.disposition) {
       // Only exposes partId for inline attachments
       const partId = part.disposition.type === 'inline' ? part.partID : null;
       const filename = part.disposition.params ? part.disposition.params.filename : null;
-      collected.push(File.build({
-        filename: filename,
-        partId: partId,
-        messageId: message.id,
-        contentType: `${part.type}/${part.subtype}`,
-        accountId: message.accountId,
-        size: part.size,
-        id: `${message.id}-${partId}-${part.size}`,
-      }));
+      const fileId = `${message.id}-${partId}-${part.size}`
+      if (!fileIds.has(fileId)) {
+        collected.push(File.build({
+          id: fileId,
+          partId: partId,
+          size: part.size,
+          filename: filename,
+          messageId: message.id,
+          accountId: message.accountId,
+          contentType: `${part.type}/${part.subtype}`,
+        }));
+        fileIds.add(fileId)
+      }
     }
   }