Revert "Revert "[feat] Add support for send later""

Arc land messed up and landed a not fully merged branch. (Seriously – I
had merged a copy of my branch with master to see how easy it would be.
Because I didn't want to merge the whole thing, I blindly committed my
changes and switched back to my real branch). To my great surprise, arc
decided to use the wrong branch when landing it.

Original commit message:

Summary:
    Finally, here it is! Send later, with support for open tracking but
without support for attachments yet. It took me some time to find the
right way to do things.

    **The send later dilemna**

    There's two ways we could handle send later:
    1. do everything on the client
    2. process the message in the cloud

    1. is very tempting because it would make the cloud server very
simple. Unfortunately, it has some serious limitations, for example,
setting the "Date" message header. That's why I chose to go with 2. When
a user presses the "Send Later" button, we save the open/link tracking
metadata and fills in all the required fields. I added a custom endpoint
to the K2 API to do this, `/drafts/build`. After that, we save the JSON
contents of the message as metadata.

    When we process metadata, we simply create a MIME message from the
JSON and send it.

    **Limitations**

    Right now, send later doesn't support and attachments. There's also
some minor code duplication which needs to be refactored away.

Test Plan: Tested manually. Checked that regular send still worked, too.

Reviewers: mark, spang, halla, juan, evan

Reviewed By: evan

Differential Revision: https://phab.nylas.com/D4054

packages/client-sync/src/local-api/routes/drafts.js

@@ -1,6 +1,8 @@
+const {MessageFactory, Errors: {APIError}} = require('isomorphic-core')
 const Joi = require('joi');
+const crypto = require('crypto');
 
-// TODO: This is a placeholder
+// TODO: This is a placeholder.
 module.exports = (server) => {
   server.route({
     method: 'GET',
@@ -24,4 +26,74 @@ module.exports = (server) => {
       reply('[]');
     },
   });
+
+  // This is a placeholder route we use to make send-later happy.
+  // Eventually, we should flesh it out and actually sync back drafts.
+  server.route({
+    method: ['PUT', 'POST'],
+    path: `/drafts/{objectId?}`,
+    config: {
+      description: `Dummy draft update`,
+      tags: ['drafts'],
+      payload: {
+        output: 'data',
+        parse: true,
+      },
+      validate: {
+        params: {
+          objectId: Joi.string(),
+        },
+      },
+    },
+    handler: (request, reply) => {
+      const data = request.payload;
+      data.id = crypto.createHash('sha256').update(data.client_id, 'utf8').digest('hex')
+      return reply(data);
+    },
+  })
+
+  server.route({
+    method: ['PUT', 'POST'],
+    path: `/drafts/build`,
+    config: {
+      description: `Returns a ready-made draft message. Used by our send later plugin.`,
+      tags: ['drafts'],
+      payload: {
+        output: 'data',
+        parse: true,
+      },
+    },
+    handler: async (request, reply) => {
+      const db = await request.getAccountDatabase();
+      const account = request.auth.credentials;
+
+      let sentFolderName;
+      let sentFolder;
+      let trashFolderName;
+
+      if (account.provider === 'gmail') {
+        sentFolder = await db.Label.find({where: {role: 'sent'}});
+      } else {
+        sentFolder = await db.Folder.find({where: {role: 'sent'}});
+      }
+
+      if (sentFolder) {
+        sentFolderName = sentFolder.name;
+      } else {
+        throw new APIError(`Can't find sent folder name.`, 500);
+      }
+
+      const trashFolder = await db.Folder.find({where: {role: 'trash'}});
+
+      if (trashFolder) {
+        trashFolderName = trashFolder.name;
+      } else {
+        throw new APIError(`Can't find trash folder name.`, 500);
+      }
+
+      const message = await MessageFactory.buildForSend(db, request.payload);
+      const ret = Object.assign(message.toJSON(), { sentFolderName, trashFolderName });
+      reply(JSON.stringify(ret));
+    },
+  });
 }

packages/client-sync/src/local-api/routes/send.js

@@ -1,5 +1,5 @@
 const Joi = require('joi');
-const Utils = require('../../shared/utils');
+const {SendUtils} = require('isomorphic-core');
 const {createAndReplyWithSyncbackRequest} = require('../route-helpers');
 
 
@@ -70,7 +70,7 @@ module.exports = (server) => {
       const {messageId} = request.params;
       const {sentPerRecipient} = request.payload;
 
-      if (!Utils.isValidId(messageId)) {
+      if (!SendUtils.isValidId(messageId)) {
         reply.badRequest(`messageId is not a base-36 integer`)
         return
       }

packages/client-sync/src/local-sync-worker/syncback-tasks/ensure-message-in-sent-folder.imap.js

@@ -1,7 +1,7 @@
-const {SendmailClient, Provider, Errors: {APIError}} = require('isomorphic-core')
+const {SendmailClient, Provider,
+       Errors: {APIError}, MessageFactory: {getReplyHeaders}} = require('isomorphic-core')
 const {SyncbackIMAPTask} = require('./syncback-task')
 const SyncTaskFactory = require('../sync-task-factory');
-const {getReplyHeaders} = require('../../shared/message-factory')
 
 
 async function deleteGmailSentMessages({db, imap, provider, headerMessageId}) {

packages/client-sync/src/local-sync-worker/syncback-tasks/send-message-per-recipient.smtp.js

@@ -1,7 +1,6 @@
-const {Errors: {APIError}} = require('isomorphic-core')
-const Utils = require('../../shared/utils')
+const {Errors: {APIError}, SendUtils} = require('isomorphic-core')
 const {SyncbackSMTPTask} = require('./syncback-task')
-const MessageFactory = require('../../shared/message-factory')
+const {MessageFactory} = require('isomorphic-core')
 
 
 /**
@@ -36,7 +35,14 @@ class SendMessagePerRecipientSMTP extends SyncbackSMTPTask {
     await syncbackRequest.update({
       status: 'INPROGRESS-NOTRETRYABLE',
     })
-    const sendResult = await this._sendPerRecipient({db, smtp, baseMessage, usesOpenTracking, usesLinkTracking})
+
+    let sendResult;
+    try {
+      sendResult = await this._sendPerRecipient({
+        db, smtp, baseMessage, logger: this._logger, usesOpenTracking, usesLinkTracking})
+    } catch (err) {
+      throw new APIError('SendMessagePerRecipient: Sending failed for all recipients', 500);
+    }
     /**
      * Once messages have actually been delivered, we need to be very
      * careful not to throw an error from this task. An Error in the send
@@ -79,7 +85,7 @@ class SendMessagePerRecipientSMTP extends SyncbackSMTPTask {
         usesLinkTracking,
       })
 
-      const individualizedMessage = Utils.copyModel(Message, baseMessage, {
+      const individualizedMessage = SendUtils.copyModel(Message, baseMessage, {
         body: customBody,
       })
       // TODO we set these temporary properties which aren't stored in the

packages/client-sync/src/local-sync-worker/syncback-tasks/send-message.smtp.js

@@ -1,4 +1,4 @@
-const MessageFactory = require('../../shared/message-factory')
+const {MessageFactory} = require('isomorphic-core')
 const {SyncbackSMTPTask} = require('../syncback-tasks/syncback-task')
 
 /**

packages/client-sync/src/message-processor/index.js

@@ -6,7 +6,7 @@ const mkdirp = require('mkdirp');
 const detectThread = require('./detect-thread');
 const extractFiles = require('./extract-files');
 const extractContacts = require('./extract-contacts');
-const MessageFactory = require('../shared/message-factory')
+const {MessageFactory} = require('isomorphic-core');
 const LocalDatabaseConnector = require('../shared/local-database-connector');
 const {BatteryStatusManager} = require('nylas-exports');
 

packages/client-sync/src/shared/utils.js

@@ -1,24 +0,0 @@
-module.exports = {
-  copyModel(Model, model, updates = {}) {
-    const fields = Object.keys(model.dataValues)
-    const data = {}
-    for (const field of fields) {
-    // We can't just copy over the values directly from `dataValues` because
-    // they are the raw values, and we would ignore custom getters.
-    // Rather, we access them from the model instance.
-    // For example our JSON database type, is simply a string and the custom
-    // getter parses it into json. We want to get the parsed json, not the
-    // string
-      data[field] = model[field]
-    }
-    return Model.build(Object.assign({}, data, updates))
-  },
-
-  isValidId(value) {
-    if (value == null) { return false; }
-    if (isNaN(parseInt(value, 36))) {
-      return false
-    }
-    return true
-  },
-}

packages/isomorphic-core/index.js

@@ -20,5 +20,7 @@ module.exports = {
   BackoffScheduler: require('./src/backoff-schedulers').BackoffScheduler,
   ExponentialBackoffScheduler: require('./src/backoff-schedulers').ExponentialBackoffScheduler,
   MetricsReporter: require('./src/metrics-reporter').default,
+  MessageFactory: require('./src/message-factory'),
+  SendUtils: require('./src/send-utils'),
   executeJasmine: require('./spec/jasmine/execute').default,
 }

packages/isomorphic-core/package.json

@@ -8,6 +8,7 @@
   },
   "dependencies": {
     "atob": "2.0.3",
+    "btoa": "1.1.2",
     "imap": "github:jstejada/node-imap#fix-parse-body-list",
     "imap-provider-settings": "github:nylas/imap-provider-settings",
     "jasmine": "2.x.x",
@@ -19,7 +20,10 @@
     "rx-lite": "4.0.8",
     "sequelize": "3.28.0",
     "underscore": "1.8.3",
-    "xoauth2": "1.2.0"
+    "xoauth2": "1.2.0",
+    "he": "1.1.0",
+    "iconv": "2.2.1",
+    "mimelib": "0.2.19"
   },
   "author": "Nylas",
   "license": "ISC"

packages/isomorphic-core/spec/message-factory-spec.js

@@ -1,4 +1,4 @@
-const {parseFromImap, parseSnippet, parseContacts} = require('../../src/shared/message-factory');
+const {parseFromImap, parseSnippet, parseContacts} = require('../src/message-factory');
 const {forEachJSONFixture, forEachHTMLAndTXTFixture, ACCOUNT_ID, getTestDatabase} = require('../helpers');
 
 xdescribe('MessageFactory', function MessageFactorySpecs() {

packages/isomorphic-core/src/message-factory.es6

@@ -1,4 +1,5 @@
 /* eslint no-useless-escape: 0 */
+import _ from 'underscore';
 const mimelib = require('mimelib');
 const encoding = require('encoding');
 const he = require('he');
@@ -6,13 +7,29 @@ const os = require('os');
 const fs = require('fs');
 const path = require('path');
 const mkdirp = require('mkdirp');
-const {Errors: {APIError}} = require('isomorphic-core');
-const {N1CloudAPI, RegExpUtils, Utils} = require('nylas-exports');
+const btoa = require('btoa')
+const {APIError} = require('./errors');
+const {deepClone} = require('./send-utils');
 
 // Aiming for the former in length, but the latter is the hard db cutoff
 const SNIPPET_SIZE = 100;
 const SNIPPET_MAX_SIZE = 255;
 
+// Copied from regexp-utils.coffee.
+// FIXME @karim: share code, somehow.
+// Test cases: https://regex101.com/r/cK0zD8/4
+// Catches link tags containing which are:
+// - Non empty
+// - Not a mailto: link
+// Returns the following capturing groups:
+// 1. start of the opening a tag to href="
+// 2. The contents of the href without quotes
+// 3. the rest of the opening a tag
+// 4. the contents of the a tag
+// 5. the closing tag
+function urlLinkTagRegex() {
+  return new RegExp(/(<a.*?href\s*?=\s*?['"])((?!mailto).+?)(['"].*?>)([\s\S]*?)(<\/a>)/gim);
+}
 
 // Format of input: ['a@example.com, B <b@example.com>', 'c@example.com'],
 // where each element of the array is the unparsed contents of a single
@@ -40,7 +57,6 @@ function parseContacts(input) {
   return contacts;
 }
 
-
 function parseSnippet(body) {
   const doc = new DOMParser().parseFromString(body, 'text/html')
   const skipTags = new Set(['TITLE', 'SCRIPT', 'STYLE', 'IMG']);
@@ -117,7 +133,7 @@ function htmlifyPlaintext(text) {
 
 
 function replaceMessageIdInBodyTrackingLinks(messageId, originalBody) {
-  const regex = new RegExp(`(${N1CloudAPI.APIRoot}.+?)MESSAGE_ID`, 'g')
+  const regex = new RegExp(`(https://.+?)MESSAGE_ID`, 'g')
   return originalBody.replace(regex, `$1${messageId}`)
 }
 
@@ -126,7 +142,7 @@ function stripTrackingLinksFromBody(originalBody) {
   let body = originalBody.replace(/<img class="n1-open"[^<]+src="([a-zA-Z0-9-_:/.]*)">/g, () => {
     return "";
   });
-  body = body.replace(RegExpUtils.urlLinkTagRegex(), (match, prefix, url, suffix, content, closingTag) => {
+  body = body.replace(urlLinkTagRegex(), (match, prefix, url, suffix, content, closingTag) => {
     const param = url.split("?")[1];
     if (param) {
       const link = decodeURIComponent(param.split("=")[1]);
@@ -150,7 +166,7 @@ function buildTrackingBodyForRecipient({baseMessage, recipient, usesOpenTracking
     });
   }
   if (usesLinkTracking) {
-    customBody = customBody.replace(RegExpUtils.urlLinkTagRegex(), (match, prefix, url, suffix, content, closingTag) => {
+    customBody = customBody.replace(urlLinkTagRegex(), (match, prefix, url, suffix, content, closingTag) => {
       return `${prefix}${url}&r=${encodedEmail}${suffix}${content}${closingTag}`;
     });
   }
@@ -306,7 +322,7 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
    * zero pads digit dates. To make the hashes line up, we need to ensure
    * that the date string used in the ID generation is also zero-padded.
    */
-  const messageForHashing = Utils.deepClone(parsedMessage)
+  const messageForHashing = deepClone(parsedMessage)
   messageForHashing.date = Message.dateString(parsedMessage.date);
   // Inversely to `buildForSend`, we leave the date header as it is so that the
   // format is consistent for the generative IDs, then convert it to a Date object
@@ -407,7 +423,7 @@ async function buildForSend(db, json) {
   // nodemailer buildmail function that gives us the raw message and replaces
   // the date header with this modified UTC string
   // https://github.com/nodemailer/buildmail/blob/master/lib/buildmail.js#L470
-  const messageForHashing = Utils.deepClone(message)
+  const messageForHashing = deepClone(message)
   messageForHashing.date = Message.dateString(date)
   message.id = Message.hash(messageForHashing)
   message.body = replaceMessageIdInBodyTrackingLinks(message.id, message.body)

packages/isomorphic-core/src/models/account.js

@@ -1,8 +1,10 @@
 const atob = require('atob')
 const crypto = require('crypto');
+
 const {JSONColumn, JSONArrayColumn} = require('../database-types');
 const {SUPPORTED_PROVIDERS, credentialsForProvider} = require('../auth-helpers');
 
+
 const {DB_ENCRYPTION_ALGORITHM, DB_ENCRYPTION_PASSWORD} = process.env;
 
 module.exports = (sequelize, Sequelize) => {
@@ -136,11 +138,13 @@ module.exports = (sequelize, Sequelize) => {
             secure: ssl_required,
           };
         }
+
         if (this.provider === 'gmail') {
           const {xoauth2} = connectionCredentials;
           if (!xoauth2) {
             throw new Error("Missing XOAuth2 Token")
           }
+
           const token = this.bearerToken(xoauth2);
           config.auth = { user: connectionSettings.smtp_username, xoauth2: token }
         } else if (SUPPORTED_PROVIDERS.has(this.provider)) {

packages/isomorphic-core/src/send-utils.es6

@@ -0,0 +1,60 @@
+const _ = require('underscore');
+
+function copyModel(Model, model, updates = {}) {
+  const fields = Object.keys(model.dataValues)
+  const data = {}
+  for (const field of fields) {
+  // We can't just copy over the values directly from `dataValues` because
+  // they are the raw values, and we would ignore custom getters.
+  // Rather, we access them from the model instance.
+  // For example our JSON database type, is simply a string and the custom
+  // getter parses it into json. We want to get the parsed json, not the
+  // string
+    data[field] = model[field]
+  }
+  return Model.build(Object.assign({}, data, updates))
+}
+
+function isValidId(value) {
+  if (value == null) { return false; }
+  if (isNaN(parseInt(value, 36))) {
+    return false
+  }
+  return true
+}
+
+function deepClone(object, customizer, stackSeen = [], stackRefs = []) {
+  let newObject;
+  if (!_.isObject(object)) { return object; }
+  if (_.isFunction(object)) { return object; }
+
+  if (_.isArray(object)) {
+    // http://perfectionkills.com/how-ecmascript-5-still-does-not-allow-to-subclass-an-array/
+    newObject = [];
+  } else if (object instanceof Date) {
+    // You can't clone dates by iterating through `getOwnPropertyNames`
+    // of the Date object. We need to special-case Dates.
+    newObject = new Date(object);
+  } else {
+    newObject = Object.create(Object.getPrototypeOf(object));
+  }
+
+  // Circular reference check
+  const seenIndex = stackSeen.indexOf(object);
+  if (seenIndex >= 0) { return stackRefs[seenIndex]; }
+  stackSeen.push(object); stackRefs.push(newObject);
+
+  // It's important to use getOwnPropertyNames instead of Object.keys to
+  // get the non-enumerable items as well.
+  for (const key of Array.from(Object.getOwnPropertyNames(object))) {
+    const newVal = deepClone(object[key], customizer, stackSeen, stackRefs);
+    if (_.isFunction(customizer)) {
+      newObject[key] = customizer(key, newVal);
+    } else {
+      newObject[key] = newVal;
+    }
+  }
+  return newObject;
+}
+
+module.exports = {copyModel, isValidId, deepClone};

packages/isomorphic-core/src/sendmail-client.es6

@@ -20,6 +20,7 @@ class SendmailClient {
   async _send(msgData) {
     let error;
     let results;
+
     // disable nodemailer's automatic X-Mailer header
     msgData.xMailer = false;
     for (let i = 0; i <= MAX_RETRIES; i++) {
@@ -73,7 +74,7 @@ class SendmailClient {
     msgData.date = message.date;
     msgData.subject = message.subject;
     msgData.html = message.body;
-    msgData.messageId = message.headerMessageId;
+    msgData.messageId = message.headerMessageId || message.message_id_header;
 
     msgData.attachments = []
     const uploads = message.uploads || []