[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
/* eslint no-useless-escape: 0 */
|
2016-11-30 03:18:51 +08:00
|
|
|
const mimelib = require('mimelib');
|
2016-12-09 10:30:57 +08:00
|
|
|
const encoding = require('encoding');
|
2016-12-16 01:17:45 +08:00
|
|
|
const he = require('he');
|
2016-12-16 02:18:41 +08:00
|
|
|
const os = require('os');
|
|
|
|
const fs = require('fs');
|
2016-12-17 06:39:50 +08:00
|
|
|
const path = require('path');
|
2016-12-16 02:18:41 +08:00
|
|
|
const mkdirp = require('mkdirp');
|
2016-12-30 02:35:24 +08:00
|
|
|
const {Errors: {APIError}} = require('isomorphic-core');
|
2016-12-24 08:15:34 +08:00
|
|
|
const {N1CloudAPI, RegExpUtils, Utils} = require('nylas-exports');
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-17 06:39:50 +08:00
|
|
|
// Aiming for the former in length, but the latter is the hard db cutoff
|
2016-12-09 10:30:57 +08:00
|
|
|
const SNIPPET_SIZE = 100;
|
|
|
|
const SNIPPET_MAX_SIZE = 255;
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-17 06:39:50 +08:00
|
|
|
|
2016-12-13 05:33:34 +08:00
|
|
|
// The input is the value of a to/cc/bcc/from header as parsed by the imap
|
|
|
|
// library we're using, but it currently parses them in a weird format. If an
|
|
|
|
// email is sent to a@example.com and b@example.com, the parsed output of the
|
|
|
|
// 'to' header is ['a@example.com, b@example.com']. (Note both emails are in
|
|
|
|
// the same string.) When fixed, this function will need to update accordingly.
|
|
|
|
function extractContacts(input) {
|
|
|
|
if (!input || input.length === 0 || !input[0]) {
|
|
|
|
return [];
|
|
|
|
}
|
2016-12-30 02:56:00 +08:00
|
|
|
const values = mimelib.parseAddresses(input[0]);
|
|
|
|
if (!values || values.length === 0 || !input[0]) {
|
|
|
|
return [];
|
|
|
|
}
|
2016-11-30 03:18:51 +08:00
|
|
|
return values.map(v => {
|
2016-12-30 02:56:00 +08:00
|
|
|
if (!v || v.length === 0) {
|
2016-12-15 17:15:51 +08:00
|
|
|
return null
|
|
|
|
}
|
2016-12-30 02:56:00 +08:00
|
|
|
const {name, address: email} = v;
|
|
|
|
// contacts without an email address are worthless, especially when
|
|
|
|
// extracted from emails
|
|
|
|
if (!email) {
|
|
|
|
return null;
|
|
|
|
}
|
2016-11-30 03:18:51 +08:00
|
|
|
return {name, email}
|
|
|
|
})
|
2016-12-15 17:15:51 +08:00
|
|
|
.filter(c => c != null)
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
|
2016-12-17 06:39:50 +08:00
|
|
|
|
2016-12-14 04:42:38 +08:00
|
|
|
function extractSnippet(plainBody, htmlBody) {
|
2016-12-21 07:10:22 +08:00
|
|
|
let snippetText = plainBody ? plainBody.trim() : '';
|
2016-12-14 04:42:38 +08:00
|
|
|
if (htmlBody) {
|
|
|
|
const doc = new DOMParser().parseFromString(htmlBody, 'text/html')
|
2016-12-21 07:10:22 +08:00
|
|
|
const skipTags = new Set(['TITLE', 'SCRIPT', 'STYLE', 'IMG']);
|
2016-12-23 09:08:18 +08:00
|
|
|
const noSpaceTags = new Set(['B', 'I', 'STRONG', 'EM', 'SPAN']);
|
2016-12-14 04:42:38 +08:00
|
|
|
|
2016-12-21 07:10:22 +08:00
|
|
|
const treeWalker = document.createTreeWalker(doc, NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT, (node) => {
|
|
|
|
if (skipTags.has(node.tagName)) {
|
|
|
|
// skip this node and all its children
|
|
|
|
return NodeFilter.FILTER_REJECT;
|
|
|
|
}
|
2016-12-14 04:42:38 +08:00
|
|
|
if (node.nodeType === Node.TEXT_NODE) {
|
|
|
|
const nodeValue = node.nodeValue ? node.nodeValue.trim() : null;
|
|
|
|
if (nodeValue) {
|
2016-12-21 07:10:22 +08:00
|
|
|
return NodeFilter.FILTER_ACCEPT;
|
2016-12-14 04:42:38 +08:00
|
|
|
}
|
2016-12-23 09:08:18 +08:00
|
|
|
return NodeFilter.FILTER_SKIP;
|
2016-12-14 04:42:38 +08:00
|
|
|
}
|
2016-12-23 09:08:18 +08:00
|
|
|
return NodeFilter.FILTER_ACCEPT;
|
2016-12-21 07:10:22 +08:00
|
|
|
});
|
2016-12-14 04:42:38 +08:00
|
|
|
|
2016-12-21 07:10:22 +08:00
|
|
|
let extractedText = "";
|
2016-12-23 09:08:18 +08:00
|
|
|
let lastNodeTag = "";
|
2016-12-21 07:10:22 +08:00
|
|
|
while (treeWalker.nextNode()) {
|
2016-12-23 09:08:18 +08:00
|
|
|
if (treeWalker.currentNode.nodeType === Node.ELEMENT_NODE) {
|
|
|
|
lastNodeTag = treeWalker.currentNode.nodeName;
|
|
|
|
} else {
|
|
|
|
if (extractedText && !noSpaceTags.has(lastNodeTag)) {
|
|
|
|
extractedText += " ";
|
|
|
|
}
|
|
|
|
extractedText += treeWalker.currentNode.nodeValue;
|
|
|
|
if (extractedText.length > SNIPPET_MAX_SIZE) {
|
|
|
|
break;
|
|
|
|
}
|
2016-12-21 07:10:22 +08:00
|
|
|
}
|
2016-12-14 04:42:38 +08:00
|
|
|
}
|
2016-12-21 07:10:22 +08:00
|
|
|
|
2016-12-23 09:08:18 +08:00
|
|
|
snippetText = extractedText.trim();
|
2016-12-14 04:42:38 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// clean up and trim snippet
|
2016-12-21 07:10:22 +08:00
|
|
|
let trimmed = snippetText.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ').substr(0, SNIPPET_MAX_SIZE);
|
2016-12-14 04:42:38 +08:00
|
|
|
if (trimmed) {
|
2016-12-17 06:39:50 +08:00
|
|
|
// TODO: strip quoted text from snippets also
|
2016-12-14 04:42:38 +08:00
|
|
|
// trim down to approx. SNIPPET_SIZE w/out cutting off words right in the
|
|
|
|
// middle (if possible)
|
|
|
|
const wordBreak = trimmed.indexOf(' ', SNIPPET_SIZE);
|
|
|
|
if (wordBreak !== -1) {
|
|
|
|
trimmed = trimmed.substr(0, wordBreak);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return trimmed;
|
|
|
|
}
|
|
|
|
|
2016-12-17 06:39:50 +08:00
|
|
|
|
|
|
|
// Preserve whitespacing on plaintext emails -- has the side effect of
|
|
|
|
// monospacing, but that seems OK and perhaps sometimes even desired (for e.g.
|
|
|
|
// ascii art, alignment)
|
|
|
|
function htmlifyPlaintext(text) {
|
2016-12-16 01:17:45 +08:00
|
|
|
const escapedText = he.escape(text);
|
|
|
|
return `<pre class="nylas-plaintext">${escapedText}</pre>`;
|
2016-12-14 04:42:38 +08:00
|
|
|
}
|
|
|
|
|
2016-12-17 06:39:50 +08:00
|
|
|
|
|
|
|
function replaceMessageIdInBodyTrackingLinks(messageId, originalBody) {
|
|
|
|
const regex = new RegExp(`(${N1CloudAPI.APIRoot}.+?)MESSAGE_ID`, 'g')
|
|
|
|
return originalBody.replace(regex, `$1${messageId}`)
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
function stripTrackingLinksFromBody(originalBody) {
|
|
|
|
let body = originalBody.replace(/<img class="n1-open"[^<]+src="([a-zA-Z0-9-_:/.]*)">/g, () => {
|
|
|
|
return "";
|
|
|
|
});
|
|
|
|
body = body.replace(RegExpUtils.urlLinkTagRegex(), (match, prefix, url, suffix, content, closingTag) => {
|
|
|
|
const param = url.split("?")[1];
|
|
|
|
if (param) {
|
|
|
|
const link = decodeURIComponent(param.split("=")[1]);
|
|
|
|
return `${prefix}${link}${suffix}${content}${closingTag}`;
|
|
|
|
}
|
|
|
|
return match;
|
|
|
|
});
|
|
|
|
return body;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
function buildTrackingBodyForRecipient({baseMessage, recipient, usesOpenTracking, usesLinkTracking} = {}) {
|
|
|
|
const {id: messageId, body} = baseMessage
|
|
|
|
const encodedEmail = btoa(recipient.email)
|
|
|
|
.replace(/\+/g, '-')
|
|
|
|
.replace(/\//g, '_');
|
|
|
|
let customBody = body
|
|
|
|
if (usesOpenTracking) {
|
|
|
|
customBody = customBody.replace(/<img class="n1-open"[^<]+src="([a-zA-Z0-9-_:/.]*)">/g, (match, url) => {
|
|
|
|
return `<img class="n1-open" width="0" height="0" style="border:0; width:0; height:0;" src="${url}?r=${encodedEmail}">`;
|
|
|
|
});
|
|
|
|
}
|
|
|
|
if (usesLinkTracking) {
|
|
|
|
customBody = customBody.replace(RegExpUtils.urlLinkTagRegex(), (match, prefix, url, suffix, content, closingTag) => {
|
|
|
|
return `${prefix}${url}&r=${encodedEmail}${suffix}${content}${closingTag}`;
|
|
|
|
});
|
|
|
|
}
|
|
|
|
return replaceMessageIdInBodyTrackingLinks(messageId, customBody);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
function getReplyHeaders(messageReplyingTo) {
|
|
|
|
let inReplyTo;
|
|
|
|
let references;
|
|
|
|
if (messageReplyingTo.headerMessageId) {
|
|
|
|
inReplyTo = messageReplyingTo.headerMessageId;
|
|
|
|
if (messageReplyingTo.references) {
|
|
|
|
references = messageReplyingTo.references.concat(messageReplyingTo.headerMessageId);
|
|
|
|
} else {
|
|
|
|
references = [messageReplyingTo.headerMessageId];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return {inReplyTo, references}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Since we only fetch the MIME structure and specific desired MIME parts from
|
|
|
|
// IMAP, we unfortunately can't use an existing library like mailparser to parse
|
|
|
|
// the message, and have to do fun stuff like deal with character sets and
|
|
|
|
// content-transfer-encodings ourselves.
|
2016-12-06 04:16:53 +08:00
|
|
|
async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder}) {
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
const {Message, Label} = db
|
2016-12-09 10:30:57 +08:00
|
|
|
const {attributes} = imapMessage
|
|
|
|
|
2016-11-30 03:18:51 +08:00
|
|
|
const body = {}
|
2016-12-09 10:30:57 +08:00
|
|
|
for (const {id, mimetype, transferEncoding, charset} of desiredParts) {
|
|
|
|
// see https://www.w3.org/Protocols/rfc1341/5_Content-Transfer-Encoding.html
|
2016-12-17 02:42:28 +08:00
|
|
|
if (!transferEncoding || new Set(['7bit', '8bit', 'binary']).has(transferEncoding.toLowerCase())) {
|
2016-12-09 10:30:57 +08:00
|
|
|
// NO transfer encoding has been performed --- how to decode to a string
|
|
|
|
// depends ONLY on the charset, which defaults to 'ascii' according to
|
|
|
|
// https://tools.ietf.org/html/rfc2045#section-5.2
|
|
|
|
const convertedBuffer = encoding.convert(imapMessage.parts[id], 'utf-8', charset || 'ascii')
|
|
|
|
body[mimetype] = convertedBuffer.toString('utf-8');
|
|
|
|
} else if (transferEncoding.toLowerCase() === 'quoted-printable') {
|
|
|
|
body[mimetype] = mimelib.decodeQuotedPrintable(imapMessage.parts[id], charset || 'ascii');
|
|
|
|
} else if (transferEncoding.toLowerCase() === 'base64') {
|
|
|
|
body[mimetype] = mimelib.decodeBase64(imapMessage.parts[id], charset || 'ascii');
|
2016-11-30 03:18:51 +08:00
|
|
|
} else {
|
2016-12-17 03:06:04 +08:00
|
|
|
// custom x-token content-transfer-encodings
|
2016-12-09 10:30:57 +08:00
|
|
|
return Promise.reject(new Error(`Unsupported Content-Transfer-Encoding ${transferEncoding}, mimetype ${mimetype}`))
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
}
|
2016-12-09 10:30:57 +08:00
|
|
|
const headers = imapMessage.headers.toString('ascii');
|
2016-12-23 00:51:16 +08:00
|
|
|
const parsedHeaders = mimelib.parseHeaders(headers);
|
2016-11-30 03:18:51 +08:00
|
|
|
for (const key of ['x-gm-thrid', 'x-gm-msgid', 'x-gm-labels']) {
|
|
|
|
parsedHeaders[key] = attributes[key];
|
|
|
|
}
|
|
|
|
|
2016-12-09 10:30:57 +08:00
|
|
|
const parsedMessage = {
|
2016-11-30 03:18:51 +08:00
|
|
|
to: extractContacts(parsedHeaders.to),
|
|
|
|
cc: extractContacts(parsedHeaders.cc),
|
|
|
|
bcc: extractContacts(parsedHeaders.bcc),
|
|
|
|
from: extractContacts(parsedHeaders.from),
|
|
|
|
replyTo: extractContacts(parsedHeaders['reply-to']),
|
|
|
|
accountId: accountId,
|
|
|
|
body: body['text/html'] || body['text/plain'] || body['application/pgp-encrypted'] || '',
|
2016-12-09 10:30:57 +08:00
|
|
|
snippet: null,
|
2016-11-30 03:18:51 +08:00
|
|
|
unread: !attributes.flags.includes('\\Seen'),
|
|
|
|
starred: attributes.flags.includes('\\Flagged'),
|
2016-12-17 06:39:50 +08:00
|
|
|
// Make sure we use the date from the headers because we use the header date
|
|
|
|
// for generating message ids.
|
|
|
|
// `attributes.date` is the server generated date and might differ from the
|
2016-12-24 02:03:56 +08:00
|
|
|
// header across accounts
|
|
|
|
// TODO: how to exclude the date header from the hash if there is no
|
|
|
|
// Date: header and we have to use the IMAP server date for message sort
|
|
|
|
// & display? seems like it should be OK within an account, but might
|
|
|
|
// generate different message IDs across different accounts (which I
|
|
|
|
// don't think is a problem we're intending to solve...)
|
|
|
|
date: parsedHeaders.date ? parsedHeaders.date[0] : imapMessage.attributes.date,
|
2016-11-30 03:18:51 +08:00
|
|
|
folderImapUID: attributes.uid,
|
2016-12-06 04:16:53 +08:00
|
|
|
folderId: folder.id,
|
2016-11-30 03:18:51 +08:00
|
|
|
folder: null,
|
|
|
|
labels: [],
|
|
|
|
headers: parsedHeaders,
|
|
|
|
headerMessageId: parsedHeaders['message-id'] ? parsedHeaders['message-id'][0] : '',
|
2016-12-08 02:22:31 +08:00
|
|
|
gMsgId: parsedHeaders['x-gm-msgid'],
|
2016-12-17 02:42:28 +08:00
|
|
|
subject: parsedHeaders.subject ? parsedHeaders.subject[0] : '(no subject)',
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
2016-12-24 08:15:34 +08:00
|
|
|
// Inversely to `buildForSend`, we leave the date header as it is so that the
|
|
|
|
// format is consistent for the generative IDs, then convert it to a Date object
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
parsedMessage.id = Message.hash(parsedMessage)
|
2016-12-24 08:15:34 +08:00
|
|
|
parsedMessage.date = new Date(Date.parse(parsedMessage.date))
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2017-01-05 07:35:10 +08:00
|
|
|
// sometimes decoding results in a NUL-terminated body string, which makes
|
|
|
|
// SQLite blow up with an 'unrecognized token' error
|
|
|
|
parsedMessage.body = parsedMessage.body.replace(/\0/g, '');
|
|
|
|
|
2016-12-07 03:19:39 +08:00
|
|
|
if (!body['text/html'] && body['text/plain']) {
|
2016-12-17 06:39:50 +08:00
|
|
|
parsedMessage.body = htmlifyPlaintext(body['text/plain']);
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
|
2016-12-14 04:42:38 +08:00
|
|
|
parsedMessage.snippet = extractSnippet(body['text/plain'], body['text/html']);
|
2016-12-09 10:30:57 +08:00
|
|
|
parsedMessage.folder = folder
|
|
|
|
|
|
|
|
// TODO: unclear if this is necessary given we already have parsed labels
|
|
|
|
const xGmLabels = attributes['x-gm-labels']
|
2016-12-06 04:16:53 +08:00
|
|
|
if (xGmLabels) {
|
2016-12-09 10:30:57 +08:00
|
|
|
parsedMessage.folderImapXGMLabels = JSON.stringify(xGmLabels)
|
|
|
|
parsedMessage.labels = await Label.findXGMLabels(xGmLabels)
|
2016-12-06 04:16:53 +08:00
|
|
|
}
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-16 02:18:41 +08:00
|
|
|
if (process.env.NYLAS_DEBUG) {
|
|
|
|
const outJSON = JSON.stringify({imapMessage, desiredParts, result: parsedMessage});
|
|
|
|
const outDir = path.join(os.tmpdir(), "k2-parse-output", folder.name)
|
|
|
|
const outFile = path.join(outDir, imapMessage.attributes.uid.toString());
|
|
|
|
mkdirp.sync(outDir);
|
|
|
|
fs.writeFileSync(outFile, outJSON);
|
|
|
|
}
|
|
|
|
|
2016-12-09 10:30:57 +08:00
|
|
|
return parsedMessage;
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
|
2016-12-09 09:48:34 +08:00
|
|
|
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
async function buildForSend(db, json) {
|
|
|
|
const {Thread, Message} = db
|
2016-12-09 09:48:34 +08:00
|
|
|
let replyToThread;
|
|
|
|
let replyToMessage;
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
|
|
|
|
if (json.thread_id != null) {
|
2016-12-09 09:48:34 +08:00
|
|
|
replyToThread = await Thread.find({
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
where: {id: json.thread_id},
|
2016-12-09 09:48:34 +08:00
|
|
|
include: [{
|
|
|
|
model: Message,
|
|
|
|
as: 'messages',
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
attributes: ['id'],
|
2016-12-09 09:48:34 +08:00
|
|
|
}],
|
|
|
|
});
|
|
|
|
}
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
|
|
|
|
if (json.reply_to_message_id != null) {
|
|
|
|
replyToMessage = await Message.findById(json.reply_to_message_id);
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if (replyToThread && replyToMessage) {
|
|
|
|
if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
|
2016-12-15 15:46:36 +08:00
|
|
|
throw new APIError(`Message ${replyToMessage.id} is not in thread ${replyToThread.id}`, 400)
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let thread;
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
let replyHeaders = {};
|
2016-12-09 09:48:34 +08:00
|
|
|
if (replyToMessage) {
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
replyHeaders = getReplyHeaders(replyToMessage);
|
|
|
|
thread = await replyToMessage.getThread();
|
2016-12-09 09:48:34 +08:00
|
|
|
} else if (replyToThread) {
|
|
|
|
thread = replyToThread;
|
|
|
|
const previousMessages = thread.messages.filter(msg => !msg.isDraft);
|
|
|
|
if (previousMessages.length > 0) {
|
|
|
|
const lastMessage = previousMessages[previousMessages.length - 1]
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
replyHeaders = getReplyHeaders(lastMessage);
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
const {inReplyTo, references} = replyHeaders
|
2016-12-24 08:15:34 +08:00
|
|
|
const date = new Date()
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
const message = {
|
|
|
|
accountId: json.account_id,
|
|
|
|
threadId: thread ? thread.id : null,
|
|
|
|
headerMessageId: Message.buildHeaderMessageId(json.client_id),
|
|
|
|
from: json.from,
|
|
|
|
to: json.to,
|
|
|
|
cc: json.cc,
|
|
|
|
bcc: json.bcc,
|
|
|
|
references,
|
|
|
|
inReplyTo,
|
|
|
|
replyTo: json.reply_to,
|
|
|
|
subject: json.subject,
|
|
|
|
body: json.body,
|
|
|
|
unread: true,
|
|
|
|
isDraft: json.draft,
|
|
|
|
isSent: false,
|
|
|
|
version: 0,
|
2016-12-24 08:15:34 +08:00
|
|
|
date: date,
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
uploads: json.uploads,
|
|
|
|
}
|
2016-12-24 08:15:34 +08:00
|
|
|
// We have to clone the message and change the date for hashing because the
|
|
|
|
// date we get later when we parse from IMAP is a different format, per the
|
|
|
|
// nodemailer buildmail function that gives us the raw message and replaces
|
|
|
|
// the date header with this modified UTC string
|
|
|
|
// https://github.com/nodemailer/buildmail/blob/master/lib/buildmail.js#L470
|
|
|
|
const messageForHashing = Utils.deepClone(message)
|
|
|
|
messageForHashing.date = date.toUTCString().replace(/GMT/, '+0000')
|
|
|
|
message.id = Message.hash(messageForHashing)
|
2016-12-17 06:39:50 +08:00
|
|
|
message.body = replaceMessageIdInBodyTrackingLinks(message.id, message.body)
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
return Message.build(message)
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
2016-11-30 03:18:51 +08:00
|
|
|
module.exports = {
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
buildForSend,
|
2016-11-30 03:18:51 +08:00
|
|
|
parseFromImap,
|
2016-12-14 04:42:38 +08:00
|
|
|
extractSnippet,
|
2016-12-30 02:56:00 +08:00
|
|
|
extractContacts,
|
2016-12-17 06:39:50 +08:00
|
|
|
stripTrackingLinksFromBody,
|
|
|
|
buildTrackingBodyForRecipient,
|
|
|
|
replaceMessageIdInBodyTrackingLinks,
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|