[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
/* eslint no-useless-escape: 0 */
|
2016-11-30 03:18:51 +08:00
|
|
|
const mimelib = require('mimelib');
|
2016-12-09 10:30:57 +08:00
|
|
|
const encoding = require('encoding');
|
2016-12-16 01:17:45 +08:00
|
|
|
const he = require('he');
|
2016-12-16 02:18:41 +08:00
|
|
|
const os = require('os');
|
|
|
|
|
const fs = require('fs');
|
|
|
|
|
const path = require('path')
|
|
|
|
|
const mkdirp = require('mkdirp');
|
2016-12-09 09:48:34 +08:00
|
|
|
const {Imap} = require('isomorphic-core');
|
2016-12-15 15:46:36 +08:00
|
|
|
const {APIError} = require('./errors')
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-09 10:30:57 +08:00
|
|
|
// aiming for the former in length, but the latter is the hard db cutoff
|
|
|
|
|
const SNIPPET_SIZE = 100;
|
|
|
|
|
const SNIPPET_MAX_SIZE = 255;
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-13 05:33:34 +08:00
|
|
|
// The input is the value of a to/cc/bcc/from header as parsed by the imap
|
|
|
|
|
// library we're using, but it currently parses them in a weird format. If an
|
|
|
|
|
// email is sent to a@example.com and b@example.com, the parsed output of the
|
|
|
|
|
// 'to' header is ['a@example.com, b@example.com']. (Note both emails are in
|
|
|
|
|
// the same string.) When fixed, this function will need to update accordingly.
|
|
|
|
|
function extractContacts(input) {
|
|
|
|
|
if (!input || input.length === 0 || !input[0]) {
|
|
|
|
|
return [];
|
|
|
|
|
}
|
|
|
|
|
const s = `["${input[0].replace(/"/g, '\\"').replace(/, /g, '", "')}"]`;
|
|
|
|
|
const values = JSON.parse(s);
|
2016-11-30 03:18:51 +08:00
|
|
|
return values.map(v => {
|
|
|
|
|
const {name, address: email} = mimelib.parseAddresses(v).pop()
|
|
|
|
|
return {name, email}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
2016-12-14 04:42:38 +08:00
|
|
|
/*
|
|
|
|
|
Iteratively walk the DOM of this document's <body>, calling the callback on
|
|
|
|
|
each node. Skip any nodes and the skipTags set, including their children.
|
|
|
|
|
*/
|
|
|
|
|
function _walkBodyDOM(doc, callback, skipTags) {
|
|
|
|
|
let nodes = Array.from(doc.body.childNodes);
|
|
|
|
|
|
|
|
|
|
while (nodes.length) {
|
|
|
|
|
const node = nodes.shift();
|
|
|
|
|
|
|
|
|
|
callback(node);
|
|
|
|
|
|
|
|
|
|
if (!skipTags.has(node.tagName)) {
|
|
|
|
|
if (node.childNodes && node.childNodes.length) {
|
|
|
|
|
nodes = Array.from(node.childNodes).concat(nodes);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function extractSnippet(plainBody, htmlBody) {
|
|
|
|
|
let snippetText = plainBody || '';
|
|
|
|
|
if (htmlBody) {
|
|
|
|
|
const doc = new DOMParser().parseFromString(htmlBody, 'text/html')
|
|
|
|
|
const extractedTextElements = [];
|
|
|
|
|
|
|
|
|
|
_walkBodyDOM(doc, (node) => {
|
|
|
|
|
if (node.nodeType === Node.TEXT_NODE) {
|
|
|
|
|
const nodeValue = node.nodeValue ? node.nodeValue.trim() : null;
|
|
|
|
|
if (nodeValue) {
|
|
|
|
|
extractedTextElements.push(nodeValue);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}, new Set(['TITLE', 'SCRIPT', 'STYLE', 'IMG']));
|
|
|
|
|
|
|
|
|
|
const extractedText = extractedTextElements.join(' ').trim();
|
|
|
|
|
if (extractedText) {
|
|
|
|
|
snippetText = extractedText;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// clean up and trim snippet
|
|
|
|
|
let trimmed = snippetText.trim().replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ').substr(0, SNIPPET_MAX_SIZE);
|
|
|
|
|
if (trimmed) {
|
|
|
|
|
// TODO: strip quoted text from snippets also
|
|
|
|
|
// trim down to approx. SNIPPET_SIZE w/out cutting off words right in the
|
|
|
|
|
// middle (if possible)
|
|
|
|
|
const wordBreak = trimmed.indexOf(' ', SNIPPET_SIZE);
|
|
|
|
|
if (wordBreak !== -1) {
|
|
|
|
|
trimmed = trimmed.substr(0, wordBreak);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return trimmed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
Preserve whitespacing on plaintext emails -- has the side effect of
|
|
|
|
|
monospacing, but that seems OK and perhaps sometimes even desired (for e.g.
|
|
|
|
|
ascii art, alignment)
|
|
|
|
|
*/
|
|
|
|
|
function HTMLifyPlaintext(text) {
|
2016-12-16 01:17:45 +08:00
|
|
|
const escapedText = he.escape(text);
|
|
|
|
|
return `<pre class="nylas-plaintext">${escapedText}</pre>`;
|
2016-12-14 04:42:38 +08:00
|
|
|
}
|
|
|
|
|
|
2016-12-09 10:30:57 +08:00
|
|
|
/*
|
|
|
|
|
Since we only fetch the MIME structure and specific desired MIME parts from
|
|
|
|
|
IMAP, we unfortunately can't use an existing library like mailparser to parse
|
|
|
|
|
the message, and have to do fun stuff like deal with character sets and
|
|
|
|
|
content-transfer-encodings ourselves.
|
|
|
|
|
*/
|
2016-12-06 04:16:53 +08:00
|
|
|
async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder}) {
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
const {Message, Label} = db
|
2016-12-09 10:30:57 +08:00
|
|
|
const {attributes} = imapMessage
|
|
|
|
|
|
2016-11-30 03:18:51 +08:00
|
|
|
const body = {}
|
2016-12-09 10:30:57 +08:00
|
|
|
for (const {id, mimetype, transferEncoding, charset} of desiredParts) {
|
|
|
|
|
// see https://www.w3.org/Protocols/rfc1341/5_Content-Transfer-Encoding.html
|
|
|
|
|
if (!transferEncoding || new Set(['7bit', '8bit']).has(transferEncoding.toLowerCase())) {
|
|
|
|
|
// NO transfer encoding has been performed --- how to decode to a string
|
|
|
|
|
// depends ONLY on the charset, which defaults to 'ascii' according to
|
|
|
|
|
// https://tools.ietf.org/html/rfc2045#section-5.2
|
|
|
|
|
const convertedBuffer = encoding.convert(imapMessage.parts[id], 'utf-8', charset || 'ascii')
|
|
|
|
|
body[mimetype] = convertedBuffer.toString('utf-8');
|
|
|
|
|
} else if (transferEncoding.toLowerCase() === 'quoted-printable') {
|
|
|
|
|
body[mimetype] = mimelib.decodeQuotedPrintable(imapMessage.parts[id], charset || 'ascii');
|
|
|
|
|
} else if (transferEncoding.toLowerCase() === 'base64') {
|
|
|
|
|
body[mimetype] = mimelib.decodeBase64(imapMessage.parts[id], charset || 'ascii');
|
2016-11-30 03:18:51 +08:00
|
|
|
} else {
|
2016-12-09 10:30:57 +08:00
|
|
|
// 'binary' and custom x-token content-transfer-encodings
|
|
|
|
|
return Promise.reject(new Error(`Unsupported Content-Transfer-Encoding ${transferEncoding}, mimetype ${mimetype}`))
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
|
}
|
2016-12-09 10:30:57 +08:00
|
|
|
const headers = imapMessage.headers.toString('ascii');
|
2016-11-30 03:18:51 +08:00
|
|
|
const parsedHeaders = Imap.parseHeader(headers);
|
|
|
|
|
for (const key of ['x-gm-thrid', 'x-gm-msgid', 'x-gm-labels']) {
|
|
|
|
|
parsedHeaders[key] = attributes[key];
|
|
|
|
|
}
|
|
|
|
|
|
2016-12-09 10:30:57 +08:00
|
|
|
const parsedMessage = {
|
2016-11-30 03:18:51 +08:00
|
|
|
to: extractContacts(parsedHeaders.to),
|
|
|
|
|
cc: extractContacts(parsedHeaders.cc),
|
|
|
|
|
bcc: extractContacts(parsedHeaders.bcc),
|
|
|
|
|
from: extractContacts(parsedHeaders.from),
|
|
|
|
|
replyTo: extractContacts(parsedHeaders['reply-to']),
|
|
|
|
|
accountId: accountId,
|
|
|
|
|
body: body['text/html'] || body['text/plain'] || body['application/pgp-encrypted'] || '',
|
2016-12-09 10:30:57 +08:00
|
|
|
snippet: null,
|
2016-11-30 03:18:51 +08:00
|
|
|
unread: !attributes.flags.includes('\\Seen'),
|
|
|
|
|
starred: attributes.flags.includes('\\Flagged'),
|
|
|
|
|
date: attributes.date,
|
|
|
|
|
folderImapUID: attributes.uid,
|
2016-12-06 04:16:53 +08:00
|
|
|
folderId: folder.id,
|
2016-11-30 03:18:51 +08:00
|
|
|
folder: null,
|
|
|
|
|
labels: [],
|
|
|
|
|
headers: parsedHeaders,
|
|
|
|
|
headerMessageId: parsedHeaders['message-id'] ? parsedHeaders['message-id'][0] : '',
|
2016-12-08 02:22:31 +08:00
|
|
|
gMsgId: parsedHeaders['x-gm-msgid'],
|
2016-11-30 03:18:51 +08:00
|
|
|
subject: parsedHeaders.subject[0],
|
|
|
|
|
}
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
parsedMessage.id = Message.hash(parsedMessage)
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-07 03:19:39 +08:00
|
|
|
if (!body['text/html'] && body['text/plain']) {
|
2016-12-14 04:42:38 +08:00
|
|
|
parsedMessage.body = HTMLifyPlaintext(body['text/plain']);
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
|
|
2016-12-14 04:42:38 +08:00
|
|
|
parsedMessage.snippet = extractSnippet(body['text/plain'], body['text/html']);
|
2016-12-09 10:30:57 +08:00
|
|
|
parsedMessage.folder = folder
|
|
|
|
|
|
|
|
|
|
// TODO: unclear if this is necessary given we already have parsed labels
|
|
|
|
|
const xGmLabels = attributes['x-gm-labels']
|
2016-12-06 04:16:53 +08:00
|
|
|
if (xGmLabels) {
|
2016-12-09 10:30:57 +08:00
|
|
|
parsedMessage.folderImapXGMLabels = JSON.stringify(xGmLabels)
|
|
|
|
|
parsedMessage.labels = await Label.findXGMLabels(xGmLabels)
|
2016-12-06 04:16:53 +08:00
|
|
|
}
|
2016-11-30 03:18:51 +08:00
|
|
|
|
2016-12-16 02:18:41 +08:00
|
|
|
if (process.env.NYLAS_DEBUG) {
|
|
|
|
|
const outJSON = JSON.stringify({imapMessage, desiredParts, result: parsedMessage});
|
|
|
|
|
const outDir = path.join(os.tmpdir(), "k2-parse-output", folder.name)
|
|
|
|
|
const outFile = path.join(outDir, imapMessage.attributes.uid.toString());
|
|
|
|
|
mkdirp.sync(outDir);
|
|
|
|
|
fs.writeFileSync(outFile, outJSON);
|
|
|
|
|
}
|
|
|
|
|
|
2016-12-09 10:30:57 +08:00
|
|
|
return parsedMessage;
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
|
|
|
|
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
function getReplyHeaders(messageReplyingTo) {
|
|
|
|
|
let inReplyTo;
|
|
|
|
|
let references;
|
|
|
|
|
if (messageReplyingTo.headerMessageId) {
|
|
|
|
|
inReplyTo = messageReplyingTo.headerMessageId;
|
|
|
|
|
if (messageReplyingTo.references) {
|
|
|
|
|
references = messageReplyingTo.references.concat(messageReplyingTo.headerMessageId);
|
|
|
|
|
} else {
|
|
|
|
|
references = [messageReplyingTo.headerMessageId];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return {inReplyTo, references}
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
function replaceBodyMessageIds(messageId, originalBody) {
|
|
|
|
|
const env = NylasEnv.config.get('env')
|
|
|
|
|
const serverUrl = {
|
|
|
|
|
local: 'http:\/\/lvh\.me:5100',
|
|
|
|
|
development: 'http:\/\/lvh\.me:5100',
|
|
|
|
|
staging: 'https:\/\/n1-staging\.nylas\.com',
|
|
|
|
|
production: 'https:\/\/n1\.nylas\.com',
|
|
|
|
|
}[env];
|
|
|
|
|
const regex = new RegExp(`(${serverUrl}.+?)MESSAGE_ID`, 'g')
|
|
|
|
|
return originalBody.replace(regex, `$1${messageId}`)
|
|
|
|
|
}
|
2016-12-09 09:48:34 +08:00
|
|
|
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
async function buildForSend(db, json) {
|
|
|
|
|
const {Thread, Message} = db
|
2016-12-09 09:48:34 +08:00
|
|
|
let replyToThread;
|
|
|
|
|
let replyToMessage;
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
|
|
|
|
|
if (json.thread_id != null) {
|
2016-12-09 09:48:34 +08:00
|
|
|
replyToThread = await Thread.find({
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
where: {id: json.thread_id},
|
2016-12-09 09:48:34 +08:00
|
|
|
include: [{
|
|
|
|
|
model: Message,
|
|
|
|
|
as: 'messages',
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
attributes: ['id'],
|
2016-12-09 09:48:34 +08:00
|
|
|
}],
|
|
|
|
|
});
|
|
|
|
|
}
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
|
|
|
|
|
if (json.reply_to_message_id != null) {
|
|
|
|
|
replyToMessage = await Message.findById(json.reply_to_message_id);
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (replyToThread && replyToMessage) {
|
|
|
|
|
if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
|
2016-12-15 15:46:36 +08:00
|
|
|
throw new APIError(`Message ${replyToMessage.id} is not in thread ${replyToThread.id}`, 400)
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let thread;
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
let replyHeaders = {};
|
2016-12-09 09:48:34 +08:00
|
|
|
if (replyToMessage) {
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
replyHeaders = getReplyHeaders(replyToMessage);
|
|
|
|
|
thread = await replyToMessage.getThread();
|
2016-12-09 09:48:34 +08:00
|
|
|
} else if (replyToThread) {
|
|
|
|
|
thread = replyToThread;
|
|
|
|
|
const previousMessages = thread.messages.filter(msg => !msg.isDraft);
|
|
|
|
|
if (previousMessages.length > 0) {
|
|
|
|
|
const lastMessage = previousMessages[previousMessages.length - 1]
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
replyHeaders = getReplyHeaders(lastMessage);
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
const {inReplyTo, references} = replyHeaders
|
|
|
|
|
const message = {
|
|
|
|
|
accountId: json.account_id,
|
|
|
|
|
threadId: thread ? thread.id : null,
|
|
|
|
|
headerMessageId: Message.buildHeaderMessageId(json.client_id),
|
|
|
|
|
from: json.from,
|
|
|
|
|
to: json.to,
|
|
|
|
|
cc: json.cc,
|
|
|
|
|
bcc: json.bcc,
|
|
|
|
|
references,
|
|
|
|
|
inReplyTo,
|
|
|
|
|
replyTo: json.reply_to,
|
|
|
|
|
subject: json.subject,
|
|
|
|
|
body: json.body,
|
|
|
|
|
unread: true,
|
|
|
|
|
isDraft: json.draft,
|
|
|
|
|
isSent: false,
|
|
|
|
|
version: 0,
|
|
|
|
|
date: new Date(),
|
|
|
|
|
uploads: json.uploads,
|
|
|
|
|
}
|
|
|
|
|
message.id = Message.hash(message)
|
|
|
|
|
message.body = replaceBodyMessageIds(message.id, message.body)
|
|
|
|
|
return Message.build(message)
|
2016-12-09 09:48:34 +08:00
|
|
|
}
|
|
|
|
|
|
2016-11-30 03:18:51 +08:00
|
|
|
module.exports = {
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
buildForSend,
|
2016-11-30 03:18:51 +08:00
|
|
|
parseFromImap,
|
2016-12-14 04:42:38 +08:00
|
|
|
extractSnippet,
|
[local-sync, cloud-api, cloud-workers] Fix msg id collision, tracking and sending issues, some refactoring
Summary:
This diff solves a few separate issues from T7313, T7316, T7282, and it refactors
the send code a little bit.
Initially, the problem that led to this diff was generating message ids that
wouldn't collide (which was causing errors in the message-processor). Collisions
in ids were being caused by messages that contained the exact same participants,
subject and date (most likely due bots or scripts sending emails in quick
succession)
To prevent collisions this commit adds the `message-id` header as part of the
database message id, and ensures that we set it correctly before sending, and
that it remains consistent through send, multi-send, and the sync loop.
During the refactor and review, I removed some code that assumed that we were
syncing drafts (which we aren't), and also fixes a few other known and
unknown issues around sending, message creation, and tracking, like assigning
the correct date header (we were previously assigning the draft creation date
from within N1), fixing the tracking regex, among other smaller bugs/typos.
Will address inline TODOs in a separate diff
Test Plan: TODO!!! I will add tests in another diff
Reviewers: evan, halla, jackie, khamidou
Reviewed By: halla, jackie
Differential Revision: https://phab.nylas.com/D3507
2016-12-15 11:35:19 +08:00
|
|
|
replaceBodyMessageIds,
|
2016-11-30 03:18:51 +08:00
|
|
|
}
|
