[local-sync] Add specs for message parsing

Summary:
This commit also fixes snippets for HTML-only messages to strip out HTML
tags, and makes us preserve whitespace for plaintext emails by
displaying them in <pre class="nylas-plaintext"> tags, and makes us log
messages that fail to parse at all to a tempdir.

The only issue I found with using <pre> tags for plaintext email was
that some lines may trigger scrolling, so there is an associated commit
(D3484) that changes the CSS for <pre class="nylas-plaintext"> to wrap
lines.

In the future, we can add regression tests to this test suite whenever
we fix parsing bugs.

Test Plan: unit tests included

Reviewers: bengotow

Reviewed By: bengotow

Differential Revision: https://phab.nylas.com/D3483

packages/local-sync/package.json

@@ -23,7 +23,8 @@
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz",
     "striptags": "2.1.1",
     "underscore": "1.8.3",
-    "utf7": "1.0.2",
+    "utf7": "^1.0.2",
+    "striptags": "2.1.1",
     "vision": "4.1.0"
   },
   "scripts": {

packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json

@@ -1,5 +0,0 @@
-{
-  "imapMessage": {},
-  "desiredParts": {},
-  "result": {}
-}

packages/local-sync/spec/message-factory-spec.js

@@ -26,10 +26,17 @@ describe('MessageFactory', function MessageFactorySpecs() {
     forEachJSONFixture('MessageFactory/parseFromImap', (filename, json) => {
       it(`should correctly build message properties for ${filename}`, () => {
         const {imapMessage, desiredParts, result} = json;
+        // requiring these to match makes it overly arduous to generate test
+        // cases from real accounts
+        const excludeKeys = new Set(['id', 'accountId', 'folderId', 'folder', 'labels']);
 
         waitsForPromise(async () => {
           const actual = await parseFromImap(imapMessage, desiredParts, this.options);
-          expect(actual).toEqual(result)
+          for (const key of Object.keys(result)) {
+            if (!excludeKeys.has(key)) {
+              expect(actual[key]).toEqual(result[key]);
+            }
+          }
         });
       });
     })

packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js

@@ -1,4 +1,7 @@
 const _ = require('underscore');
+const os = require('os');
+const fs = require('fs');
+const mkdirp = require('mkdirp');
 
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core');
 const {Capabilities} = IMAPConnection;
@@ -265,6 +268,11 @@ class FetchMessagesInFolder {
         imapMessage,
         desiredParts,
       }, `FetchMessagesInFolder: Could not build message`)
+      const outJSON = JSON.stringify({'imapMessage': imapMessage, 'desiredParts': desiredParts, 'result': {}});
+      const outDir = path.join(os.tmpdir(), "k2-parse-errors", this._folder.name)
+      const outFile = path.join(outDir, imapMessage.attributes.uid.toString());
+      mkdirp.sync(outDir);
+      fs.writeFileSync(outFile, outJSON);
     }
   }
 

packages/local-sync/src/shared/message-factory.js

@@ -1,7 +1,9 @@
 const utf7 = require('utf7').imap;
 const mimelib = require('mimelib');
 const QuotedPrintable = require('quoted-printable');
-const {Imap} = require('isomorphic-core')
+const {Imap} = require('isomorphic-core');
+const mkdirp = require('mkdirp');
+const striptags = require('striptags');
 
 const SNIPPET_SIZE = 100
 
@@ -59,6 +61,13 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
     subject: parsedHeaders.subject[0],
   }
 
+  // preserve whitespacing on plaintext emails -- has the side effect of monospacing, but
+  // that seems OK and perhaps sometimes even desired (for e.g. ascii art, alignment)
+  if (!body['text/html'] && body['text/plain']) {
+    values.body = `<pre class="nylas-plaintext">${values.body}</pre>`;
+  }
+
+  // TODO: strip quoted text from snippets also
   if (values.snippet) {
     // trim and clean snippet which is alreay present (from values plaintext)
     values.snippet = values.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ')
@@ -68,8 +77,7 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
     }
   } else if (values.body) {
     // create snippet from body, which is most likely html
-    // TODO: Fanciness
-    values.snippet = values.body.substr(0, Math.min(values.body.length, SNIPPET_SIZE));
+    values.snippet = striptags(values.body).trim().substr(0, Math.min(values.body.length, SNIPPET_SIZE));
   }
 
   values.folder = folder