2017-03-22 05:07:45 +08:00
/ *
2017-04-06 02:17:19 +08:00
const { parseFromImap , parseSnippet , parseContacts } = require ( '../src/message-utils' ) ;
2017-03-22 05:07:45 +08:00
const { forEachJSONFixture , forEachHTMLAndTXTFixture , ACCOUNT _ID , getTestDatabase } = require ( './helpers' ) ;
2016-12-06 04:16:53 +08:00
2017-04-06 02:17:19 +08:00
xdescribe ( 'MessageUtils' , function MessageUtilsSpecs ( ) {
2016-12-06 04:16:53 +08:00
beforeEach ( ( ) => {
waitsForPromise ( async ( ) => {
2017-02-03 06:00:06 +08:00
const db = await getTestDatabase ( )
2016-12-06 04:16:53 +08:00
const folder = await db . Folder . create ( {
id : 'test-folder-id' ,
2016-12-06 08:07:46 +08:00
accountId : ACCOUNT _ID ,
2016-12-06 04:16:53 +08:00
version : 1 ,
name : 'Test Folder' ,
role : null ,
} ) ;
2016-12-06 08:07:46 +08:00
this . options = { accountId : ACCOUNT _ID , db , folder } ;
2016-12-06 04:16:53 +08:00
} )
} )
2016-12-06 08:07:46 +08:00
describe ( "parseFromImap" , ( ) => {
2017-04-06 02:17:19 +08:00
forEachJSONFixture ( 'MessageUtils/parseFromImap' , ( filename , json ) => {
2016-12-06 04:16:53 +08:00
it ( ` should correctly build message properties for ${ filename } ` , ( ) => {
2016-12-06 08:07:46 +08:00
const { imapMessage , desiredParts , result } = json ;
2016-12-07 03:19:39 +08:00
// requiring these to match makes it overly arduous to generate test
// cases from real accounts
const excludeKeys = new Set ( [ 'id' , 'accountId' , 'folderId' , 'folder' , 'labels' ] ) ;
2016-12-06 04:16:53 +08:00
waitsForPromise ( async ( ) => {
const actual = await parseFromImap ( imapMessage , desiredParts , this . options ) ;
2016-12-07 03:19:39 +08:00
for ( const key of Object . keys ( result ) ) {
if ( ! excludeKeys . has ( key ) ) {
expect ( actual [ key ] ) . toEqual ( result [ key ] ) ;
}
}
2016-12-06 04:16:53 +08:00
} ) ;
} ) ;
} )
} ) ;
} ) ;
2016-12-14 04:42:38 +08:00
const snippetTestCases = [ {
purpose : 'trim whitespace in basic plaintext' ,
2017-01-10 23:56:01 +08:00
body : '<pre>The quick brown fox\n\n\tjumps over the lazy</pre>' ,
2016-12-14 04:42:38 +08:00
snippet : 'The quick brown fox jumps over the lazy' ,
} , {
purpose : 'truncate long plaintext without breaking words' ,
2017-01-10 23:56:01 +08:00
body : '<pre>The quick brown fox jumps over the lazy dog and then the lazy dog rolls over and sighs. The fox turns around in a circle and then jumps onto a bush! It grins wickedly and wags its fat tail. As the lazy dog puts its head on its paws and cracks a sleepy eye open, a slow grin forms on its face. The fox has fallen into the bush and is yelping and squeaking.</pre>' ,
2016-12-14 04:42:38 +08:00
snippet : 'The quick brown fox jumps over the lazy dog and then the lazy dog rolls over and sighs. The fox turns' ,
} , {
2017-01-10 23:56:01 +08:00
purpose : 'process basic HTML correctly' ,
body : '<html><title>All About Ponies</title><h1>PONIES AND RAINBOWS AND UNICORNS</h1><p>Unicorns are native to the hillsides of Flatagonia.</p></html>' ,
2016-12-14 04:42:38 +08:00
snippet : 'PONIES AND RAINBOWS AND UNICORNS Unicorns are native to the hillsides of Flatagonia.' ,
} , {
purpose : 'properly strip rogue styling inside of <body> and trim whitespace in HTML' ,
2017-01-10 23:56:01 +08:00
body : '<html>\n <head></head>\n <body>\n <style>\n body { width: 100% !important; min-width: 100%; -webkit-font-smoothing: antialiased; -webkit-text-size-adjust: 100%; -ms-text-size-adjust: 100%; margin: 0; padding: 0; background: #fafafa;\n </style>\n <p>Look ma, no CSS!</p></body></html>' ,
2016-12-14 04:42:38 +08:00
snippet : 'Look ma, no CSS!' ,
} , {
purpose : 'properly process <br/> and <div/>' ,
2017-01-10 23:56:01 +08:00
body : '<p>Unicorns are <div>native</div>to the<br/>hillsides of<br/>Flatagonia.</p>' ,
2016-12-14 04:42:38 +08:00
snippet : 'Unicorns are native to the hillsides of Flatagonia.' ,
} , {
purpose : 'properly strip out HTML comments' ,
2017-01-10 23:56:01 +08:00
body : '<p>Unicorns are<!-- an HTML comment! -->native to the</p>' ,
2016-12-14 04:42:38 +08:00
snippet : 'Unicorns are native to the' ,
2016-12-23 09:08:18 +08:00
} , {
purpose : "don't add extraneous spaces after text format markup" ,
2017-01-10 23:56:01 +08:00
body : `
2016-12-23 09:08:18 +08:00
< td style = "padding: 0px 10px" >
Hey there , < b > Nylas < / b > ! < b r >
You have a new follower on Product Hunt .
< / t d > ` ,
snippet : 'Hey there, Nylas! You have a new follower on Product Hunt.' ,
2016-12-14 04:42:38 +08:00
} ,
]
2016-12-30 02:56:00 +08:00
const contactsTestCases = [ {
purpose : "not erroneously split contact names on commas" ,
// NOTE: inputs must be in same format as output by mimelib.parseHeader
input : [ '"Little Bo Peep, The Hill" <bopeep@example.com>' ] ,
output : [ { name : "Little Bo Peep, The Hill" , email : "bopeep@example.com" } ] ,
} , {
purpose : "extract two separate contacts, removing quotes properly & respecing unicode" ,
input : [ 'AppleBees Zé <a@example.com>, "Tiger Zen" b@example.com' ] ,
output : [
{ name : 'AppleBees Zé' , email : 'a@example.com' } ,
{ name : 'Tiger Zen' , email : 'b@example.com' } ,
] ,
2017-01-11 04:04:58 +08:00
} , {
purpose : "correctly concatenate multiple array elements (from multiple header lines)" ,
input : [ 'Yubi Key <yubi@example.com>' , 'Smokey the Bear <smokey@example.com>' ] ,
output : [
{ name : 'Yubi Key' , email : 'yubi@example.com' } ,
{ name : 'Smokey the Bear' , email : 'smokey@example.com' } ,
] ,
2016-12-30 02:56:00 +08:00
} ,
]
2017-04-06 02:17:19 +08:00
describe ( 'MessageUtilsHelpers' , function MessageUtilsHelperSpecs ( ) {
[local-sync] For generic IMAP, Thread based on Message-Id, In-Reply-To & References
Summary:
This swaps out our generic IMAP threading mechanism to use the threading
headers on the message instead of the prior way of grouping by subject
and then differentiating based on participants, as that design was
somewhat driven by what we could accomplish easily given legacy data
schema decisions and has serious caveats, such as different threads between
the same people with the same subject being misthreaded together. With K2, we
have free reign to change the data format, so we can do it right.
The algorithm is super simple:
- Define "references" as the union of the Message-Id, In-Reply-To, and
References headers on a message, filtered for valid RFC2822 Message-IDs
- On message sync, if any element of the new message's references
matches any element of an existing message's references, thread them
together
In order to accomplish this, we need to store References in a way that
allows each element to be indexed for fast lookup. That meant either
using the sqlite JSON1 extension + expression-based indices, or creating
a new table. I chose the latter as a time-tested and simple solution,
since we don't need the flexibility of JSON here.
Test Plan: manual - unit tests coming
Reviewers: khamidou, evan, juan
Reviewed By: evan, juan
Differential Revision: https://phab.nylas.com/D3651
2017-01-08 06:31:28 +08:00
describe ( 'parseSnippet (basic)' , ( ) => {
2017-01-10 23:56:01 +08:00
snippetTestCases . forEach ( ( { purpose , body , snippet } ) => {
2016-12-14 04:42:38 +08:00
it ( ` should ${ purpose } ` , ( ) => {
[local-sync] For generic IMAP, Thread based on Message-Id, In-Reply-To & References
Summary:
This swaps out our generic IMAP threading mechanism to use the threading
headers on the message instead of the prior way of grouping by subject
and then differentiating based on participants, as that design was
somewhat driven by what we could accomplish easily given legacy data
schema decisions and has serious caveats, such as different threads between
the same people with the same subject being misthreaded together. With K2, we
have free reign to change the data format, so we can do it right.
The algorithm is super simple:
- Define "references" as the union of the Message-Id, In-Reply-To, and
References headers on a message, filtered for valid RFC2822 Message-IDs
- On message sync, if any element of the new message's references
matches any element of an existing message's references, thread them
together
In order to accomplish this, we need to store References in a way that
allows each element to be indexed for fast lookup. That meant either
using the sqlite JSON1 extension + expression-based indices, or creating
a new table. I chose the latter as a time-tested and simple solution,
since we don't need the flexibility of JSON here.
Test Plan: manual - unit tests coming
Reviewers: khamidou, evan, juan
Reviewed By: evan, juan
Differential Revision: https://phab.nylas.com/D3651
2017-01-08 06:31:28 +08:00
const parsedSnippet = parseSnippet ( body ) ;
2016-12-14 04:42:38 +08:00
expect ( parsedSnippet ) . toEqual ( snippet ) ;
} ) ;
} ) ;
} ) ;
[local-sync] For generic IMAP, Thread based on Message-Id, In-Reply-To & References
Summary:
This swaps out our generic IMAP threading mechanism to use the threading
headers on the message instead of the prior way of grouping by subject
and then differentiating based on participants, as that design was
somewhat driven by what we could accomplish easily given legacy data
schema decisions and has serious caveats, such as different threads between
the same people with the same subject being misthreaded together. With K2, we
have free reign to change the data format, so we can do it right.
The algorithm is super simple:
- Define "references" as the union of the Message-Id, In-Reply-To, and
References headers on a message, filtered for valid RFC2822 Message-IDs
- On message sync, if any element of the new message's references
matches any element of an existing message's references, thread them
together
In order to accomplish this, we need to store References in a way that
allows each element to be indexed for fast lookup. That meant either
using the sqlite JSON1 extension + expression-based indices, or creating
a new table. I chose the latter as a time-tested and simple solution,
since we don't need the flexibility of JSON here.
Test Plan: manual - unit tests coming
Reviewers: khamidou, evan, juan
Reviewed By: evan, juan
Differential Revision: https://phab.nylas.com/D3651
2017-01-08 06:31:28 +08:00
describe ( 'parseSnippet (real world)' , ( ) => {
2017-04-06 02:17:19 +08:00
forEachHTMLAndTXTFixture ( 'MessageUtils/parseSnippet' , ( filename , html , txt ) => {
2016-12-14 04:42:38 +08:00
it ( ` should correctly extract the snippet from the html ` , ( ) => {
[local-sync] For generic IMAP, Thread based on Message-Id, In-Reply-To & References
Summary:
This swaps out our generic IMAP threading mechanism to use the threading
headers on the message instead of the prior way of grouping by subject
and then differentiating based on participants, as that design was
somewhat driven by what we could accomplish easily given legacy data
schema decisions and has serious caveats, such as different threads between
the same people with the same subject being misthreaded together. With K2, we
have free reign to change the data format, so we can do it right.
The algorithm is super simple:
- Define "references" as the union of the Message-Id, In-Reply-To, and
References headers on a message, filtered for valid RFC2822 Message-IDs
- On message sync, if any element of the new message's references
matches any element of an existing message's references, thread them
together
In order to accomplish this, we need to store References in a way that
allows each element to be indexed for fast lookup. That meant either
using the sqlite JSON1 extension + expression-based indices, or creating
a new table. I chose the latter as a time-tested and simple solution,
since we don't need the flexibility of JSON here.
Test Plan: manual - unit tests coming
Reviewers: khamidou, evan, juan
Reviewed By: evan, juan
Differential Revision: https://phab.nylas.com/D3651
2017-01-08 06:31:28 +08:00
const parsedSnippet = parseSnippet ( html ) ;
2016-12-14 04:42:38 +08:00
expect ( parsedSnippet ) . toEqual ( txt ) ;
} ) ;
} ) ;
} ) ;
[local-sync] For generic IMAP, Thread based on Message-Id, In-Reply-To & References
Summary:
This swaps out our generic IMAP threading mechanism to use the threading
headers on the message instead of the prior way of grouping by subject
and then differentiating based on participants, as that design was
somewhat driven by what we could accomplish easily given legacy data
schema decisions and has serious caveats, such as different threads between
the same people with the same subject being misthreaded together. With K2, we
have free reign to change the data format, so we can do it right.
The algorithm is super simple:
- Define "references" as the union of the Message-Id, In-Reply-To, and
References headers on a message, filtered for valid RFC2822 Message-IDs
- On message sync, if any element of the new message's references
matches any element of an existing message's references, thread them
together
In order to accomplish this, we need to store References in a way that
allows each element to be indexed for fast lookup. That meant either
using the sqlite JSON1 extension + expression-based indices, or creating
a new table. I chose the latter as a time-tested and simple solution,
since we don't need the flexibility of JSON here.
Test Plan: manual - unit tests coming
Reviewers: khamidou, evan, juan
Reviewed By: evan, juan
Differential Revision: https://phab.nylas.com/D3651
2017-01-08 06:31:28 +08:00
describe ( 'parseContacts (basic)' , ( ) => {
2016-12-30 02:56:00 +08:00
contactsTestCases . forEach ( ( { purpose , input , output } ) => {
it ( ` should ${ purpose } ` , ( ) => {
[local-sync] For generic IMAP, Thread based on Message-Id, In-Reply-To & References
Summary:
This swaps out our generic IMAP threading mechanism to use the threading
headers on the message instead of the prior way of grouping by subject
and then differentiating based on participants, as that design was
somewhat driven by what we could accomplish easily given legacy data
schema decisions and has serious caveats, such as different threads between
the same people with the same subject being misthreaded together. With K2, we
have free reign to change the data format, so we can do it right.
The algorithm is super simple:
- Define "references" as the union of the Message-Id, In-Reply-To, and
References headers on a message, filtered for valid RFC2822 Message-IDs
- On message sync, if any element of the new message's references
matches any element of an existing message's references, thread them
together
In order to accomplish this, we need to store References in a way that
allows each element to be indexed for fast lookup. That meant either
using the sqlite JSON1 extension + expression-based indices, or creating
a new table. I chose the latter as a time-tested and simple solution,
since we don't need the flexibility of JSON here.
Test Plan: manual - unit tests coming
Reviewers: khamidou, evan, juan
Reviewed By: evan, juan
Differential Revision: https://phab.nylas.com/D3651
2017-01-08 06:31:28 +08:00
const parsedContacts = parseContacts ( input ) ;
2016-12-30 02:56:00 +08:00
expect ( parsedContacts ) . toEqual ( output ) ;
} ) ;
} ) ;
} ) ;
2016-12-14 04:42:38 +08:00
} ) ;
2017-03-22 05:07:45 +08:00
* /
