Started separating mimeTree from larger attachments

README.md

@@ -110,7 +110,7 @@ After you have created an user you can use these credentials to log in to the IM
 Create an email account and use your IMAP client to connect to it. To send mail to this account, run the example script:
 
 ```
-node examples/push.mail.js username@example.com
+node examples/push-mail.js username@example.com
 ```
 
 This should "deliver" a new message to the INBOX of *username@example.com* by using the built-in SMTP maildrop interface. If your email client is connected then you should promptly see the new message.

examples/push-message.js

@@ -21,5 +21,8 @@ transporter.sendMail({
     to: recipient,
     subject: 'Test message [' + Date.now() + ']',
     text: 'Hello world! Current time is ' + new Date().toString(),
-    html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>'
+    html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>',
+    attachments: [{
+        path: __dirname + '/swan.jpg'
+    }]
 });

imap-core/lib/imap-tools.js

@@ -492,10 +492,14 @@ module.exports.getQueryResponse = function (query, message, options) {
                 break;
 
             case 'bodystructure':
-                if (!mimeTree) {
+                if (message.envelope) {
-                    mimeTree = indexer.parseMimeTree(message.raw);
+                    value = message.envelope;
+                } else {
+                    if (!mimeTree) {
+                        mimeTree = indexer.parseMimeTree(message.raw);
+                    }
+                    value = indexer.getBodyStructure(mimeTree);
                 }
-                value = indexer.getBodyStructure(mimeTree);
                 break;
 
             case 'envelope':

imap-core/lib/indexer/indexer.js

@@ -2,16 +2,17 @@
 
 'use strict';
 
-let stream = require('stream');
+const stream = require('stream');
-let PassThrough = stream.PassThrough;
+const PassThrough = stream.PassThrough;
 
-let BodyStructure = require('./body-structure');
+const BodyStructure = require('./body-structure');
-let createEnvelope = require('./create-envelope');
+const createEnvelope = require('./create-envelope');
-let parseMimeTree = require('./parse-mime-tree');
+const parseMimeTree = require('./parse-mime-tree');
-let fetch = require('nodemailer-fetch');
+const LengthLimiter = require('../length-limiter');
-let libbase64 = require('libbase64');
+// const ObjectID = require('mongodb').ObjectID;
-let util = require('util');
+const GridFs = require('grid-fs');
-let LengthLimiter = require('../length-limiter');
+
+// TODO: store large attachments to GridStore
 
 class Indexer {
 
@@ -19,6 +20,9 @@ class Indexer {
         this.options = options || {};
         this.fetchOptions = this.options.fetchOptions || {};
 
+        this.database = this.options.database;
+        this.gridstore = new GridFs(this.database, 'attachments');
+
         // create logger
         this.logger = this.options.logger || {
             info: () => false,
@@ -53,7 +57,7 @@ class Indexer {
         let walk = (node, next) => {
 
             if (!textOnly || !root) {
-                append(filterHeaders(node.header).join('\r\n') + '\r\n');
+                append(formatHeaders(node.header).join('\r\n') + '\r\n');
             }
 
             let finalize = () => {
@@ -67,32 +71,29 @@ class Indexer {
 
             root = false;
 
-            if (node.body || node.parsedHeader['x-attachment-stream-url']) {
+            if (node.body || node.attachmentId) {
                 append(false, true); // force newline
                 size += node.size;
             }
 
             if (node.boundary) {
                 append('--' + node.boundary);
-            } else if (node.parsedHeader['x-attachment-stream-url']) {
-                return finalize();
             }
 
-            let pos = 0;
-            let processChildNodes = () => {
-                if (pos >= node.childNodes.length) {
-                    return finalize();
-                }
-                let childNode = node.childNodes[pos++];
-                walk(childNode, () => {
-                    if (pos < node.childNodes.length) {
-                        append('--' + node.boundary);
-                    }
-                    return processChildNodes();
-                });
-            };
-
             if (Array.isArray(node.childNodes)) {
+                let pos = 0;
+                let processChildNodes = () => {
+                    if (pos >= node.childNodes.length) {
+                        return finalize();
+                    }
+                    let childNode = node.childNodes[pos++];
+                    walk(childNode, () => {
+                        if (pos < node.childNodes.length) {
+                            append('--' + node.boundary);
+                        }
+                        return processChildNodes();
+                    });
+                };
                 processChildNodes();
             } else {
                 finalize();
@@ -132,7 +133,7 @@ class Indexer {
         let walk = (node, next) => {
 
             if (!textOnly || !root) {
-                append(filterHeaders(node.header).join('\r\n') + '\r\n');
+                append(formatHeaders(node.header).join('\r\n') + '\r\n');
             }
 
             root = false;
@@ -150,58 +151,25 @@ class Indexer {
 
             if (node.boundary) {
                 append('--' + node.boundary);
-            } else if (node.parsedHeader['x-attachment-stream-url']) {
+            } else if (node.attachmentId) {
-                let streamUrl = node.parsedHeader['x-attachment-stream-url'].replace(/^<|>$/g, '');
-                let streamEncoded = /^\s*YES\s*$/i.test(node.parsedHeader['x-attachment-stream-encoded']);
-
                 append(false, true); // force newline between header and contents
 
-                let headers = {};
-                if (this.fetchOptions.userAgent) {
-                    headers['User-Agent'] = this.fetchOptions.userAgent;
-                }
-
-                if (this.fetchOptions.cookies) {
-                    headers.Cookie = this.fetchOptions.cookies.get(streamUrl);
-                }
-
-                this.logger.debug('Fetching <%s>\nHeaders: %s', streamUrl, util.inspect(headers, false, 22));
-
                 let limiter = new LengthLimiter(node.size);
+                let attachmentStream = this.gridstore.createReadStream(node.attachmentId);
 
-                let fetchStream = fetch(streamUrl, {
+                attachmentStream.once('error', err => {
-                    userAgent: this.fetchOptions.userAgent,
-                    maxRedirects: this.fetchOptions.maxRedirects,
-                    cookies: this.fetchOptions.cookies,
-                    timeout: 60 * 1000 // timeout after one minute of inactivity
-                });
-
-                fetchStream.on('error', err => {
                     res.emit('error', err);
                 });
 
-                limiter.on('error', err => {
+                limiter.once('error', err => {
                     res.emit('error', err);
                 });
 
-                limiter.on('end', () => finalize());
+                limiter.once('end', () => finalize());
-
-                if (!streamEncoded) {
-                    let b64encoder = new libbase64.Encoder();
-                    b64encoder.on('error', err => {
-                        res.emit('error', err);
-                    });
-                    // encode stream as base64
-                    fetchStream.pipe(b64encoder).pipe(limiter).pipe(res, {
-                        end: false
-                    });
-                } else {
-                    // already encoded, pipe directly to output
-                    fetchStream.pipe(limiter).pipe(res, {
-                        end: false
-                    });
-                }
 
+                attachmentStream.pipe(limiter).pipe(res, {
+                    end: false
+                });
                 return;
             }
 
@@ -406,7 +374,7 @@ class Indexer {
             case 'header':
                 if (!selector.path) {
                     // BODY[HEADER] mail header
-                    return filterHeaders(node.header).join('\r\n') + '\r\n\r\n';
+                    return formatHeaders(node.header).join('\r\n') + '\r\n\r\n';
                 } else if (node.message) {
                     // BODY[1.2.3.HEADER] embedded message/rfc822 header
                     return (node.message.header || []).join('\r\n') + '\r\n\r\n';
@@ -418,7 +386,7 @@ class Indexer {
                 if (!selector.headers || !selector.headers.length) {
                     return '\r\n\r\n';
                 }
-                return filterHeaders(node.header).filter(line => {
+                return formatHeaders(node.header).filter(line => {
                     let key = line.split(':').shift().toLowerCase().trim();
                     return selector.headers.indexOf(key) >= 0;
                 }).join('\r\n') + '\r\n\r\n';
@@ -426,16 +394,16 @@ class Indexer {
             case 'header.fields.not':
                 // BODY[HEADER.FIELDS.NOT (Key1 Key2 KeyN)] all but selected header keys
                 if (!selector.headers || !selector.headers.length) {
-                    return filterHeaders(node.header).join('\r\n') + '\r\n\r\n';
+                    return formatHeaders(node.header).join('\r\n') + '\r\n\r\n';
                 }
-                return filterHeaders(node.header).filter(line => {
+                return formatHeaders(node.header).filter(line => {
                     let key = line.split(':').shift().toLowerCase().trim();
                     return selector.headers.indexOf(key) < 0;
                 }).join('\r\n') + '\r\n\r\n';
 
             case 'mime':
                 // BODY[1.2.3.MIME] mime node header
-                return filterHeaders(node.header).join('\r\n') + '\r\n\r\n';
+                return formatHeaders(node.header).join('\r\n') + '\r\n\r\n';
 
             case 'text':
                 if (!selector.path) {
@@ -451,15 +419,14 @@ class Indexer {
                 return '';
         }
     }
-
 }
 
-function filterHeaders(headers) {
+function formatHeaders(headers) {
     headers = headers || [];
     if (!Array.isArray(headers)) {
         headers = [].concat(headers || []);
     }
-    return headers.filter(header => !/^X-Attachment-Stream/i.test(header));
+    return headers;
 }
 
 module.exports = Indexer;

imap-core/lib/indexer/parse-mime-tree.js

@@ -110,10 +110,10 @@ class MIMEParser {
             if (node.body) {
                 let lineCount = node.body.length;
                 node.body = node.body.join('').
-                    // ensure proper line endings
+                // ensure proper line endings
                 replace(/\r?\n/g, '\r\n');
-                node.size = this.getNodeSize(node);
+                node.size = (node.body || '').length;
-                node.lineCount = this.getLineCount(node, lineCount);
+                node.lineCount = lineCount;
             }
             node.childNodes.forEach(walker);
 
@@ -128,41 +128,6 @@ class MIMEParser {
         walker(this.tree);
     }
 
-    getNodeSize(node) {
-        let bodyLength = (node.body || '').length;
-        let streamSize = 0;
-        let streamEncoded = /^\s*YES\s*$/i.test(node.parsedHeader['x-attachment-stream-encoded']);
-
-        if (node.parsedHeader['x-attachment-stream-url']) {
-            streamSize = Number(node.parsedHeader['x-attachment-stream-size']) || 0;
-
-            if (!streamEncoded) {
-                // stream needs base64 encoding, calculate post-encoded size
-                streamSize = Math.ceil(streamSize / 3 * 4); // convert to base64 length
-                if (streamSize % 4) {
-                    // add base64 padding
-                    streamSize += (4 - (streamSize % 4));
-                }
-                streamSize += Math.floor(streamSize / 76) * 2; // add newlines
-            }
-        }
-
-        return streamSize + bodyLength;
-    }
-
-    getLineCount(node, lineCount) {
-        if (node.parsedHeader['x-attachment-stream-lines']) {
-            // use pre-calculated line count
-            return Math.max(lineCount - 1, 0) + (Number(node.parsedHeader['x-attachment-stream-lines']) || 0);
-        } else if (node.parsedHeader['x-attachment-stream-url']) {
-            // calculate line count for standard base64 encoded content
-            let streamSize = this.getNodeSize(node);
-            return Math.max(lineCount - 1, 0) + Math.ceil(streamSize / 78);
-        }
-
-        return lineCount;
-    }
-
     /**
      * Creates a new node with default values for the parse tree
      */

imap-core/lib/search.js

@@ -169,11 +169,6 @@ let queryHandlers = {
             parts = headers[i].split(':');
             key = (parts.shift() || '').trim().toLowerCase();
 
-            if (/^X-Attachment-Stream/i.test(key)) {
-                // skip special headers
-                continue;
-            }
-
             value = (parts.join(':') || '');
 
             if (key === header && (!term || value.toLowerCase().indexOf(term) >= 0)) {

imap-core/test/imap-indexer-test.js

@@ -6,14 +6,14 @@
 let chai = require('chai');
 let expect = chai.expect;
 
-let http = require('http');
+//let http = require('http');
 let fs = require('fs');
 let Indexer = require('../lib/indexer/indexer');
 let indexer = new Indexer();
 
 chai.config.includeStack = true;
 
-const HTTP_PORT = 9998;
+//const HTTP_PORT = 9998;
 
 let fixtures = {
     simple: {
@@ -44,6 +44,7 @@ describe('#parseMimeTree', function () {
     });
 });
 
+/*
 describe('#rebuild', function () {
     let httpServer;
 
@@ -67,6 +68,7 @@ describe('#rebuild', function () {
         httpServer.close(done);
     });
 
+
     it('should rebuild using stream', function (done) {
         let message = `Content-Type: multipart/mixed;
  boundary="foo"
@@ -230,3 +232,4 @@ X-Attachment-Stream-Encoded: Yes
         });
     });
 });
+*/

imap-core/test/test-server.js

@@ -564,20 +564,31 @@ module.exports = function (options) {
         let folder = folders.get(mailbox);
         let highestModseq = 0;
 
-        let uidList = folder.messages.filter(message => {
+        let uidList = [];
-            let match = session.matchSearchQuery(message, options.query);
+        let checked = 0;
-            if (match && highestModseq < message.modseq) {
+        let checkNext = () => {
-                highestModseq = message.modseq;
+            if (checked >= folder.messages.length) {
+                return callback(null, {
+                    uidList,
+                    highestModseq
+                });
             }
-            return match;
+            let message = folder.messages[checked++];
-        }).map(message => message.uid);
+            session.matchSearchQuery(message, options.query, (err, match) => {
-
+                if (err) {
-        callback(null, {
+                    // ignore
-            uidList,
+                }
-            highestModseq
+                if (match && highestModseq < message.modseq) {
-        });
+                    highestModseq = message.modseq;
+                }
+                if (match) {
+                    uidList.push(message.uid);
+                }
+                checkNext();
+            });
+        };
+        checkNext();
     };
 
-
     return server;
 };

imap.js

@@ -718,7 +718,8 @@ server.onFetch = function (path, options, session, callback) {
             internaldate: true,
             flags: true,
             envelope: true,
-            bodystructure: true
+            bodystructure: true,
+            size: true
         };
 
         if (!options.metadataOnly) {
@@ -1126,7 +1127,7 @@ server.addToMailbox = (username, path, meta, date, flags, raw, callback) => {
                     headerdate,
                     flags,
                     unseen: !flags.includes('\\Seen'),
-                    size: raw.length,
+                    size: server.indexer.getSize(mimeTree),
                     meta,
                     modseq: 0,
                     mimeTree,

package.json

@@ -24,6 +24,7 @@
     "bcryptjs": "^2.4.3",
     "clone": "^2.1.1",
     "config": "^1.25.1",
+    "grid-fs": "^1.0.1",
     "joi": "^10.2.2",
     "libbase64": "^0.1.0",
     "mailparser": "^2.0.2",