Hello world! Current time is ' + new Date().toString() + '
' + html: 'Hello world! Current time is ' + new Date().toString() + '
', + attachments: [{ + path: __dirname + '/swan.jpg' + }] }); diff --git a/examples/swan.jpg b/examples/swan.jpg new file mode 100644 index 00000000..c957f692 Binary files /dev/null and b/examples/swan.jpg differ diff --git a/imap-core/lib/imap-tools.js b/imap-core/lib/imap-tools.js index c5282691..79192c6b 100644 --- a/imap-core/lib/imap-tools.js +++ b/imap-core/lib/imap-tools.js @@ -492,10 +492,14 @@ module.exports.getQueryResponse = function (query, message, options) { break; case 'bodystructure': - if (!mimeTree) { - mimeTree = indexer.parseMimeTree(message.raw); + if (message.envelope) { + value = message.envelope; + } else { + if (!mimeTree) { + mimeTree = indexer.parseMimeTree(message.raw); + } + value = indexer.getBodyStructure(mimeTree); } - value = indexer.getBodyStructure(mimeTree); break; case 'envelope': diff --git a/imap-core/lib/indexer/indexer.js b/imap-core/lib/indexer/indexer.js index 90e769a1..72d71e42 100644 --- a/imap-core/lib/indexer/indexer.js +++ b/imap-core/lib/indexer/indexer.js @@ -2,16 +2,17 @@ 'use strict'; -let stream = require('stream'); -let PassThrough = stream.PassThrough; +const stream = require('stream'); +const PassThrough = stream.PassThrough; -let BodyStructure = require('./body-structure'); -let createEnvelope = require('./create-envelope'); -let parseMimeTree = require('./parse-mime-tree'); -let fetch = require('nodemailer-fetch'); -let libbase64 = require('libbase64'); -let util = require('util'); -let LengthLimiter = require('../length-limiter'); +const BodyStructure = require('./body-structure'); +const createEnvelope = require('./create-envelope'); +const parseMimeTree = require('./parse-mime-tree'); +const LengthLimiter = require('../length-limiter'); +// const ObjectID = require('mongodb').ObjectID; +const GridFs = require('grid-fs'); + +// TODO: store large attachments to GridStore class Indexer { @@ -19,6 +20,9 @@ class Indexer { this.options = options || {}; this.fetchOptions = this.options.fetchOptions || {}; + this.database = this.options.database; + this.gridstore = new GridFs(this.database, 'attachments'); + // create logger this.logger = this.options.logger || { info: () => false, @@ -53,7 +57,7 @@ class Indexer { let walk = (node, next) => { if (!textOnly || !root) { - append(filterHeaders(node.header).join('\r\n') + '\r\n'); + append(formatHeaders(node.header).join('\r\n') + '\r\n'); } let finalize = () => { @@ -67,32 +71,29 @@ class Indexer { root = false; - if (node.body || node.parsedHeader['x-attachment-stream-url']) { + if (node.body || node.attachmentId) { append(false, true); // force newline size += node.size; } if (node.boundary) { append('--' + node.boundary); - } else if (node.parsedHeader['x-attachment-stream-url']) { - return finalize(); } - let pos = 0; - let processChildNodes = () => { - if (pos >= node.childNodes.length) { - return finalize(); - } - let childNode = node.childNodes[pos++]; - walk(childNode, () => { - if (pos < node.childNodes.length) { - append('--' + node.boundary); - } - return processChildNodes(); - }); - }; - if (Array.isArray(node.childNodes)) { + let pos = 0; + let processChildNodes = () => { + if (pos >= node.childNodes.length) { + return finalize(); + } + let childNode = node.childNodes[pos++]; + walk(childNode, () => { + if (pos < node.childNodes.length) { + append('--' + node.boundary); + } + return processChildNodes(); + }); + }; processChildNodes(); } else { finalize(); @@ -132,7 +133,7 @@ class Indexer { let walk = (node, next) => { if (!textOnly || !root) { - append(filterHeaders(node.header).join('\r\n') + '\r\n'); + append(formatHeaders(node.header).join('\r\n') + '\r\n'); } root = false; @@ -150,58 +151,25 @@ class Indexer { if (node.boundary) { append('--' + node.boundary); - } else if (node.parsedHeader['x-attachment-stream-url']) { - let streamUrl = node.parsedHeader['x-attachment-stream-url'].replace(/^<|>$/g, ''); - let streamEncoded = /^\s*YES\s*$/i.test(node.parsedHeader['x-attachment-stream-encoded']); - + } else if (node.attachmentId) { append(false, true); // force newline between header and contents - let headers = {}; - if (this.fetchOptions.userAgent) { - headers['User-Agent'] = this.fetchOptions.userAgent; - } - - if (this.fetchOptions.cookies) { - headers.Cookie = this.fetchOptions.cookies.get(streamUrl); - } - - this.logger.debug('Fetching <%s>\nHeaders: %s', streamUrl, util.inspect(headers, false, 22)); - let limiter = new LengthLimiter(node.size); + let attachmentStream = this.gridstore.createReadStream(node.attachmentId); - let fetchStream = fetch(streamUrl, { - userAgent: this.fetchOptions.userAgent, - maxRedirects: this.fetchOptions.maxRedirects, - cookies: this.fetchOptions.cookies, - timeout: 60 * 1000 // timeout after one minute of inactivity - }); - - fetchStream.on('error', err => { + attachmentStream.once('error', err => { res.emit('error', err); }); - limiter.on('error', err => { + limiter.once('error', err => { res.emit('error', err); }); - limiter.on('end', () => finalize()); - - if (!streamEncoded) { - let b64encoder = new libbase64.Encoder(); - b64encoder.on('error', err => { - res.emit('error', err); - }); - // encode stream as base64 - fetchStream.pipe(b64encoder).pipe(limiter).pipe(res, { - end: false - }); - } else { - // already encoded, pipe directly to output - fetchStream.pipe(limiter).pipe(res, { - end: false - }); - } + limiter.once('end', () => finalize()); + attachmentStream.pipe(limiter).pipe(res, { + end: false + }); return; } @@ -406,7 +374,7 @@ class Indexer { case 'header': if (!selector.path) { // BODY[HEADER] mail header - return filterHeaders(node.header).join('\r\n') + '\r\n\r\n'; + return formatHeaders(node.header).join('\r\n') + '\r\n\r\n'; } else if (node.message) { // BODY[1.2.3.HEADER] embedded message/rfc822 header return (node.message.header || []).join('\r\n') + '\r\n\r\n'; @@ -418,7 +386,7 @@ class Indexer { if (!selector.headers || !selector.headers.length) { return '\r\n\r\n'; } - return filterHeaders(node.header).filter(line => { + return formatHeaders(node.header).filter(line => { let key = line.split(':').shift().toLowerCase().trim(); return selector.headers.indexOf(key) >= 0; }).join('\r\n') + '\r\n\r\n'; @@ -426,16 +394,16 @@ class Indexer { case 'header.fields.not': // BODY[HEADER.FIELDS.NOT (Key1 Key2 KeyN)] all but selected header keys if (!selector.headers || !selector.headers.length) { - return filterHeaders(node.header).join('\r\n') + '\r\n\r\n'; + return formatHeaders(node.header).join('\r\n') + '\r\n\r\n'; } - return filterHeaders(node.header).filter(line => { + return formatHeaders(node.header).filter(line => { let key = line.split(':').shift().toLowerCase().trim(); return selector.headers.indexOf(key) < 0; }).join('\r\n') + '\r\n\r\n'; case 'mime': // BODY[1.2.3.MIME] mime node header - return filterHeaders(node.header).join('\r\n') + '\r\n\r\n'; + return formatHeaders(node.header).join('\r\n') + '\r\n\r\n'; case 'text': if (!selector.path) { @@ -451,15 +419,14 @@ class Indexer { return ''; } } - } -function filterHeaders(headers) { +function formatHeaders(headers) { headers = headers || []; if (!Array.isArray(headers)) { headers = [].concat(headers || []); } - return headers.filter(header => !/^X-Attachment-Stream/i.test(header)); + return headers; } module.exports = Indexer; diff --git a/imap-core/lib/indexer/parse-mime-tree.js b/imap-core/lib/indexer/parse-mime-tree.js index 4ac4378b..401e6d80 100644 --- a/imap-core/lib/indexer/parse-mime-tree.js +++ b/imap-core/lib/indexer/parse-mime-tree.js @@ -110,10 +110,10 @@ class MIMEParser { if (node.body) { let lineCount = node.body.length; node.body = node.body.join(''). - // ensure proper line endings + // ensure proper line endings replace(/\r?\n/g, '\r\n'); - node.size = this.getNodeSize(node); - node.lineCount = this.getLineCount(node, lineCount); + node.size = (node.body || '').length; + node.lineCount = lineCount; } node.childNodes.forEach(walker); @@ -128,41 +128,6 @@ class MIMEParser { walker(this.tree); } - getNodeSize(node) { - let bodyLength = (node.body || '').length; - let streamSize = 0; - let streamEncoded = /^\s*YES\s*$/i.test(node.parsedHeader['x-attachment-stream-encoded']); - - if (node.parsedHeader['x-attachment-stream-url']) { - streamSize = Number(node.parsedHeader['x-attachment-stream-size']) || 0; - - if (!streamEncoded) { - // stream needs base64 encoding, calculate post-encoded size - streamSize = Math.ceil(streamSize / 3 * 4); // convert to base64 length - if (streamSize % 4) { - // add base64 padding - streamSize += (4 - (streamSize % 4)); - } - streamSize += Math.floor(streamSize / 76) * 2; // add newlines - } - } - - return streamSize + bodyLength; - } - - getLineCount(node, lineCount) { - if (node.parsedHeader['x-attachment-stream-lines']) { - // use pre-calculated line count - return Math.max(lineCount - 1, 0) + (Number(node.parsedHeader['x-attachment-stream-lines']) || 0); - } else if (node.parsedHeader['x-attachment-stream-url']) { - // calculate line count for standard base64 encoded content - let streamSize = this.getNodeSize(node); - return Math.max(lineCount - 1, 0) + Math.ceil(streamSize / 78); - } - - return lineCount; - } - /** * Creates a new node with default values for the parse tree */ diff --git a/imap-core/lib/search.js b/imap-core/lib/search.js index 95c47b3b..9f7400bf 100644 --- a/imap-core/lib/search.js +++ b/imap-core/lib/search.js @@ -169,11 +169,6 @@ let queryHandlers = { parts = headers[i].split(':'); key = (parts.shift() || '').trim().toLowerCase(); - if (/^X-Attachment-Stream/i.test(key)) { - // skip special headers - continue; - } - value = (parts.join(':') || ''); if (key === header && (!term || value.toLowerCase().indexOf(term) >= 0)) { diff --git a/imap-core/test/imap-indexer-test.js b/imap-core/test/imap-indexer-test.js index def56080..6775c6d9 100644 --- a/imap-core/test/imap-indexer-test.js +++ b/imap-core/test/imap-indexer-test.js @@ -6,14 +6,14 @@ let chai = require('chai'); let expect = chai.expect; -let http = require('http'); +//let http = require('http'); let fs = require('fs'); let Indexer = require('../lib/indexer/indexer'); let indexer = new Indexer(); chai.config.includeStack = true; -const HTTP_PORT = 9998; +//const HTTP_PORT = 9998; let fixtures = { simple: { @@ -44,6 +44,7 @@ describe('#parseMimeTree', function () { }); }); +/* describe('#rebuild', function () { let httpServer; @@ -67,6 +68,7 @@ describe('#rebuild', function () { httpServer.close(done); }); + it('should rebuild using stream', function (done) { let message = `Content-Type: multipart/mixed; boundary="foo" @@ -230,3 +232,4 @@ X-Attachment-Stream-Encoded: Yes }); }); }); +*/ diff --git a/imap-core/test/test-server.js b/imap-core/test/test-server.js index 8f08d0af..093e6667 100644 --- a/imap-core/test/test-server.js +++ b/imap-core/test/test-server.js @@ -564,20 +564,31 @@ module.exports = function (options) { let folder = folders.get(mailbox); let highestModseq = 0; - let uidList = folder.messages.filter(message => { - let match = session.matchSearchQuery(message, options.query); - if (match && highestModseq < message.modseq) { - highestModseq = message.modseq; + let uidList = []; + let checked = 0; + let checkNext = () => { + if (checked >= folder.messages.length) { + return callback(null, { + uidList, + highestModseq + }); } - return match; - }).map(message => message.uid); - - callback(null, { - uidList, - highestModseq - }); + let message = folder.messages[checked++]; + session.matchSearchQuery(message, options.query, (err, match) => { + if (err) { + // ignore + } + if (match && highestModseq < message.modseq) { + highestModseq = message.modseq; + } + if (match) { + uidList.push(message.uid); + } + checkNext(); + }); + }; + checkNext(); }; - return server; }; diff --git a/imap.js b/imap.js index 4a87b919..e590e812 100644 --- a/imap.js +++ b/imap.js @@ -718,7 +718,8 @@ server.onFetch = function (path, options, session, callback) { internaldate: true, flags: true, envelope: true, - bodystructure: true + bodystructure: true, + size: true }; if (!options.metadataOnly) { @@ -1126,7 +1127,7 @@ server.addToMailbox = (username, path, meta, date, flags, raw, callback) => { headerdate, flags, unseen: !flags.includes('\\Seen'), - size: raw.length, + size: server.indexer.getSize(mimeTree), meta, modseq: 0, mimeTree, diff --git a/package.json b/package.json index 5b260c2e..69ec0d30 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "bcryptjs": "^2.4.3", "clone": "^2.1.1", "config": "^1.25.1", + "grid-fs": "^1.0.1", "joi": "^10.2.2", "libbase64": "^0.1.0", "mailparser": "^2.0.2",