nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2024-12-24 08:51:05 +08:00
Code Issues Projects Releases Packages Wiki Activity

Started separating mimeTree from larger attachments

Browse source
This commit is contained in:
Andris Reinman 2017-03-10 21:03:59 +02:00
parent cae2ad9e95
commit 9cac2be895
11 changed files with 91 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -110,7 +110,7 @@ After you have created an user you can use these credentials to log in to the IM
Create an email account and use your IMAP client to connect to it. To send mail to this account, run the example script:
```
node examples/push.mail.js username@example.com
node examples/push-mail.js username@example.com
```
This should "deliver" a new message to the INBOX of *username@example.com* by using the built-in SMTP maildrop interface. If your email client is connected then you should promptly see the new message.

5
examples/push-message.js
View file

@ -21,5 +21,8 @@ transporter.sendMail({
to: recipient,
subject: 'Test message [' + Date.now() + ']',
text: 'Hello world! Current time is ' + new Date().toString(),
html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>'
html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>',
attachments: [{
path: __dirname + '/swan.jpg'
}]
});

BIN
examples/swan.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 575 KiB

10
imap-core/lib/imap-tools.js
View file

@ -492,10 +492,14 @@ module.exports.getQueryResponse = function (query, message, options) {
break;
case 'bodystructure':
if (!mimeTree) {
mimeTree = indexer.parseMimeTree(message.raw);
if (message.envelope) {
value = message.envelope;
} else {
if (!mimeTree) {
mimeTree = indexer.parseMimeTree(message.raw);
}
value = indexer.getBodyStructure(mimeTree);
}
value = indexer.getBodyStructure(mimeTree);
break;
case 'envelope':

121
imap-core/lib/indexer/indexer.js
View file

@ -2,16 +2,17 @@
'use strict';
let stream = require('stream');
let PassThrough = stream.PassThrough;
const stream = require('stream');
const PassThrough = stream.PassThrough;
let BodyStructure = require('./body-structure');
let createEnvelope = require('./create-envelope');
let parseMimeTree = require('./parse-mime-tree');
let fetch = require('nodemailer-fetch');
let libbase64 = require('libbase64');
let util = require('util');
let LengthLimiter = require('../length-limiter');
const BodyStructure = require('./body-structure');
const createEnvelope = require('./create-envelope');
const parseMimeTree = require('./parse-mime-tree');
const LengthLimiter = require('../length-limiter');
// const ObjectID = require('mongodb').ObjectID;
const GridFs = require('grid-fs');
// TODO: store large attachments to GridStore
class Indexer {
@ -19,6 +20,9 @@ class Indexer {
this.options = options || {};
this.fetchOptions = this.options.fetchOptions || {};
this.database = this.options.database;
this.gridstore = new GridFs(this.database, 'attachments');
// create logger
this.logger = this.options.logger || {
info: () => false,
@ -53,7 +57,7 @@ class Indexer {
let walk = (node, next) => {
if (!textOnly || !root) {
append(filterHeaders(node.header).join('\r\n') + '\r\n');
append(formatHeaders(node.header).join('\r\n') + '\r\n');
}
let finalize = () => {
@ -67,32 +71,29 @@ class Indexer {
root = false;
if (node.body || node.parsedHeader['x-attachment-stream-url']) {
if (node.body || node.attachmentId) {
append(false, true); // force newline
size += node.size;
}
if (node.boundary) {
append('--' + node.boundary);
} else if (node.parsedHeader['x-attachment-stream-url']) {
return finalize();
}
let pos = 0;
let processChildNodes = () => {
if (pos >= node.childNodes.length) {
return finalize();
}
let childNode = node.childNodes[pos++];
walk(childNode, () => {
if (pos < node.childNodes.length) {
append('--' + node.boundary);
}
return processChildNodes();
});
};
if (Array.isArray(node.childNodes)) {
let pos = 0;
let processChildNodes = () => {
if (pos >= node.childNodes.length) {
return finalize();
}
let childNode = node.childNodes[pos++];
walk(childNode, () => {
if (pos < node.childNodes.length) {
append('--' + node.boundary);
}
return processChildNodes();
});
};
processChildNodes();
} else {
finalize();
@ -132,7 +133,7 @@ class Indexer {
let walk = (node, next) => {
if (!textOnly || !root) {
append(filterHeaders(node.header).join('\r\n') + '\r\n');
append(formatHeaders(node.header).join('\r\n') + '\r\n');
}
root = false;
@ -150,58 +151,25 @@ class Indexer {
if (node.boundary) {
append('--' + node.boundary);
} else if (node.parsedHeader['x-attachment-stream-url']) {
let streamUrl = node.parsedHeader['x-attachment-stream-url'].replace(/^<|>$/g, '');
let streamEncoded = /^\s*YES\s*$/i.test(node.parsedHeader['x-attachment-stream-encoded']);
} else if (node.attachmentId) {
append(false, true); // force newline between header and contents
let headers = {};
if (this.fetchOptions.userAgent) {
headers['User-Agent'] = this.fetchOptions.userAgent;
}
if (this.fetchOptions.cookies) {
headers.Cookie = this.fetchOptions.cookies.get(streamUrl);
}
this.logger.debug('Fetching <%s>\nHeaders: %s', streamUrl, util.inspect(headers, false, 22));
let limiter = new LengthLimiter(node.size);
let attachmentStream = this.gridstore.createReadStream(node.attachmentId);
let fetchStream = fetch(streamUrl, {
userAgent: this.fetchOptions.userAgent,
maxRedirects: this.fetchOptions.maxRedirects,
cookies: this.fetchOptions.cookies,
timeout: 60 * 1000 // timeout after one minute of inactivity
});
fetchStream.on('error', err => {
attachmentStream.once('error', err => {
res.emit('error', err);
});
limiter.on('error', err => {
limiter.once('error', err => {
res.emit('error', err);
});
limiter.on('end', () => finalize());
if (!streamEncoded) {
let b64encoder = new libbase64.Encoder();
b64encoder.on('error', err => {
res.emit('error', err);
});
// encode stream as base64
fetchStream.pipe(b64encoder).pipe(limiter).pipe(res, {
end: false
});
} else {
// already encoded, pipe directly to output
fetchStream.pipe(limiter).pipe(res, {
end: false
});
}
limiter.once('end', () => finalize());
attachmentStream.pipe(limiter).pipe(res, {
end: false
});
return;
}
@ -406,7 +374,7 @@ class Indexer {
case 'header':
if (!selector.path) {
// BODY[HEADER] mail header
return filterHeaders(node.header).join('\r\n') + '\r\n\r\n';
return formatHeaders(node.header).join('\r\n') + '\r\n\r\n';
} else if (node.message) {
// BODY[1.2.3.HEADER] embedded message/rfc822 header
return (node.message.header || []).join('\r\n') + '\r\n\r\n';
@ -418,7 +386,7 @@ class Indexer {
if (!selector.headers || !selector.headers.length) {
return '\r\n\r\n';
}
return filterHeaders(node.header).filter(line => {
return formatHeaders(node.header).filter(line => {
let key = line.split(':').shift().toLowerCase().trim();
return selector.headers.indexOf(key) >= 0;
}).join('\r\n') + '\r\n\r\n';
@ -426,16 +394,16 @@ class Indexer {
case 'header.fields.not':
// BODY[HEADER.FIELDS.NOT (Key1 Key2 KeyN)] all but selected header keys
if (!selector.headers || !selector.headers.length) {
return filterHeaders(node.header).join('\r\n') + '\r\n\r\n';
return formatHeaders(node.header).join('\r\n') + '\r\n\r\n';
}
return filterHeaders(node.header).filter(line => {
return formatHeaders(node.header).filter(line => {
let key = line.split(':').shift().toLowerCase().trim();
return selector.headers.indexOf(key) < 0;
}).join('\r\n') + '\r\n\r\n';
case 'mime':
// BODY[1.2.3.MIME] mime node header
return filterHeaders(node.header).join('\r\n') + '\r\n\r\n';
return formatHeaders(node.header).join('\r\n') + '\r\n\r\n';
case 'text':
if (!selector.path) {
@ -451,15 +419,14 @@ class Indexer {
return '';
}
}
}
function filterHeaders(headers) {
function formatHeaders(headers) {
headers = headers || [];
if (!Array.isArray(headers)) {
headers = [].concat(headers || []);
}
return headers.filter(header => !/^X-Attachment-Stream/i.test(header));
return headers;
}
module.exports = Indexer;

41
imap-core/lib/indexer/parse-mime-tree.js
View file

@ -110,10 +110,10 @@ class MIMEParser {
if (node.body) {
let lineCount = node.body.length;
node.body = node.body.join('').
// ensure proper line endings
// ensure proper line endings
replace(/\r?\n/g, '\r\n');
node.size = this.getNodeSize(node);
node.lineCount = this.getLineCount(node, lineCount);
node.size = (node.body || '').length;
node.lineCount = lineCount;
}
node.childNodes.forEach(walker);
@ -128,41 +128,6 @@ class MIMEParser {
walker(this.tree);
}
getNodeSize(node) {
let bodyLength = (node.body || '').length;
let streamSize = 0;
let streamEncoded = /^\s*YES\s*$/i.test(node.parsedHeader['x-attachment-stream-encoded']);
if (node.parsedHeader['x-attachment-stream-url']) {
streamSize = Number(node.parsedHeader['x-attachment-stream-size']) || 0;
if (!streamEncoded) {
// stream needs base64 encoding, calculate post-encoded size
streamSize = Math.ceil(streamSize / 3 * 4); // convert to base64 length
if (streamSize % 4) {
// add base64 padding
streamSize += (4 - (streamSize % 4));
}
streamSize += Math.floor(streamSize / 76) * 2; // add newlines
}
}
return streamSize + bodyLength;
}
getLineCount(node, lineCount) {
if (node.parsedHeader['x-attachment-stream-lines']) {
// use pre-calculated line count
return Math.max(lineCount - 1, 0) + (Number(node.parsedHeader['x-attachment-stream-lines']) || 0);
} else if (node.parsedHeader['x-attachment-stream-url']) {
// calculate line count for standard base64 encoded content
let streamSize = this.getNodeSize(node);
return Math.max(lineCount - 1, 0) + Math.ceil(streamSize / 78);
}
return lineCount;
}
/**
* Creates a new node with default values for the parse tree
*/

5
imap-core/lib/search.js
View file

@ -169,11 +169,6 @@ let queryHandlers = {
parts = headers[i].split(':');
key = (parts.shift() || '').trim().toLowerCase();
if (/^X-Attachment-Stream/i.test(key)) {
// skip special headers
continue;
}
value = (parts.join(':') || '');
if (key === header && (!term || value.toLowerCase().indexOf(term) >= 0)) {

7
imap-core/test/imap-indexer-test.js
View file

@ -6,14 +6,14 @@
let chai = require('chai');
let expect = chai.expect;
let http = require('http');
//let http = require('http');
let fs = require('fs');
let Indexer = require('../lib/indexer/indexer');
let indexer = new Indexer();
chai.config.includeStack = true;
const HTTP_PORT = 9998;
//const HTTP_PORT = 9998;
let fixtures = {
simple: {
@ -44,6 +44,7 @@ describe('#parseMimeTree', function () {
});
});
/*
describe('#rebuild', function () {
let httpServer;
@ -67,6 +68,7 @@ describe('#rebuild', function () {
httpServer.close(done);
});
it('should rebuild using stream', function (done) {
let message = `Content-Type: multipart/mixed;
boundary="foo"
@ -230,3 +232,4 @@ X-Attachment-Stream-Encoded: Yes
});
});
});
*/

35
imap-core/test/test-server.js
View file

@ -564,20 +564,31 @@ module.exports = function (options) {
let folder = folders.get(mailbox);
let highestModseq = 0;
let uidList = folder.messages.filter(message => {
let match = session.matchSearchQuery(message, options.query);
if (match && highestModseq < message.modseq) {
highestModseq = message.modseq;
let uidList = [];
let checked = 0;
let checkNext = () => {
if (checked >= folder.messages.length) {
return callback(null, {
uidList,
highestModseq
});
}
return match;
}).map(message => message.uid);
callback(null, {
uidList,
highestModseq
});
let message = folder.messages[checked++];
session.matchSearchQuery(message, options.query, (err, match) => {
if (err) {
// ignore
}
if (match && highestModseq < message.modseq) {
highestModseq = message.modseq;
}
if (match) {
uidList.push(message.uid);
}
checkNext();
});
};
checkNext();
};
return server;
};

5
imap.js
View file

@ -718,7 +718,8 @@ server.onFetch = function (path, options, session, callback) {
internaldate: true,
flags: true,
envelope: true,
bodystructure: true
bodystructure: true,
size: true
};
if (!options.metadataOnly) {
@ -1126,7 +1127,7 @@ server.addToMailbox = (username, path, meta, date, flags, raw, callback) => {
headerdate,
flags,
unseen: !flags.includes('\\Seen'),
size: raw.length,
size: server.indexer.getSize(mimeTree),
meta,
modseq: 0,
mimeTree,

1
package.json
View file

@ -24,6 +24,7 @@
"bcryptjs": "^2.4.3",
"clone": "^2.1.1",
"config": "^1.25.1",
"grid-fs": "^1.0.1",
"joi": "^10.2.2",
"libbase64": "^0.1.0",
"mailparser": "^2.0.2",