nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2025-09-08 14:15:40 +08:00
Code Issues Projects Releases Packages Wiki Activity

Added API method to fetch message content

Browse source
This commit is contained in:
Andris Reinman 2017-04-03 13:39:39 +03:00
parent e7f2fd1445
commit fe9277bc7d
6 changed files with 231 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
api.js
View file

@ -8,6 +8,7 @@ const bcrypt = require('bcryptjs');
const tools = require('./lib/tools');
const MessageHandler = require('./lib/message-handler');
const db = require('./lib/db');
const ObjectID = require('mongodb').ObjectID;
const server = restify.createServer({
name: 'Wild Duck API',
@ -711,7 +712,83 @@ server.del('/message', (req, res, next) => {
});
return next();
});
});
server.get('/message/:id', (req, res, next) => {
res.charSet('utf-8');
const schema = Joi.object().keys({
id: Joi.string().hex().lowercase().length(24).required(),
mailbox: Joi.string().hex().lowercase().length(24).optional()
});
const result = Joi.validate({
id: req.params.id,
mailbox: req.params.mailbox
}, schema, {
abortEarly: false,
convert: true,
allowUnknown: true
});
if (result.error) {
res.json({
error: result.error.message
});
return next();
}
let id = result.value.id;
let mailbox = result.value.mailbox;
let query = {
_id: new ObjectID(id)
};
if (mailbox) {
query.mailbox = new ObjectID(mailbox);
}
db.database.collection('messages').findOne(query, {
mailbox: true,
headers: true,
html: true,
text: true,
attachments: true,
internaldate: true,
flags: true
}, (err, message) => {
if (err) {
res.json({
error: 'MongoDB Error: ' + err.message,
id
});
return next();
}
if (!message) {
res.json({
error: 'This message does not exist',
id
});
return next();
}
res.json({
success: true,
message: {
id,
mailbox: message.mailbox,
headers: message.headers,
date: message.internaldate,
flags: message.flags,
text: message.text,
html: message.html,
attachments: message.attachments
}
});
return next();
});
});
module.exports = done => {

9
examples/push-message.js
View file

@ -32,11 +32,12 @@ function send() {
from: 'andris@kreata.ee',
to: [recipient]
},
from: 'andris@kreata.ee',
to: recipient,
from: 'Kärbes 🐧 <andris@kreata.ee>',
to: 'Ämblik 🦉 <' + recipient + '>, andmekala@hot.ee, Müriaad Polüteism <müriaad@müriaad-polüteism.dev>',
subject: 'Test ööö message [' + Date.now() + ']',
text: 'Hello world! Current time is ' + new Date().toString() + ' <img src="cid:note@example.com"/>',
html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>',
text: 'Hello world! Current time is ' + new Date().toString(),
html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em> <img src="cid:note@example.com"/> <img src="http://www.neti.ee/img/neti-logo-2015-1.png"></p>',
attachments: [
// attachment as plaintext

137
imap-core/lib/indexer/indexer.js
View file

@ -12,6 +12,12 @@ const LengthLimiter = require('../length-limiter');
const ObjectID = require('mongodb').ObjectID;
const GridFs = require('grid-fs');
const libmime = require('libmime');
const libqp = require('libqp');
const libbase64 = require('libbase64');
const iconv = require('iconv-lite');
const marked = require('marked');
const htmlToText = require('html-to-text');
const crypto = require('crypto');
class Indexer {
@ -242,12 +248,24 @@ class Indexer {
/**
* Stores attachments to GridStore, decode text/plain and text/html parts
*/
processContent(messageId, mimeTree, sizeLimit, callback) {
let attachments = [];
processContent(messageId, mimeTree, callback) {
let response = {
attachments: [],
text: '',
html: ''
};
let walk = (node, next) => {
let htmlContent = [];
let textContent = [];
let cidMap = new Map();
let walk = (node, alternative, related, next) => {
let continueProcessing = () => {
if (node.message) {
node = node.message;
}
if (Array.isArray(node.childNodes)) {
let pos = 0;
let processChildNode = () => {
@ -255,7 +273,7 @@ class Indexer {
return next();
}
let childNode = node.childNodes[pos++];
walk(childNode, processChildNode);
walk(childNode, alternative, related, processChildNode);
};
setImmediate(processChildNode);
} else {
@ -263,26 +281,80 @@ class Indexer {
}
};
let flowed = false;
let delSp = false;
let parsedContentType = node.parsedHeader['content-type'];
let parsedDisposition = node.parsedHeader['content-disposition'];
let transferEncoding = (node.parsedHeader['content-transfer-encoding'] || '7bit').toLowerCase().trim();
let contentType = (parsedContentType && parsedContentType.value || (node.rootNode ? 'text/plain' : 'application/octet-stream')).toLowerCase().trim();
let disposition = (parsedDisposition && parsedDisposition.value || '').toLowerCase().trim() || false;
alternative = alternative || contentType === 'multipart/alternative';
related = related || contentType === 'multipart/related';
let curSizeLimit = sizeLimit;
// If the current node is HTML or Plaintext then allow larger content included in the mime tree
// Also decode text value
if (['text/plain', 'text/html'].includes(contentType) && (!disposition || disposition === 'inline')) {
curSizeLimit = Math.max(sizeLimit, 200 * 1024);
if (parsedContentType && parsedContentType.params.format && parsedContentType.params.format.toLowerCase().trim() === 'flowed') {
flowed = true;
if (parsedContentType.params.delsp && parsedContentType.params.delsp.toLowerCase().trim() === 'yes') {
delSp = true;
}
}
if (node.body && node.size > curSizeLimit) {
let disposition = (parsedDisposition && parsedDisposition.value || '').toLowerCase().trim() || false;
// If the current node is HTML or Plaintext then allow larger content included in the mime tree
// Also decode text/html value
if (['text/plain', 'text/html'].includes(contentType) && (!disposition || disposition === 'inline')) {
if (node.body && node.body.length) {
let charset = parsedContentType.params.charset || 'windows-1257';
let content = node.body;
if (transferEncoding === 'base64') {
content = libbase64.decode(content.toString());
} else if (transferEncoding === 'quoted-printable') {
content = libqp.decode(content.toString());
}
if (!['ascii', 'usascii', 'utf8'].includes(charset.replace(/[^a-z0-9]+/g, '').trim().toLowerCase())) {
try {
content = iconv.decode(content, charset);
} catch (E) {
// do not decode charset
}
}
if (flowed) {
content = libmime.decodeFlowed(content.toString(), delSp);
} else {
content = content.toString();
}
if (contentType === 'text/plain') {
textContent.push(content.trim());
if (!alternative) {
htmlContent.push(marked(content, {
breaks: true,
sanitize: true,
gfm: true,
tables: true,
smartypants: true
}).trim());
}
} else if (contentType === 'text/html') {
htmlContent.push(content.trim());
if (!alternative) {
textContent.push(htmlToText.fromString(content).trim());
}
}
}
}
// remove attachments and very large text nodes from the mime tree
if (node.body && (node.size > 300 * 1024 || disposition === 'attachment')) {
let attachmentId = new ObjectID();
let fileName = (node.parsedHeader['content-disposition'] && node.parsedHeader['content-disposition'].params && node.parsedHeader['content-disposition'].params.filename) || (node.parsedHeader['content-type'] && node.parsedHeader['content-type'].params && node.parsedHeader['content-type'].params.name) || false;
let contentId = (node.parsedHeader['content-id'] || '').toString().replace(/<|>/g, '').trim();
if (fileName) {
try {
@ -290,8 +362,15 @@ class Indexer {
} catch (E) {
// failed to parse filename, keep as is (most probably an unknown charset is used)
}
} else {
fileName = (crypto.randomBytes(4).toString('hex') + '.' + libmime.detectExtension(contentType));
}
cidMap.set(contentId, {
id: attachmentId,
fileName
});
let returned = false;
let store = this.gridstore.createWriteStream(attachmentId, {
fsync: true,
@ -312,12 +391,16 @@ class Indexer {
});
if (!['text/plain', 'text/html'].includes(contentType) || disposition === 'attachment') {
attachments.push({
// list in the attachments array
response.attachments.push({
id: attachmentId,
fileName,
contentType,
disposition,
transferEncoding
transferEncoding,
related,
// approximite size in kilobytes
sizeKb: Math.ceil((transferEncoding === 'base64' ? this.expectedB64Size(node.size) : node.size) / 1024)
});
}
@ -346,14 +429,36 @@ class Indexer {
continueProcessing();
}
};
walk(mimeTree, err => {
walk(mimeTree, false, false, err => {
if (err) {
return callback(err);
}
callback(null, attachments);
let updateCidLinks = str => str.replace(/\bcid:([^\s"']+)/g, (match, cid) => {
if (cidMap.has(cid)) {
let attachment = cidMap.get(cid);
return 'attachment:' + messageId + '/' + attachment.id.toString();
}
return match;
});
response.html = htmlContent.filter(str => str.trim()).map(updateCidLinks).join('\n').trim();
response.text = textContent.filter(str => str.trim()).map(updateCidLinks).join('\n').trim();
callback(null, response);
});
}
expectedB64Size(b64size) {
b64size = Number(b64size) || 0;
if (!b64size || b64size <= 0) {
return 0;
}
let newlines = Math.floor(b64size / 78);
return Math.ceil((b64size - newlines * 2) / 4 * 3);
}
/**
* Generates IMAP compatible BODY object from message tree
*

13
indexes.json
View file

@ -75,39 +75,52 @@
}, {
"name": "by_modseq",
"key": {
"mailbox": 1,
"modseq": 1
}
}, {
"name": "by_flags",
"key": {
"mailbox": 1,
"flags": 1
}
}, {
"name": "by_internaldate",
"key": {
"mailbox": 1,
"internaldate": 1
}
}, {
"name": "by_headerdate",
"key": {
"mailbox": 1,
"headerdate": 1
}
}, {
"name": "by_size",
"key": {
"mailbox": 1,
"size": 1
}
}, {
"name": "by_uid",
"key": {
"mailbox": 1,
"uid": 1
}
}, {
"name": "by_headers",
"key": {
"mailbox": 1,
"headers.key": 1,
"headers.value": 1
}
}, {
"name": "bhas_attachments",
"key": {
"mailbox": 1,
"hasAttachments": 1
}
}]
}, {
"collection": "attachment.files",

20
lib/message-handler.js
View file

@ -63,7 +63,9 @@ class MessageHandler {
}).defaultView;
let domPurify = createDOMPurify(win);
return domPurify.sanitize(html);
return domPurify.sanitize(html, {
ALLOW_UNKNOWN_PROTOCOLS: true // allow cid: images
});
}
add(options, callback) {
@ -100,7 +102,7 @@ class MessageHandler {
return callback(err);
}
this.indexer.processContent(id, mimeTree, 50 * 1024, (err, attachments) => {
this.indexer.processContent(id, mimeTree, (err, maildata) => {
if (err) {
return callback(err);
}
@ -196,23 +198,23 @@ class MessageHandler {
messageId
};
if (attachments && attachments.length) {
message.attachments = attachments;
if (maildata.attachments && maildata.attachments.length) {
message.attachments = maildata.attachments;
message.hasAttachments = true;
}else{
message.hasAttachments = false;
}
/*
// use mailparser to parse plaintext and html content
let maxTextLength = 200 * 1024;
if (maildata.plain) {
message.text = maildata.plain.replace(/\r\n/g, '\n').trim();
if (maildata.text) {
message.text = maildata.text.replace(/\r\n/g, '\n').trim();
message.text = message.text.length <= maxTextLength ? message.text : message.text.substr(0, maxTextLength);
}
if (maildata.html) {
message.html = this.cleanHtml(maildata.html.replace(/\r\n/g, '\n')).trim();
message.html = message.html.length <= maxTextLength ? message.html : message.html.substr(0, maxTextLength);
}
*/
this.database.collection('messages').insertOne(message, err => {
if (err) {

4
package.json
View file

@ -24,10 +24,14 @@
"config": "^1.25.1",
"dompurify": "^0.8.5",
"grid-fs": "^1.0.1",
"html-to-text": "^3.2.0",
"iconv-lite": "^0.4.15",
"joi": "^10.3.4",
"jsdom": "^9.12.0",
"libbase64": "^0.1.0",
"libmime": "^3.1.0",
"libqp": "^1.1.0",
"marked": "^0.3.6",
"mongodb": "^2.2.25",
"nodemailer": "^3.1.8",
"npmlog": "^4.0.2",