diff --git a/lib/api/messages.js b/lib/api/messages.js index 9dced021..061533eb 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -597,6 +597,8 @@ module.exports = (db, server, messageHandler) => { draft: true, attachments: true, html: true, + text: true, + textFooter: true, forwardTargets: true } }, (err, messageData) => { @@ -665,12 +667,18 @@ module.exports = (db, server, messageHandler) => { expires = new Date(messageData.rdate).toISOString(); } + messageData.text = (messageData.text || '') + (messageData.textFooter || ''); + if (replaceCidLinks) { messageData.html = (messageData.html || []).map(html => html.replace(/attachment:([a-f0-9]+)\/(ATT\d+)/g, (str, mid, aid) => server.router.render('attachment', { user, mailbox, message, attachment: aid }) ) ); + + messageData.text = messageData.text.replace(/attachment:([a-f0-9]+)\/(ATT\d+)/g, (str, mid, aid) => + server.router.render('attachment', { user, mailbox, message, attachment: aid }) + ); } let ensureSeen = done => { @@ -708,6 +716,7 @@ module.exports = (db, server, messageHandler) => { flagged: messageData.flagged, draft: messageData.draft, html: messageData.html, + text: messageData.text, forwardTargets: messageData.forwardTargets, attachments: (messageData.attachments || []).map(attachment => { attachment.url = server.router.render('attachment', { user, mailbox, message, attachment: attachment.id }); diff --git a/lib/consts.js b/lib/consts.js index 0651a275..b47c31fd 100644 --- a/lib/consts.js +++ b/lib/consts.js @@ -19,8 +19,12 @@ module.exports = { MAILBOX_COUNTER_TTL: 24 * 3600, SCHEMA_VERSION: '1.0', - // how much plaintext to store. this is indexed with a fulltext index - MAX_PLAINTEXT_CONTENT: 2 * 1024, + + // how much plaintext to store in a full text indexed field + MAX_PLAINTEXT_INDEXED: 2 * 1024, + + // how much plaintext to store before truncating + MAX_PLAINTEXT_CONTENT: 100 * 1024, // how much HTML content to store. not indexed MAX_HTML_CONTENT: 300 * 1024, diff --git a/lib/message-handler.js b/lib/message-handler.js index 2f875fb6..f8611b56 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -191,12 +191,37 @@ class MessageHandler { if (maildata.text) { messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); + // text is indexed with a fulltext index, so only store the beginning of it + if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { + messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); + messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); + + // truncate remaining text if total length exceeds maximum allowed + if ( + consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && + messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED + ) { + messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); + } + } messageData.text = messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT ? messageData.text : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - messageData.intro = messageData.text.replace(/\s+/g, ' ').trim(); + + messageData.intro = messageData.text + // assume we get the intro text from first 2 kB + .substr(0, 2 * 1024) + // remove quoted parts + // "> quote from previous message" + .replace(/^>.*$/gm, '') + // remove lines with repetetive chars + // "---------------------" + .replace(/^\s*(.)\1+\s*$/gm, '') + // join lines + .replace(/\s+/g, ' ') + .trim(); if (messageData.intro.length > 128) { let intro = messageData.intro.substr(0, 128); let lastSp = intro.lastIndexOf(' ');