From d1c67e6264f7f01f52646404ad1b38fef7cdbd3f Mon Sep 17 00:00:00 2001 From: Nikolai Ovtsinnikov Date: Wed, 4 Dec 2024 21:39:08 +0200 Subject: [PATCH] fix streams in on-copy and message-handler. message-handler optimizations, filter-handler optimizations --- lib/filter-handler.js | 15 +- lib/handlers/on-copy.js | 22 +- lib/message-handler.js | 831 +++++++++++++++++++++------------------- 3 files changed, 467 insertions(+), 401 deletions(-) diff --git a/lib/filter-handler.js b/lib/filter-handler.js index 3bda7899..ab5ba65d 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -141,10 +141,9 @@ class FilterHandler { let rawchunks = chunks; + let raw; + let prepared; - - let raw = Buffer.concat(chunks, chunklen); - if (options.mimeTree) { if (options.mimeTree && options.mimeTree.header) { // remove old headers @@ -159,6 +158,7 @@ class FilterHandler { mimeTree: options.mimeTree }); } else { + raw = Buffer.concat(chunks, chunklen); prepared = await this.prepareMessage({ raw }); @@ -661,11 +661,14 @@ class FilterHandler { date: false, flags, - - raw, - rawchunks + rawchunks, + chunklen }; + if (raw) { + messageOpts.raw = raw; + } + if (options.verificationResults) { messageOpts.verificationResults = options.verificationResults; } diff --git a/lib/handlers/on-copy.js b/lib/handlers/on-copy.js index 25a319c8..8e802358 100644 --- a/lib/handlers/on-copy.js +++ b/lib/handlers/on-copy.js @@ -233,15 +233,27 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, const newPrepared = await new Promise((resolve, reject) => { if (targetMailboxEncrypted && !isMessageEncrypted && userData.pubKey) { // encrypt message - const outputStream = messageHandler.indexer.rebuild(messageData.mimeTree).value; // get raw rebuilder stream - let raw = Buffer.from([], 'binary'); // set initial raw + // get raw from existing mimetree + let outputStream = messageHandler.indexer.rebuild(messageData.mimeTree); // get raw rebuilder response obj (.value is the stream) + if (!outputStream || outputStream.type !== 'stream' || !outputStream.value) { + return reject(new Error('Cannot fetch message')); + } + outputStream = outputStream.value; // set stream to actual stream object (.value) + + let chunks = []; + let chunklen = 0; outputStream - .on('data', data => { - raw = Buffer.concat([raw, data]); + .on('readable', () => { + let chunk; + while ((chunk = outputStream.read()) !== null) { + chunks.push(chunk); + chunklen += chunk.length; + } }) .on('end', () => { - messageHandler.encryptMessages(userData.pubKey || '', raw, (err, res) => { + const raw = Buffer.concat(chunks, chunklen); + messageHandler.encryptMessages(userData.pubKey, raw, (err, res) => { if (err) { return reject(err); } diff --git a/lib/message-handler.js b/lib/message-handler.js index a9d97efb..e9544217 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -204,474 +204,510 @@ class MessageHandler { } // get target user data + let prepared = options.prepared; // might be undefined - this.prepareMessage(options, (err, prepared) => { - if (err) { - return callback(err); - } + // check if already encrypted + let alreadyEncrypted = false; - // check if already encrypted - let alreadyEncrypted = false; - - // message already prepared, check if encrypted + // message already prepared, check if encrypted + if (prepared) { + // got prepared const parsedHeader = (prepared.mimeTree && prepared.mimeTree?.parsedHeader) || {}; const parsedContentType = parsedHeader['content-type']; if (parsedContentType && parsedContentType.subtype === 'encrypted') { alreadyEncrypted = true; } + } else { + // no prepared, use raw + if (options.rawchunks && !options.raw) { + // got rawchunks instead of raw + if (options.chunklen) { + options.raw = Buffer.concat(options.rawchunks, options.chunklen); + } else { + options.raw = Buffer.concat(options.rawchunks); + } + } - let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); + const rawString = options.raw.toString('binary'); // get string from the raw bytes of message + const regex = /Content-Type:\s*multipart\/encrypted/gim; - let addMessage = () => { - let id = prepared.id; - let mimeTree = prepared.mimeTree; - let size = prepared.size; - let bodystructure = prepared.bodystructure; - let envelope = prepared.envelope; - let idate = prepared.idate; - let hdate = prepared.hdate; - let msgid = prepared.msgid; - let subject = prepared.subject; - let headers = prepared.headers; + if (regex.test(rawString)) { + // if there is encrypted content-type then message already encrypted, no need to re-encrypt it + alreadyEncrypted = true; + } + } - let maildata = options.maildata || this.indexer.getMaildata(mimeTree); + let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); - let cleanup = (...args) => { - if (!args[0]) { - return callback(...args); - } + let addMessage = () => { + let id = prepared.id; + let mimeTree = prepared.mimeTree; + let size = prepared.size; + let bodystructure = prepared.bodystructure; + let envelope = prepared.envelope; + let idate = prepared.idate; + let hdate = prepared.hdate; + let msgid = prepared.msgid; + let subject = prepared.subject; + let headers = prepared.headers; - let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); - if (!attachmentIds.length) { - return callback(...args); - } + let maildata = options.maildata || this.indexer.getMaildata(mimeTree); - this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); + let cleanup = (...args) => { + if (!args[0]) { + return callback(...args); + } + + let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); + if (!attachmentIds.length) { + return callback(...args); + } + + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); + }; + + this.indexer.storeNodeBodies(maildata, mimeTree, err => { + if (err) { + return cleanup(err); + } + + // prepare message object + let messageData = { + _id: id, + + // should be kept when COPY'ing or MOVE'ing + root: id, + + v: consts.SCHEMA_VERSION, + + // if true then expires after rdate + retention + exp: !!mailboxData.retention, + rdate: Date.now() + (mailboxData.retention || 0), + + // make sure the field exists. it is set to true when user is deleted + userDeleted: false, + + idate, + hdate, + flags, + size, + + // some custom metadata about the delivery + meta: options.meta || {}, + + // list filter IDs that matched this message + filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), + + headers, + mimeTree, + envelope, + bodystructure, + msgid, + + // use boolean for more commonly used (and searched for) flags + unseen: !flags.includes('\\Seen'), + flagged: flags.includes('\\Flagged'), + undeleted: !flags.includes('\\Deleted'), + draft: flags.includes('\\Draft'), + + magic: maildata.magic, + + subject, + + // do not archive deleted messages that have been copied + copied: false }; - this.indexer.storeNodeBodies(maildata, mimeTree, err => { - if (err) { - return cleanup(err); - } + if (options.verificationResults) { + messageData.verificationResults = options.verificationResults; + } - // prepare message object - let messageData = { - _id: id, + if (options.outbound) { + messageData.outbound = [].concat(options.outbound || []); + } - // should be kept when COPY'ing or MOVE'ing - root: id, + if (options.forwardTargets) { + messageData.forwardTargets = [].concat(options.forwardTargets || []); + } - v: consts.SCHEMA_VERSION, + if (maildata.attachments && maildata.attachments.length) { + messageData.attachments = maildata.attachments; + messageData.ha = maildata.attachments.some(a => !a.related); + } else { + messageData.ha = false; + } - // if true then expires after rdate + retention - exp: !!mailboxData.retention, - rdate: Date.now() + (mailboxData.retention || 0), + if (maildata.text) { + messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); - // make sure the field exists. it is set to true when user is deleted - userDeleted: false, + // text is indexed with a fulltext index, so only store the beginning of it + if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { + messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); + messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); - idate, - hdate, - flags, - size, - - // some custom metadata about the delivery - meta: options.meta || {}, - - // list filter IDs that matched this message - filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []), - - headers, - mimeTree, - envelope, - bodystructure, - msgid, - - // use boolean for more commonly used (and searched for) flags - unseen: !flags.includes('\\Seen'), - flagged: flags.includes('\\Flagged'), - undeleted: !flags.includes('\\Deleted'), - draft: flags.includes('\\Draft'), - - magic: maildata.magic, - - subject, - - // do not archive deleted messages that have been copied - copied: false - }; - - if (options.verificationResults) { - messageData.verificationResults = options.verificationResults; - } - - if (options.outbound) { - messageData.outbound = [].concat(options.outbound || []); - } - - if (options.forwardTargets) { - messageData.forwardTargets = [].concat(options.forwardTargets || []); - } - - if (maildata.attachments && maildata.attachments.length) { - messageData.attachments = maildata.attachments; - messageData.ha = maildata.attachments.some(a => !a.related); - } else { - messageData.ha = false; - } - - if (maildata.text) { - messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); - - // text is indexed with a fulltext index, so only store the beginning of it - if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) { - messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED); - messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED); - - // truncate remaining text if total length exceeds maximum allowed - if ( - consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && - messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED - ) { - messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); - } + // truncate remaining text if total length exceeds maximum allowed + if ( + consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED && + messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED + ) { + messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED); } - messageData.text = - messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT - ? messageData.text - : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - - messageData.intro = this.createIntro(messageData.text); } + messageData.text = + messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT + ? messageData.text + : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - if (maildata.html && maildata.html.length) { - let htmlSize = 0; - messageData.html = maildata.html - .map(html => { - if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { - return ''; - } + messageData.intro = this.createIntro(messageData.text); + } - if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { - htmlSize += Buffer.byteLength(html); - return html; - } + if (maildata.html && maildata.html.length) { + let htmlSize = 0; + messageData.html = maildata.html + .map(html => { + if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { + return ''; + } - html = html.substr(0, consts.MAX_HTML_CONTENT); + if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { htmlSize += Buffer.byteLength(html); return html; - }) - .filter(html => html); - - // if message has HTML content use it instead of text/plain content for intro - messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); - } - - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true - } - }, - (err, r) => { - if (err) { - return cleanup(err); } - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] +', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id - }); - } + html = html.substr(0, consts.MAX_HTML_CONTENT); + htmlSize += Buffer.byteLength(html); + return html; + }) + .filter(html => html); - let rollback = err => { - this.users.collection('users').findOneAndUpdate( - { - _id: mailboxData.user - }, - { - $inc: { - storageUsed: -size - } - }, - { - returnDocument: 'after', - projection: { - storageUsed: true - } - }, - (...args) => { - let r = args && args[1]; + // if message has HTML content use it instead of text/plain content for intro + messageData.intro = this.createIntro(htmlToText(messageData.html.join(''))); + } - if (r && r.value) { - this.loggelf({ - short_message: '[QUOTA] -', - _mail_action: 'quota', - _user: mailboxData.user, - _inc: -size, - _storage_used: r.value.storageUsed, - _sess: options.session && options.session.id, - _mailbox: mailboxData._id, - _rollback: 'yes', - _error: err.message, - _code: err.code - }); - } + this.users.collection('users').findOneAndUpdate( + { + _id: mailboxData.user + }, + { + $inc: { + storageUsed: size + } + }, + { + returnDocument: 'after', + projection: { + storageUsed: true + } + }, + (err, r) => { + if (err) { + return cleanup(err); + } - cleanup(err); - } - ); - }; + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] +', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id + }); + } - // acquire new UID+MODSEQ - this.database.collection('mailboxes').findOneAndUpdate( + let rollback = err => { + this.users.collection('users').findOneAndUpdate( { - _id: mailboxData._id + _id: mailboxData.user }, { $inc: { - // allocate bot UID and MODSEQ values so when journal is later sorted by - // modseq then UIDs are always in ascending order - uidNext: 1, - modifyIndex: 1 + storageUsed: -size } }, { - // use original value to get correct UIDNext - returnDocument: 'before' + returnDocument: 'after', + projection: { + storageUsed: true + } }, - (err, item) => { + (...args) => { + let r = args && args[1]; + + if (r && r.value) { + this.loggelf({ + short_message: '[QUOTA] -', + _mail_action: 'quota', + _user: mailboxData.user, + _inc: -size, + _storage_used: r.value.storageUsed, + _sess: options.session && options.session.id, + _mailbox: mailboxData._id, + _rollback: 'yes', + _error: err.message, + _code: err.code + }); + } + + cleanup(err); + } + ); + }; + + // acquire new UID+MODSEQ + this.database.collection('mailboxes').findOneAndUpdate( + { + _id: mailboxData._id + }, + { + $inc: { + // allocate bot UID and MODSEQ values so when journal is later sorted by + // modseq then UIDs are always in ascending order + uidNext: 1, + modifyIndex: 1 + } + }, + { + // use original value to get correct UIDNext + returnDocument: 'before' + }, + (err, item) => { + if (err) { + return rollback(err); + } + + if (!item || !item.value) { + // was not able to acquire a lock + let err = new Error('Mailbox is missing'); + err.imapResponse = 'TRYCREATE'; + return rollback(err); + } + + let mailboxData = item.value; + + // updated message object by setting mailbox specific values + messageData.mailbox = mailboxData._id; + messageData.user = mailboxData.user; + messageData.uid = mailboxData.uidNext; + messageData.modseq = mailboxData.modifyIndex + 1; + + if (!flags.includes('\\Deleted')) { + messageData.searchable = true; + } + + if (mailboxData.specialUse === '\\Junk') { + messageData.junk = true; + } + + this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { if (err) { return rollback(err); } - if (!item || !item.value) { - // was not able to acquire a lock - let err = new Error('Mailbox is missing'); - err.imapResponse = 'TRYCREATE'; - return rollback(err); - } + messageData.thread = thread; - let mailboxData = item.value; - - // updated message object by setting mailbox specific values - messageData.mailbox = mailboxData._id; - messageData.user = mailboxData.user; - messageData.uid = mailboxData.uidNext; - messageData.modseq = mailboxData.modifyIndex + 1; - - if (!flags.includes('\\Deleted')) { - messageData.searchable = true; - } - - if (mailboxData.specialUse === '\\Junk') { - messageData.junk = true; - } - - this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { + this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { if (err) { return rollback(err); } - messageData.thread = thread; + if (!r || !r.acknowledged) { + let err = new Error('Failed to store message [1]'); + err.responseCode = 500; + err.code = 'StoreError'; + return rollback(err); + } - this.database.collection('messages').insertOne(messageData, { writeConcern: 'majority' }, (err, r) => { - if (err) { - return rollback(err); + let logTime = messageData.meta.time || new Date(); + if (typeof logTime === 'number') { + logTime = new Date(logTime); + } + + let uidValidity = mailboxData.uidValidity; + let uid = messageData.uid; + + if ( + options.session && + options.session.selected && + options.session.selected.mailbox && + options.session.selected.mailbox.toString() === mailboxData._id.toString() + ) { + options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); + } + + let updateAddressRegister = next => { + let addresses = []; + + if (messageData.junk || flags.includes('\\Draft')) { + // skip junk and draft messages + return next(); } - if (!r || !r.acknowledged) { - let err = new Error('Failed to store message [1]'); - err.responseCode = 500; - err.code = 'StoreError'; - return rollback(err); - } + let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; - let logTime = messageData.meta.time || new Date(); - if (typeof logTime === 'number') { - logTime = new Date(logTime); - } + if (parsed) { + let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; - let uidValidity = mailboxData.uidValidity; - let uid = messageData.uid; - - if ( - options.session && - options.session.selected && - options.session.selected.mailbox && - options.session.selected.mailbox.toString() === mailboxData._id.toString() - ) { - options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); - } - - let updateAddressRegister = next => { - let addresses = []; - - if (messageData.junk || flags.includes('\\Draft')) { - // skip junk and draft messages - return next(); + for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { + // if email contains headers that we do not want, + // don't add any emails to address register + if (parsed[disallowedHeader]) { + return next(); + } } - let parsed = messageData.mimeTree && messageData.mimeTree.parsedHeader; - - if (parsed) { - let keyList = mailboxData.specialUse === '\\Sent' ? ['to', 'cc', 'bcc'] : ['from']; - - for (const disallowedHeader of DISALLOWED_HEADERS_FOR_ADDRESS_REGISTER) { - // if email contains headers that we do not want, - // don't add any emails to address register - if (parsed[disallowedHeader]) { - return next(); - } - } - - for (let key of keyList) { - if (parsed[key] && parsed[key].length) { - for (let addr of parsed[key]) { - if (/no-?reply/i.test(addr.address)) { - continue; - } - if (!addresses.some(a => a.address === addr.address)) { - addresses.push(addr); - } + for (let key of keyList) { + if (parsed[key] && parsed[key].length) { + for (let addr of parsed[key]) { + if (/no-?reply/i.test(addr.address)) { + continue; + } + if (!addresses.some(a => a.address === addr.address)) { + addresses.push(addr); } } } } + } - if (!addresses.length) { - return next(); - } + if (!addresses.length) { + return next(); + } - this.updateAddressRegister(mailboxData.user, addresses) - .then(() => next()) - .catch(err => next(err)); - }; + this.updateAddressRegister(mailboxData.user, addresses) + .then(() => next()) + .catch(err => next(err)); + }; - updateAddressRegister(() => { - this.notifier.addEntries( - mailboxData, - { - command: 'EXISTS', - uid: messageData.uid, - ignore: options.session && options.session.id, - message: messageData._id, - modseq: messageData.modseq, - unseen: messageData.unseen, - idate: messageData.idate, - thread: messageData.thread - }, - () => { - this.notifier.fire(mailboxData.user); + updateAddressRegister(() => { + this.notifier.addEntries( + mailboxData, + { + command: 'EXISTS', + uid: messageData.uid, + ignore: options.session && options.session.id, + message: messageData._id, + modseq: messageData.modseq, + unseen: messageData.unseen, + idate: messageData.idate, + thread: messageData.thread + }, + () => { + this.notifier.fire(mailboxData.user); - let raw = options.rawchunks || options.raw; - let processAudits = async () => { - let audits = await this.database - .collection('audits') - .find({ user: mailboxData.user, expires: { $gt: new Date() } }) - .toArray(); + let raw = options.rawchunks || options.raw; + let processAudits = async () => { + let audits = await this.database + .collection('audits') + .find({ user: mailboxData.user, expires: { $gt: new Date() } }) + .toArray(); - let now = new Date(); - for (let auditData of audits) { - if ( - (auditData.start && auditData.start > now) || - (auditData.end && auditData.end < now) - ) { - // audit not active - continue; - } - await this.auditHandler.store(auditData._id, raw, { - date: messageData.idate, - msgid: messageData.msgid, - header: messageData.mimeTree && messageData.mimeTree.parsedHeader, - ha: messageData.ha, - mailbox: mailboxData._id, - mailboxPath: mailboxData.path, - info: Object.assign({ queueId: messageData.outbound }, messageData.meta) - }); + let now = new Date(); + for (let auditData of audits) { + if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) { + // audit not active + continue; } - }; - - let next = () => { - cleanup(null, true, { - uidValidity, - uid, - id: messageData._id.toString(), - mailbox: mailboxData._id.toString(), + await this.auditHandler.store(auditData._id, raw, { + date: messageData.idate, + msgid: messageData.msgid, + header: messageData.mimeTree && messageData.mimeTree.parsedHeader, + ha: messageData.ha, + mailbox: mailboxData._id, mailboxPath: mailboxData.path, - size, - status: 'new' + info: Object.assign({ queueId: messageData.outbound }, messageData.meta) }); - }; + } + }; - // do not use more suitable .finally() as it is not supported in Node v8 - return processAudits().then(next).catch(next); - } - ); - }); + let next = () => { + cleanup(null, true, { + uidValidity, + uid, + id: messageData._id.toString(), + mailbox: mailboxData._id.toString(), + mailboxPath: mailboxData.path, + size, + status: 'new' + }); + }; + + // do not use more suitable .finally() as it is not supported in Node v8 + return processAudits().then(next).catch(next); + } + ); }); }); - } - ); - } - ); - }); - }; + }); + } + ); + } + ); + }); + }; - if (!alreadyEncrypted) { - // not already encrypted, check if user has encryption on or target mailbox is encrypted - if ((userData.encryptMessages || !!mailboxData.encryptMessages) && userData.pubKey && !flags.includes('\\Draft')) { - // user has encryption on or target mailbox encrypted, encrypt message and prepare again - // do not encrypt drafts - this.encryptMessage(userData.pubKey, options.raw, (err, res) => { + if (!alreadyEncrypted) { + // not already encrypted, check if user has encryption on or target mailbox is encrypted + if ((userData.encryptMessages || !!mailboxData.encryptMessages) && userData.pubKey && !flags.includes('\\Draft')) { + if (options.rawchunks && !options.raw) { + // got rawchunks instead of raw + if (options.chunklen) { + options.raw = Buffer.concat(options.rawchunks, options.chunklen); + } else { + options.raw = Buffer.concat(options.rawchunks); + } + } + // user has encryption on or target mailbox encrypted, encrypt message and prepare again + // do not encrypt drafts + // may have a situation where we got prepared and no options.raw but options.rawchunks instead, concat them + this.encryptMessage(userData.pubKey, options.raw, (err, res) => { + if (err) { + return callback(err); + } + + if (res) { + // new encrypted raw available + options.raw = res; + } + + delete options.prepared; // delete any existing prepared as new will be generated + this.prepareMessage(options, (err, newPrepared) => { if (err) { return callback(err); } - if (res) { - // new encrypted raw available - options.raw = res; - } + newPrepared.id = prepared.id; // retain original - delete options.prepared; // delete any existing prepared as new will be generated - this.prepareMessage(options, (err, newPrepared) => { - if (err) { - return callback(err); - } - - newPrepared.id = prepared.id; // retain original - - options.prepared = newPrepared; // new prepared in options just in case - prepared = newPrepared; // overwrite top-level original prepared - options.maildata = this.indexer.getMaildata(newPrepared.mimeTree); // get new maildata of encrypted message - addMessage(); - }); + options.prepared = newPrepared; // new prepared in options just in case + prepared = newPrepared; // overwrite top-level original prepared + options.maildata = this.indexer.getMaildata(newPrepared.mimeTree); // get new maildata of encrypted message + addMessage(); }); - } else { - // not already encrypted and no need to - addMessage(); - } + }); } else { - // message already encrypted - addMessage(); + // not already encrypted and no need to + this.prepareMessage(options, (err, newPrepared) => { + if (err) { + return callback(err); + } + + prepared = newPrepared; + addMessage(); + }); } - }); + } else { + // message already encrypted + this.prepareMessage(options, (err, newPrepared) => { + if (err) { + return callback(err); + } + + prepared = newPrepared; + addMessage(); + }); + } }); }); } @@ -1204,17 +1240,32 @@ class MessageHandler { return done(err); } // get user data + if (!res.pubKey) { + return updateMessage(); + } // get raw from existing mimetree - const outputStream = this.indexer.rebuild(message.mimeTree).value; // get raw rebuilder stream - let raw = Buffer.from([], 'binary'); // set initial raw + let outputStream = this.indexer.rebuild(message.mimeTree); // get raw rebuilder response obj (.value is the stream) + + if (!outputStream || outputStream.type !== 'stream' || !outputStream.value) { + return done(new Error('Cannot fetch message')); + } + outputStream = outputStream.value; // set stream to actual stream object (.value) + + let chunks = []; + let chunklen = 0; outputStream - .on('data', data => { - raw = Buffer.concat([raw, data]); + .on('readable', () => { + let chunk; + while ((chunk = outputStream.read()) !== null) { + chunks.push(chunk); + chunklen += chunk.length; + } }) .on('end', () => { // when done rebuilding - this.encryptMessage(res.pubKey || '', raw, (err, res) => { + const raw = Buffer.concat(chunks, chunklen); + this.encryptMessage(res.pubKey, raw, (err, res) => { if (err) { return done(err); }