From 8560fa58b588813f1c528718836357823babd5ca Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Sat, 1 Apr 2017 12:22:51 +0300 Subject: [PATCH] v1.0.9 --- imap-core/lib/indexer/parse-mime-tree.js | 4 +- imap-core/test/memory-notifier.js | 150 +++++++++++++++ imap-core/test/test-server.js | 23 +-- imap.js | 228 ++++++++++++++--------- package.json | 4 +- 5 files changed, 307 insertions(+), 102 deletions(-) create mode 100644 imap-core/test/memory-notifier.js diff --git a/imap-core/lib/indexer/parse-mime-tree.js b/imap-core/lib/indexer/parse-mime-tree.js index 702c08e9..c2257323 100644 --- a/imap-core/lib/indexer/parse-mime-tree.js +++ b/imap-core/lib/indexer/parse-mime-tree.js @@ -165,7 +165,7 @@ class MIMEParser { // Do not touch headers that have strange looking keys, keep these // only in the unparsed array - if (/^[a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) { + if (/[^a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) { continue; } @@ -245,7 +245,7 @@ class MIMEParser { // Do not touch headers that have strange looking keys, keep these // only in the unparsed array - if (/^[a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) { + if (/[^a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) { return; } diff --git a/imap-core/test/memory-notifier.js b/imap-core/test/memory-notifier.js new file mode 100644 index 00000000..331f7550 --- /dev/null +++ b/imap-core/test/memory-notifier.js @@ -0,0 +1,150 @@ +'use strict'; + +let crypto = require('crypto'); +let EventEmitter = require('events').EventEmitter; + +// Expects that the folder listing is a Map + +class MemoryNotifier extends EventEmitter { + + constructor(options) { + super(); + this.folders = options.folders || new Map(); + + let logfunc = (...args) => { + let level = args.shift() || 'DEBUG'; + let message = args.shift() || ''; + + console.log([level].concat(message || '').join(' '), ...args); // eslint-disable-line no-console + }; + + this.logger = options.logger || { + info: logfunc.bind(null, 'INFO'), + debug: logfunc.bind(null, 'DEBUG'), + error: logfunc.bind(null, 'ERROR') + }; + + this._listeners = new EventEmitter(); + this._listeners.setMaxListeners(0); + + EventEmitter.call(this); + } + + /** + * Generates hashed event names for mailbox:username pairs + * + * @param {String} mailbox + * @param {String} username + * @returns {String} md5 hex + */ + _eventName(mailbox, username) { + return crypto.createHash('md5').update(username + ':' + mailbox).digest('hex'); + } + + /** + * Registers an event handler for mailbox:username events + * + * @param {String} username + * @param {String} mailbox + * @param {Function} handler Function to run once there are new entries in the journal + */ + addListener(session, mailbox, handler) { + let eventName = this._eventName(session.user.username, mailbox); + this._listeners.addListener(eventName, handler); + + this.logger.debug('New journal listener for %s ("%s:%s")', eventName, session.user.username, mailbox); + } + + /** + * Unregisters an event handler for mailbox:username events + * + * @param {String} username + * @param {String} mailbox + * @param {Function} handler Function to run once there are new entries in the journal + */ + removeListener(session, mailbox, handler) { + let eventName = this._eventName(session.user.username, mailbox); + this._listeners.removeListener(eventName, handler); + + this.logger.debug('Removed journal listener from %s ("%s:%s")', eventName, session.user.username, mailbox); + } + + /** + * Stores multiple journal entries to db + * + * @param {String} username + * @param {String} mailbox + * @param {Array|Object} entries An array of entries to be journaled + * @param {Function} callback Runs once the entry is either stored or an error occurred + */ + addEntries(username, mailbox, entries, callback) { + let folder = this.folders.get(mailbox); + + if (!folder) { + return callback(null, new Error('Selected mailbox does not exist')); + } + + if (entries && !Array.isArray(entries)) { + entries = [entries]; + } else if (!entries || !entries.length) { + return callback(null, false); + } + + // store entires in the folder object + if (!folder.journal) { + folder.journal = []; + } + + entries.forEach(entry => { + entry.modseq = ++folder.modifyIndex; + folder.journal.push(entry); + }); + + setImmediate(callback); + } + + /** + * Sends a notification that there are new updates in the selected mailbox + * + * @param {String} username + * @param {String} mailbox + */ + fire(username, mailbox, payload) { + let eventName = this._eventName(username, mailbox); + setImmediate(() => { + this._listeners.emit(eventName, payload); + }); + } + + /** + * Returns all entries from the journal that have higher than provided modification index + * + * @param {String} username + * @param {String} mailbox + * @param {Number} modifyIndex Last known modification id + * @param {Function} callback Returns update entries as an array + */ + getUpdates(session, mailbox, modifyIndex, callback) { + modifyIndex = Number(modifyIndex) || 0; + + if (!this.folders.has(mailbox)) { + return callback(null, 'NONEXISTENT'); + } + + let folder = this.folders.get(mailbox); + let minIndex = folder.journal.length; + + for (let i = folder.journal.length - 1; i >= 0; i--) { + if (folder.journal[i].modseq > modifyIndex) { + minIndex = i; + } else { + break; + } + } + + return callback(null, folder.journal.slice(minIndex)); + } + +} + +module.exports = MemoryNotifier; diff --git a/imap-core/test/test-server.js b/imap-core/test/test-server.js index 811f9ca9..d7f0b182 100644 --- a/imap-core/test/test-server.js +++ b/imap-core/test/test-server.js @@ -1,10 +1,11 @@ 'use strict'; -let IMAPServerModule = require('../index.js'); -let IMAPServer = IMAPServerModule.IMAPServer; -let MemoryNotifier = IMAPServerModule.MemoryNotifier; -let fs = require('fs'); -let imapHandler = require('../lib/handler/imap-handler'); +const IMAPServerModule = require('../index.js'); +const IMAPServer = IMAPServerModule.IMAPServer; +const MemoryNotifier = require('./memory-notifier.js'); +const fs = require('fs'); +const parseMimeTree = require('../lib/indexer/parse-mime-tree'); +const imapHandler = require('../lib/handler/imap-handler'); module.exports = function (options) { @@ -22,19 +23,19 @@ module.exports = function (options) { flags: [], modseq: 100, internaldate: new Date('14-Sep-2013 21:22:28 -0300'), - raw: new Buffer('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz') + mimeTree: parseMimeTree(new Buffer('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz')) }, { uid: 49, flags: ['\\Seen'], internaldate: new Date(), modseq: 5000, - raw: fs.readFileSync(__dirname + '/fixtures/ryan_finnie_mime_torture.eml') + mimeTree: parseMimeTree(fs.readFileSync(__dirname + '/fixtures/ryan_finnie_mime_torture.eml')) }, { uid: 50, flags: ['\\Seen'], modseq: 45, internaldate: new Date(), - raw: 'MIME-Version: 1.0\r\n' + + mimeTree: parseMimeTree('MIME-Version: 1.0\r\n' + 'From: andris@kreata.ee\r\n' + 'To: andris@tr.ee\r\n' + 'Content-Type: multipart/mixed;\r\n' + @@ -65,13 +66,13 @@ module.exports = function (options) { 'Content-Transfer-Encoding: quoted-printable\r\n' + '\r\n' + 'Hello world 3!\r\n' + - '------mailcomposer-?=_1-1328088797399--' + '------mailcomposer-?=_1-1328088797399--') }, { uid: 52, flags: [], modseq: 4, internaldate: new Date(), - raw: 'from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nHello World!' + mimeTree: parseMimeTree('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nHello World!') }, { uid: 53, flags: [], @@ -286,7 +287,7 @@ module.exports = function (options) { uid: folder.uidNext++, modseq: ++folder.modifyIndex, date: date && new Date(date) || new Date(), - raw, + mimeTree: parseMimeTree(raw), flags }; diff --git a/imap.js b/imap.js index 073dad1b..b7c3fb5b 100644 --- a/imap.js +++ b/imap.js @@ -413,6 +413,45 @@ server.onAppend = function (path, flags, date, raw, session, callback) { }); }; +server.updateMailboxFlags = function (mailbox, update, callback) { + if (update.action === 'remove') { + // we didn't add any new flags, so there's nothing to update + return callback(); + } + + let mailboxFlags = imapTools.systemFlags.concat(mailbox.flags || []).map(flag => flag.trim().toLowerCase()); + let newFlags = []; + + // find flags that are not listed with mailbox + update.value.forEach(flag => { + // limit mailbox flags by 100 + if (mailboxFlags.length + newFlags.length >= 100) { + return; + } + // if mailbox does not have such flag, then add it + if (!mailboxFlags.includes(flag.toLowerCase().trim())) { + newFlags.push(flag); + } + }); + + // nothing new found + if (!newFlags.length) { + return callback(); + } + + // found some new flags not yet set for mailbox + // FIXME: Should we send unsolicited FLAGS and PERMANENTFLAGS notifications? + return db.database.collection('mailboxes').findOneAndUpdate({ + _id: mailbox._id + }, { + $addToSet: { + flags: { + $each: newFlags + } + } + }, {}, callback); +}; + // STORE / UID STORE, updates flags for selected UIDs server.onStore = function (path, update, session, callback) { this.logger.debug('[%s] Updating messages in "%s"', session.id, path); @@ -427,9 +466,6 @@ server.onStore = function (path, update, session, callback) { return callback(null, 'NONEXISTENT'); } - let mailboxFlags = imapTools.systemFlags.concat(mailbox.flags || []).map(flag => flag.trim().toLowerCase()); - let newFlags = []; - let cursor = db.database.collection('messages').find({ mailbox: mailbox._id, uid: { @@ -450,14 +486,25 @@ server.onStore = function (path, update, session, callback) { notifyEntries = []; setImmediate(() => this.notifier.addEntries(session.user.id, path, entries, () => { this.notifier.fire(session.user.id, path); - return callback(...args); + if (args[0]) { // first argument is an error + return callback(...args); + } else { + server.updateMailboxFlags(mailbox, update, () => callback(...args)); + } })); return; } this.notifier.fire(session.user.id, path); - return callback(...args); + if (args[0]) { // first argument is an error + return callback(...args); + } else { + server.updateMailboxFlags(mailbox, update, () => callback(...args)); + } }; + // We have to process all messages one by one instead of just calling an update + // for all messages as we need to know which messages were exactly modified, + // otherwise we can't send flag update notifications and modify modseq values let processNext = () => { cursor.next((err, message) => { if (err) { @@ -467,117 +514,124 @@ server.onStore = function (path, update, session, callback) { return cursor.close(() => done(null, true)); } - let flagsupdate = {}; + let flagsupdate = false; // query object for updates + let updated = false; + let existingFlags = message.flags.map(flag => flag.toLowerCase().trim()); switch (update.action) { case 'set': // check if update set matches current or is different - if (message.flags.length !== update.value.length || update.value.filter(flag => message.flags.indexOf(flag) < 0).length) { + if ( + // if length does not match + existingFlags.length !== update.value.length || + // or a new flag was found + update.value.filter(flag => !existingFlags.includes(flag.toLowerCase().trim())).length + ) { updated = true; } + message.flags = [].concat(update.value); + // set flags - flagsupdate.$set = { - flags: message.flags - }; + if (updated) { + flagsupdate = { + $set: { + flags: message.flags + } + }; + } break; case 'add': - message.flags = message.flags.concat(update.value.filter(flag => { - if (message.flags.indexOf(flag) < 0) { - updated = true; - return true; - } - return false; - })); + { + let newFlags = []; + message.flags = message.flags.concat(update.value.filter(flag => { + if (!existingFlags.includes(flag.toLowerCase().trim())) { + updated = true; + newFlags.push(flag); + return true; + } + return false; + })); - // add flags - flagsupdate.$addToSet = { - flags: { - $each: update.value + // add flags + if (updated) { + flagsupdate = { + $addToSet: { + flags: { + $each: newFlags + } + } + }; } - }; - break; + break; + } case 'remove': - message.flags = message.flags.filter(flag => { - if (update.value.indexOf(flag) < 0) { - return true; - } - updated = true; - return false; - }); + { + // We need to use the case of existing flags when removing + let oldFlags = []; + let flagsUpdates = update.value.map(flag => flag.toLowerCase().trim()); + message.flags = message.flags.filter(flag => { + if (!flagsUpdates.includes(flag.toLowerCase().trim())) { + return true; + } + oldFlags.push(flag); + updated = true; + return false; + }); - // remove flags - flagsupdate.$pull = { - flags: { - $in: update.value + // remove flags + if (updated) { + flagsupdate = { + $pull: { + flags: { + $in: oldFlags + } + } + }; } - }; - break; + break; + } } - message.flags.forEach(flag => { - // limit mailbox flags by 100 - if (!mailboxFlags.includes(flag.trim().toLowerCase()) && mailboxFlags.length + newFlags.length < 100) { - newFlags.push(flag); - } - }); - if (!update.silent) { + // print updated state of the message session.writeStream.write(session.formatResponse('FETCH', message.uid, { uid: update.isUid ? message.uid : false, flags: message.flags })); } - let updateMailboxFlags = next => { - if (!newFlags.length) { - return next(); - } - - // found some new flags not yet set for mailbox - return db.database.collection('mailboxes').findOneAndUpdate({ - _id: mailbox._id - }, { - $addToSet: { - flags: { - $each: newFlags - } + if (updated) { + db.database.collection('messages').findOneAndUpdate({ + _id: message._id + }, flagsupdate, {}, err => { + if (err) { + return cursor.close(() => done(err)); } - }, {}, next); - }; - updateMailboxFlags(() => { - if (updated) { - db.database.collection('messages').findOneAndUpdate({ - _id: message._id - }, flagsupdate, {}, err => { - if (err) { - return cursor.close(() => done(err)); - } - - notifyEntries.push({ - command: 'FETCH', - ignore: session.id, - uid: message.uid, - flags: message.flags, - message: message._id - }); - - if (notifyEntries.length > 100) { - let entries = notifyEntries; - notifyEntries = []; - setImmediate(() => this.notifier.addEntries(session.user.id, path, entries, processNext)); - return; - } else { - setImmediate(() => processNext()); - } + notifyEntries.push({ + command: 'FETCH', + ignore: session.id, + uid: message.uid, + flags: message.flags, + message: message._id }); - } else { - processNext(); - } - }); + + if (notifyEntries.length > 100) { + // emit notifications in batches of 100 + let entries = notifyEntries; + notifyEntries = []; + setImmediate(() => this.notifier.addEntries(session.user.id, path, entries, processNext)); + return; + } else { + setImmediate(() => processNext()); + } + }); + } else { + processNext(); + } }); }; diff --git a/package.json b/package.json index c947fc0f..c75cd2d9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wildduck", - "version": "1.0.8", + "version": "1.0.9", "description": "IMAP server built with Node.js and MongoDB", "main": "server.js", "scripts": { @@ -23,7 +23,7 @@ "clone": "^2.1.1", "config": "^1.25.1", "grid-fs": "^1.0.1", - "joi": "^10.2.2", + "joi": "^10.3.4", "libbase64": "^0.1.0", "libmime": "^3.1.0", "mailparser": "^2.0.2",