From 5cb89fb7da7bae4d200260cbcb5262a28cc40dda Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Sun, 9 Apr 2017 12:33:10 +0300 Subject: [PATCH] Working POP3 implementation --- .gitignore | 1 + README.md | 31 +++++--- config/default.js | 10 ++- imap.js | 141 ++++++---------------------------- lib/message-handler.js | 166 +++++++++++++++++++++++++++++++++++++++++ lib/pop3-connection.js | 12 ++- pop3.js | 66 +++++++++++++--- 7 files changed, 286 insertions(+), 141 deletions(-) diff --git a/.gitignore b/.gitignore index 812bd9e7..ba5ca2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules .DS_Store npm-debug.log .npmrc +config/development.js diff --git a/README.md b/README.md index 677723dd..7cbaddb5 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ ![](https://cldup.com/qlZnwOz0na.jpg) -Wild Duck is a distributed IMAP server built with Node.js, MongoDB and Redis. Node.js runs the application, MongoDB is used as the mail store and Redis is used for ephemeral actions like publish/subscribe, locking and caching. +Wild Duck is a distributed IMAP/POP3 server built with Node.js, MongoDB and Redis. Node.js runs the application, MongoDB is used as the mail store and Redis is used for ephemeral actions like publish/subscribe, locking and caching. > **NB!** Wild Duck is currently in **beta**. You should not use it in production. ## Goals of the Project -1. Build a scalable and distributed IMAP server that uses clustered database instead of single machine file system as mail store +1. Build a scalable and distributed IMAP/POP3 server that uses clustered database instead of single machine file system as mail store 2. Allow using internationalized email addresses 3. Provide Gmail-like features like pushing sent messages automatically to Sent Mail folder or notifying about messages moved to Junk folder so these could be marked as spam 4. Provide parsed mailbox and message data over HTTP. This should make creating webmail interfaces super easy, no need to parse RFC822 messages to get text content or attachments @@ -45,16 +45,27 @@ Wild Duck more or less passes the [ImapTest](https://www.imapwiki.org/ImapTest/T ### POP3 Support -POP3 supports the following commands +In addition to the required POP3 commands ([RFC1939](https://tools.ietf.org/html/rfc1939)) Wild Duck supports the following extensions: -* **NOOP** -* **QUIT** -* **USER** -* **PASS** -* **CAPA** -* **AUTH PLAIN** + * **UIDL** + * **USER** + * **PASS** + * **SASL PLAIN** + * **PIPELINING** -> **TODO:** implement missing commands. See also https://support.google.com/a/answer/6089246?hl=en +Notably missing is the **TOP** extension. + +#### LIST + +POP3 listing displays the newest 250 messages in INBOX (configurable) + +#### RETR + +If a messages is downloaded by a client this message gets marked as Seen + +#### DELE + +If a messages is deleted by a client this message gets marked as Seen and moved to Trash folder ## FAQ diff --git a/config/default.js b/config/default.js index 7a6c2927..072d2ba1 100644 --- a/config/default.js +++ b/config/default.js @@ -49,6 +49,9 @@ module.exports = { smtp: { enabled: true, port: 2525, + // If certificate path is not defined, use built-in self-signed certs for STARTTLS + //key: '/path/to/server/key.pem' + //cert: '/path/to/server/cert.pem' host: '0.0.0.0', maxMB: 5 }, @@ -57,7 +60,12 @@ module.exports = { enabled: true, port: 9995, host: '0.0.0.0', - secure: true + // If certificate path is not defined, use built-in self-signed certs + //key: '/path/to/server/key.pem' + //cert: '/path/to/server/cert.pem' + secure: true, + // how many latest messages to list for LIST and UIDL + maxMessages: 250 }, api: { diff --git a/imap.js b/imap.js index ccbe520a..a175fb77 100644 --- a/imap.js +++ b/imap.js @@ -486,6 +486,7 @@ server.onStore = function (path, update, session, callback) { if (err) { return callback(err); } + if (!mailbox) { return callback(null, 'NONEXISTENT'); } @@ -992,130 +993,30 @@ server.onCopy = function (path, update, session, callback) { // MOVE / UID MOVE sequence mailbox server.onMove = function (path, update, session, callback) { this.logger.debug('[%s] Moving messages from "%s" to "%s"', session.id, path, update.destination); - db.database.collection('mailboxes').findOne({ - user: session.user.id, - path - }, (err, mailbox) => { - if (err) { - return callback(err); - } - if (!mailbox) { - return callback(null, 'NONEXISTENT'); - } - db.database.collection('mailboxes').findOne({ + messageHandler.move({ + user: session.user.id, + // folder to move messages from + source: { + user: session.user.id, + path + }, + // folder to move messages to + destination: { user: session.user.id, path: update.destination - }, (err, target) => { - if (err) { - return callback(err); + }, + session, + // list of UIDs to move + messages: update.messages + }, (...args) => { + if (args[0]) { + if (args[0].imapResponse) { + return callback(null, args[0].imapResponse); } - if (!target) { - return callback(null, 'TRYCREATE'); - } - - let cursor = db.database.collection('messages').find({ - mailbox: mailbox._id, - uid: { - $in: update.messages - } - }).project({ - uid: 1 - }).sort([ - ['uid', 1] - ]); - - let sourceUid = []; - let destinationUid = []; - - let processNext = () => { - cursor.next((err, message) => { - if (err) { - return callback(err); - } - if (!message) { - return cursor.close(() => { - db.database.collection('mailboxes').findOneAndUpdate({ - _id: mailbox._id - }, { - $inc: { - // increase the mailbox modification index - // to indicate that something happened - modifyIndex: 1 - } - }, { - uidNext: true - }, () => { - this.notifier.fire(session.user.id, target.path); - return callback(null, true, { - uidValidity: target.uidValidity, - sourceUid, - destinationUid - }); - }); - }); - } - - sourceUid.unshift(message.uid); - db.database.collection('mailboxes').findOneAndUpdate({ - _id: target._id - }, { - $inc: { - uidNext: 1 - } - }, { - uidNext: true - }, (err, item) => { - if (err) { - return callback(err); - } - - if (!item || !item.value) { - // was not able to acquire a lock - return callback(null, 'TRYCREATE'); - } - - let uidNext = item.value.uidNext; - destinationUid.unshift(uidNext); - - // update message, change mailbox from old to new one - db.database.collection('messages').findOneAndUpdate({ - _id: message._id - }, { - $set: { - mailbox: target._id, - // new mailbox means new UID - uid: uidNext, - // this will be changed later by the notification system - modseq: 0 - } - }, err => { - if (err) { - return callback(err); - } - - session.writeStream.write(session.formatResponse('EXPUNGE', message.uid)); - - // mark messages as deleted from old mailbox - this.notifier.addEntries(session.user.id, path, { - command: 'EXPUNGE', - ignore: session.id, - uid: message.uid - }, () => { - // mark messages as added to old mailbox - this.notifier.addEntries(session.user.id, target.path, { - command: 'EXISTS', - uid: uidNext, - message: message._id - }, processNext); - }); - }); - }); - }); - }; - - processNext(); - }); + return callback(args[0]); + } + callback(...args); }); }; diff --git a/lib/message-handler.js b/lib/message-handler.js index fee9a3bd..b4f90ccb 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -9,6 +9,9 @@ const ImapNotifier = require('./imap-notifier'); const tools = require('./tools'); const libmime = require('libmime'); +// home many modifications to cache before writing +const BULK_BATCH_SIZE = 150; + class MessageHandler { constructor(database) { @@ -361,6 +364,169 @@ class MessageHandler { }); }); } + + move(options, callback) { + this.getMailbox(options.source, (err, mailbox) => { + if (err) { + return callback(err); + } + + this.getMailbox(options.destination, (err, target) => { + if (err) { + return callback(err); + } + + this.database.collection('mailboxes').findOneAndUpdate({ + _id: mailbox._id + }, { + $inc: { + // increase the mailbox modification index + // to indicate that something happened + modifyIndex: 1 + } + }, { + uidNext: true + }, () => { + + let cursor = this.database.collection('messages').find({ + mailbox: mailbox._id, + uid: { + $in: options.messages || [] + } + }).project({ + uid: 1 + }).sort([ + ['uid', 1] + ]); + + let sourceUid = []; + let destinationUid = []; + + let removeEntries = []; + let existsEntries = []; + + let done = err => { + + let next = () => { + if (err) { + return callback(err); + } + return callback(null, true, { + uidValidity: target.uidValidity, + sourceUid, + destinationUid + }); + }; + + if (existsEntries.length) { + // mark messages as deleted from old mailbox + return this.notifier.addEntries(mailbox, false, removeEntries, () => { + // mark messages as added to new mailbox + this.notifier.addEntries(target, false, existsEntries, () => { + this.notifier.fire(mailbox.user, mailbox.path); + this.notifier.fire(target.user, target.path); + next(); + }); + }); + } + next(); + }; + + let processNext = () => { + cursor.next((err, message) => { + if (err) { + return done(err); + } + if (!message) { + return cursor.close(done); + } + + sourceUid.unshift(message.uid); + this.database.collection('mailboxes').findOneAndUpdate({ + _id: target._id + }, { + $inc: { + uidNext: 1 + } + }, { + uidNext: true + }, (err, item) => { + if (err) { + return done(err); + } + + if (!item || !item.value) { + return done(new Error('Mailbox disappeared')); + } + + let uidNext = item.value.uidNext; + destinationUid.unshift(uidNext); + + let updateOptions = { + $set: { + mailbox: target._id, + // new mailbox means new UID + uid: uidNext, + // this will be changed later by the notification system + modseq: 0 + } + }; + + if (options.markAsSeen) { + updateOptions.$set.seen = true; + updateOptions.$addToSet = { + flags: '\\Seen' + }; + } + + // update message, change mailbox from old to new one + this.database.collection('messages').findOneAndUpdate({ + _id: message._id + }, updateOptions, err => { + if (err) { + return done(err); + } + + if (options.session) { + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', message.uid)); + } + + removeEntries.push({ + command: 'EXPUNGE', + ignore: options.session && options.session.id, + uid: message.uid + }); + + existsEntries.push({ + command: 'EXISTS', + uid: uidNext, + message: message._id + }); + + if (existsEntries.length >= BULK_BATCH_SIZE) { + // mark messages as deleted from old mailbox + return this.notifier.addEntries(mailbox, false, removeEntries, () => { + // mark messages as added to new mailbox + this.notifier.addEntries(target, false, existsEntries, () => { + removeEntries = []; + existsEntries = []; + this.notifier.fire(mailbox.user, mailbox.path); + this.notifier.fire(target.user, target.path); + processNext(); + }); + }); + } + processNext(); + }); + }); + }); + }; + + processNext(); + }); + }); + }); + } } module.exports = MessageHandler; diff --git a/lib/pop3-connection.js b/lib/pop3-connection.js index b653d305..426b0898 100644 --- a/lib/pop3-connection.js +++ b/lib/pop3-connection.js @@ -2,6 +2,7 @@ const crypto = require('crypto'); const EventEmitter = require('events'); +const packageData = require('../package.json'); const SOCKET_TIMEOUT = 60 * 1000; @@ -197,12 +198,16 @@ class POP3Connection extends EventEmitter { // https://tools.ietf.org/html/rfc2449#section-5 command_CAPA(args, next) { let extensions = [ + 'CAPA', // 'TOP', 'UIDL', 'USER', 'RESP-CODES', // https://tools.ietf.org/html/rfc5034#section-6 - 'SASL PLAIN' + 'SASL PLAIN', + // https://tools.ietf.org/html/rfc2449#section-6.6 + 'PIPELINING', + 'IMPLEMENTATION WildDuck-v' + packageData.version ]; this.send(['+OK Capability list follows'].concat(extensions)); @@ -562,6 +567,11 @@ class POP3Connection extends EventEmitter { let message = this.session.listing.messages[index - 1]; + if (message.popped) { + this.send('-ERR message ' + index + ' already deleted'); + return next(); + } + this._server.onFetchMessage(message.id, this.session, (err, stream) => { if (err) { return next(err); diff --git a/pop3.js b/pop3.js index 519f35fe..451fa09b 100644 --- a/pop3.js +++ b/pop3.js @@ -84,7 +84,7 @@ const serverOptions = { seen: true }).sort([ ['uid', -1] - ]).limit(MAX_MESSAGES).toArray((err, messages) => { + ]).limit(config.pop3.maxMessages || MAX_MESSAGES).toArray((err, messages) => { if (err) { return callback(err); } @@ -131,16 +131,28 @@ const serverOptions = { let handleSeen = next => { if (update.seen && update.seen.length) { - return markAsSeen(session.user.mailbox, update.seen, next); + return markAsSeen(session, update.seen, next); } next(); }; - handleSeen(err => { + let handleDeleted = next => { + if (update.deleted && update.deleted.length) { + return trashMessages(session, update.deleted, next); + } + next(); + }; + + handleSeen((err, seenCount) => { if (err) { return log.error('POP3', err); } - // TODO: delete marked messages + handleDeleted((err, deleteCount) => { + if (err) { + return log.error('POP3', err); + } + log.info('POP3', '[%s] Deleted %s messages, marked %s messages as seen', session.user.username, deleteCount, seenCount); + }); }); // return callback without waiting for the update result @@ -158,12 +170,48 @@ if (config.pop3.cert) { const server = new POP3Server(serverOptions); -// TODO: mark as seen immediatelly after RETR instead of batching later? -function markAsSeen(mailbox, messages, callback) { +// move messages to trash +function trashMessages(session, messages, callback) { + // find Trash folder + db.database.collection('mailboxes').findOne({ + user: session.user.id, + specialUse: '\\Trash' + }, (err, trashMailbox) => { + if (err) { + return callback(err); + } + + if (!trashMailbox) { + return callback(new Error('Trash mailbox not found for user')); + } + + messageHandler.move({ + user: session.user.id, + // folder to move messages from + source: { + mailbox: session.user.mailbox + }, + // folder to move messages to + destination: trashMailbox, + // list of UIDs to move + messages: messages.map(message => message.uid), + + // add \Seen flags to deleted messages + markAsSeen: true + }, (err, success, meta) => { + if (err) { + return callback(err); + } + callback(null, success && meta && meta.destinationUid && meta.destinationUid.length || 0); + }); + }); +} + +function markAsSeen(session, messages, callback) { let ids = messages.map(message => new ObjectID(message.id)); return db.database.collection('mailboxes').findOneAndUpdate({ - _id: mailbox + _id: session.user.mailbox }, { $inc: { modifyIndex: 1 @@ -181,7 +229,7 @@ function markAsSeen(mailbox, messages, callback) { } db.database.collection('messages').updateMany({ - mailbox, + mailbox: mailboxData._id, _id: { $in: ids }, @@ -214,7 +262,7 @@ function markAsSeen(mailbox, messages, callback) { return result; }), () => { messageHandler.notifier.fire(mailboxData.user, mailboxData.path); - callback(); + callback(null, messages.count); }); }); });