diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..3b141b99 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,17 @@ +language: node_js +sudo: false +services: + - mongodb + - redis-server +node_js: + - 6 + - 8 +notifications: + email: + - andris@kreata.ee + webhooks: + urls: + - https://webhooks.gitter.im/e/0ed18fd9b3e529b3c2cc + on_success: change # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: false # default: false diff --git a/config/lmtp.toml b/config/lmtp.toml index 83cdf1ff..0a759421 100644 --- a/config/lmtp.toml +++ b/config/lmtp.toml @@ -9,7 +9,7 @@ host="127.0.0.1" maxMB=25 # If true then disables STARTTLS usage -disableSTARTTLS=false +disableSTARTTLS=true # Greeting message for connecting client banner="Welcome to Wild Duck Mail Server" diff --git a/docs/api.md b/docs/api.md index 52f83dd6..7bd112f2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1322,7 +1322,7 @@ The search uses MongoDB fulltext index, see [MongoDB docs](https://docs.mongodb. #### GET /users/{user}/mailboxes/{mailbox}/messages/{message} -Returns data about a specific address. +Returns data about a specific message. **Parameters** @@ -1366,6 +1366,54 @@ Response for a successful operation: } ``` +### Get message events + +#### GET /users/{user}/mailboxes/{mailbox}/messages/{message}/events + +Returns timeline information about a specific message. + +**Parameters** + +- **user** (required) is the ID of the user +- **mailbox** (required) is the ID of the mailbox +- **message** (required) is the ID of the message + +**Example** + +``` +curl "http://localhost:8080/users/59467f27535f8f0f067ba8e6/mailboxes/596c9dd31b201716e764efc2/messages/444/events" +``` + +Response for a successful operation: + +```json +{ + "success": true, + "id": 444, + "from": { + "address": "sender@example.com", + "name": "Sender Name" + }, + "to": [ + { + "address": "testuser@example.com", + "name": "Test User" + } + ], + "subject": "Subject line", + "messageId": "", + "date": "2011-11-02T19:19:08.000Z", + "seen": true, + "deleted": false, + "flagged": false, + "draft": false, + "html": [ + "Notice that the HTML content is an array of HTML strings" + ], + "attachments": [] +} +``` + ### Update message details #### PUT /users/{user}/mailboxes/{mailbox}/messages/{message} diff --git a/indexes.yaml b/indexes.yaml index 4a4035a4..6b351dae 100644 --- a/indexes.yaml +++ b/indexes.yaml @@ -358,6 +358,20 @@ indexes: key: updated: 1 +# messagelog +- collection: messagelog + index: + name: messagelog_id_hashed + key: + id: hashed +- collection: threads + index: + name: messagelog_autoexpire + # autoremove messagelog entries after 180 days + expireAfterSeconds: 15552000 + key: + created: 1 + # Indexes for IRC - collection: chat diff --git a/lib/api/messages.js b/lib/api/messages.js index e70af873..1add86c3 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -732,6 +732,135 @@ module.exports = (db, server, messageHandler) => { }); }); + server.get({ name: 'messageevents', path: '/users/:user/mailboxes/:mailbox/messages/:message/events' }, (req, res, next) => { + res.charSet('utf-8'); + + const schema = Joi.object().keys({ + user: Joi.string() + .hex() + .lowercase() + .length(24) + .required(), + mailbox: Joi.string() + .hex() + .lowercase() + .length(24) + .required(), + message: Joi.number() + .min(1) + .required() + }); + + const result = Joi.validate(req.params, schema, { + abortEarly: false, + convert: true + }); + + if (result.error) { + res.json({ + error: result.error.message + }); + return next(); + } + + let user = new ObjectID(result.value.user); + let mailbox = new ObjectID(result.value.mailbox); + let message = result.value.message; + + db.database.collection('messages').findOne({ + mailbox, + uid: message + }, { + fields: { + _id: true, + user: true, + mailbox: true, + uid: true, + meta: true, + outbound: true + } + }, (err, messageData) => { + if (err) { + res.json({ + error: 'MongoDB Error: ' + err.message + }); + return next(); + } + if (!messageData || messageData.user.toString() !== user.toString()) { + res.json({ + error: 'This message does not exist' + }); + return next(); + } + + let getLogEntries = done => { + let logQuery = false; + if (messageData.outbound && messageData.outbound.length === 1) { + logQuery = { + id: messageData.outbound[0] + }; + } else if (messageData.outbound && messageData.outbound.length > 1) { + logQuery = { + id: { $in: messageData.outbound } + }; + } + if (!logQuery) { + return done(null, []); + } + db.database + .collection('messagelog') + .find(logQuery) + .sort({ _id: 1 }) + .toArray(done); + }; + + getLogEntries((err, logEntries) => { + if (err) { + res.json({ + error: 'MongoDB Error: ' + err.message + }); + return next(); + } + + let response = { + success: true, + id: messageData._id, + events: [ + { + action: 'STORE', + source: messageData.meta.source, + origin: messageData.meta.origin, + from: messageData.meta.from, + to: messageData.meta.to, + transtype: messageData.meta.transtype, + time: messageData.meta.time + } + ] + .concat( + logEntries.map(entry => ({ + id: entry.id, + seq: entry.seq, + action: entry.action, + origin: entry.origin || entry.source, + src: entry.ip, + dst: entry.mx, + response: entry.response, + messageId: entry['message-id'], + from: entry.from, + to: entry.forward || (entry.to && [].concat(typeof entry.to === 'string' ? entry.to.split(',') : entry.to || [])), + transtype: entry.transtype, + time: entry.created + })) + ) + .sort((a, b) => a.time - b.time) + }; + + res.json(response); + return next(); + }); + }); + }); + server.get({ name: 'raw', path: '/users/:user/mailboxes/:mailbox/messages/:message/message.eml' }, (req, res, next) => { const schema = Joi.object().keys({ user: Joi.string() diff --git a/lib/autoreply.js b/lib/autoreply.js index d888fe22..9861277f 100644 --- a/lib/autoreply.js +++ b/lib/autoreply.js @@ -100,11 +100,25 @@ module.exports = (options, callback) => { let compiler = new MailComposer(data); let message = maildrop( { + parentId: options.parentId, + reason: 'autoreply', from: '', to: options.sender, interface: 'autoreplies' }, - callback + (err, ...args) => { + if (err || !args[0]) { + return callback(err, ...args); + } + db.database.collection('messagelog').insertOne({ + id: args[0], + parentId: options.parentId, + action: 'AUTOREPLY', + from: '', + to: options.sender, + created: new Date() + }, () => callback(err, ...args)); + } ); compiler diff --git a/lib/filter-handler.js b/lib/filter-handler.js index 69d2dd1b..92ecaa7e 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -263,6 +263,7 @@ class FilterHandler { forward( { + parentId: prepared.id, userData, sender, recipient, @@ -287,6 +288,7 @@ class FilterHandler { autoreply( { + parentId: prepared.id, userData, sender, recipient, @@ -298,6 +300,8 @@ class FilterHandler { ); }; + let outbound = []; + forwardMessage((err, id) => { if (err) { log.error( @@ -312,6 +316,7 @@ class FilterHandler { err.message ); } else if (id) { + outbound.push(id); log.silly( 'LMTP', '%s FRWRDOK id=%s from=%s to=%s target=%s', @@ -320,7 +325,7 @@ class FilterHandler { sender, recipient, Array.from(forwardTargets) - .concat(forwardTargetUrls) + .concat(Array.from(forwardTargetUrls)) .join(',') ); } @@ -329,6 +334,7 @@ class FilterHandler { if (err) { log.error('LMTP', '%s AUTOREPLYFAIL from=%s to=%s error=%s', prepared.id.toString(), '<>', sender, err.message); } else if (id) { + outbound.push(id); log.silly('LMTP', '%s AUTOREPLYOK id=%s from=%s to=%s', prepared.id.toString(), id, '<>', sender); } @@ -388,6 +394,10 @@ class FilterHandler { skipExisting: true }; + if (outbound && outbound.length) { + messageOpts.outbound = [].concat(outbound || []); + } + this.messageHandler.encryptMessage(userData.encryptMessages ? userData.pubKey : false, { chunks, chunklen }, (err, encrypted) => { if (!err && encrypted) { messageOpts.prepared = this.messageHandler.prepareMessage({ diff --git a/lib/forward.js b/lib/forward.js index f5d0fe9c..afae28be 100644 --- a/lib/forward.js +++ b/lib/forward.js @@ -2,6 +2,7 @@ const config = require('wild-config'); const maildrop = require('./maildrop'); +const db = require('./db'); module.exports = (options, callback) => { if (!config.sender.enabled) { @@ -10,6 +11,9 @@ module.exports = (options, callback) => { let message = maildrop( { + parentId: options.parentId, + reason: 'forward', + from: options.sender, to: options.recipient, @@ -19,7 +23,22 @@ module.exports = (options, callback) => { interface: 'forwarder' }, - callback + (err, ...args) => { + if (err || !args[0]) { + return callback(err, ...args); + } + db.database.collection('messagelog').insertOne({ + id: args[0], + action: 'FORWARD', + parentId: options.parentId, + from: options.sender, + to: options.recipient, + forward: options.forward, + http: !!options.targetUrl, + targeUrl: options.targetUrl, + created: new Date() + }, () => callback(err, ...args)); + } ); setImmediate(() => { diff --git a/lib/handlers/on-append.js b/lib/handlers/on-append.js index 97c13d29..feb83e4e 100644 --- a/lib/handlers/on-append.js +++ b/lib/handlers/on-append.js @@ -37,8 +37,11 @@ module.exports = (server, messageHandler) => (path, flags, date, raw, session, c path, meta: { source: 'IMAP', - to: session.user.username, - time: Date.now() + from: '', + to: [session.user.address || session.user.username], + origin: session.remoteAddress, + transtype: 'APPEND', + time: new Date() }, session, date, diff --git a/lib/handlers/on-copy.js b/lib/handlers/on-copy.js index f969fc0a..3c983910 100644 --- a/lib/handlers/on-copy.js +++ b/lib/handlers/on-copy.js @@ -140,7 +140,14 @@ module.exports = (server, messageHandler) => (path, update, session, callback) = if (!message.meta) { message.meta = {}; } - message.meta.source = 'IMAPCOPY'; + + if (!message.meta.events) { + message.meta.events = []; + } + message.meta.events.push({ + action: 'IMAPCOPY', + time: new Date() + }); db.database.collection('messages').insertOne(message, err => { if (err) { diff --git a/lib/maildrop.js b/lib/maildrop.js index 2462fd96..faa934a2 100644 --- a/lib/maildrop.js +++ b/lib/maildrop.js @@ -147,6 +147,14 @@ module.exports = (options, callback) => { } }; + if (options.parentId) { + envelope.parentId = options.parentId; + } + + if (options.reason) { + envelope.reason = options.reason; + } + let deliveries = []; if (options.targeUrl) { diff --git a/lib/message-handler.js b/lib/message-handler.js index 32e4b513..1f8779d1 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -82,7 +82,6 @@ class MessageHandler { // TODO: Refactor into smaller pieces add(options, callback) { let prepared = options.prepared || this.prepareMessage(options); - let id = prepared.id; let mimeTree = prepared.mimeTree; let size = prepared.size; @@ -175,6 +174,10 @@ class MessageHandler { subject }; + if (options.outbound) { + messageData.outbound = [].concat(options.outbound || []); + } + if (maildata.attachments && maildata.attachments.length) { messageData.attachments = maildata.attachments; messageData.ha = true; @@ -364,9 +367,39 @@ class MessageHandler { let existingId = messageData._id; let existingUid = messageData.uid; + let outbound = [].concat(messageData.outbound || []).concat(options.outbound || []); + if (outbound) { + messageData.outbound = outbound; + } if (options.skipExisting) { // message already exists, just skip it + if (options.outbound) { + // new outbound ID's. update + return this.database.collection('messages').findOneAndUpdate({ + _id: messageData._id, + mailbox: messageData.mailbox, + uid: messageData.uid + }, { + $addToSet: { + outbound: { $each: [].concat(options.outbound || []) } + } + }, { + returnOriginal: true, + projection: { + _id: true, + outbound: true + } + }, () => + callback(null, true, { + uid: existingUid, + id: existingId, + mailbox: mailboxData._id, + status: 'skip' + }) + ); + } + return callback(null, true, { uid: existingUid, id: existingId, @@ -493,10 +526,10 @@ class MessageHandler { } del(options, callback) { - let message = options.message; + let messageData = options.message; this.getMailbox( options.mailbox || { - mailbox: message.mailbox + mailbox: messageData.mailbox }, (err, mailboxData) => { if (err) { @@ -504,9 +537,9 @@ class MessageHandler { } this.database.collection('messages').deleteOne({ - _id: message._id, + _id: messageData._id, mailbox: mailboxData._id, - uid: message.uid + uid: messageData.uid }, err => { if (err) { return callback(err); @@ -515,21 +548,21 @@ class MessageHandler { this.updateQuota( mailboxData._id, { - storageUsed: -message.size + storageUsed: -messageData.size }, () => { let updateAttachments = next => { - let attachmentIds = Object.keys(message.mimeTree.attachmentMap || {}).map(key => message.mimeTree.attachmentMap[key]); + let attachmentIds = Object.keys(messageData.mimeTree.attachmentMap || {}).map(key => messageData.mimeTree.attachmentMap[key]); if (!attachmentIds.length) { return next(); } - this.attachmentStorage.deleteMany(attachmentIds, message.magic, next); + this.attachmentStorage.deleteMany(attachmentIds, messageData.magic, next); }; updateAttachments(() => { if (options.session && options.session.selected && options.session.selected.mailbox === mailboxData.path) { - options.session.writeStream.write(options.session.formatResponse('EXPUNGE', message.uid)); + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageData.uid)); } this.notifier.addEntries( @@ -538,9 +571,9 @@ class MessageHandler { { command: 'EXPUNGE', ignore: options.session && options.session.id, - uid: message.uid, - message: message._id, - unseen: message.unseen + uid: messageData.uid, + message: messageData._id, + unseen: messageData.unseen }, () => { this.notifier.fire(mailboxData.user, mailboxData.path); diff --git a/lmtp.js b/lmtp.js index 79fadea9..f2945f69 100644 --- a/lmtp.js +++ b/lmtp.js @@ -170,12 +170,12 @@ const serverOptions = { transactionId, source: 'LMTP', from: sender, - to: recipient, + to: [recipient], origin: session.remoteAddress, originhost: session.clientHostname, transhost: session.hostNameAppearsAs, transtype: session.transmissionType, - time: Date.now() + time: new Date() } }, (err, response, preparedResponse) => { diff --git a/package.json b/package.json index ec9913a1..1f1dd800 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "grunt-mocha-test": "^0.13.3", "grunt-shell-spawn": "^0.3.10", "grunt-wait": "^0.1.0", - "icedfrisby": "^1.4.0", + "icedfrisby": "^1.5.0", "mailparser": "^2.1.0", "mocha": "^4.0.1", "request": "^2.83.0" @@ -36,7 +36,7 @@ "iconv-lite": "0.4.19", "ioredfour": "1.0.2-ioredis", "ioredis": "3.1.4", - "joi": "13.0.0", + "joi": "13.0.1", "js-yaml": "3.10.0", "libbase64": "0.2.0", "libmime": "3.1.0", @@ -50,7 +50,7 @@ "npmlog": "4.1.2", "openpgp": "2.5.12", "qrcode": "0.9.0", - "restify": "6.0.1", + "restify": "6.2.3", "seq-index": "1.1.0", "smtp-server": "3.3.0", "speakeasy": "2.0.0",