diff --git a/README.md b/README.md index 53856b2a..1dd33bf6 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,8 @@ Actual update data (information about new and deleted messages, flag updates and ## HTTP API +> **NB!** The HTTP API is being re-designed, do not build apps against the current API for now + Users, mailboxes and messages can be managed with HTTP requests against Wild Duck API TODO: @@ -378,267 +380,7 @@ Where Recipient limits assume that messages are sent using ZoneMTA with [zonemta-wildduck](https://github.com/wildduck-email/zonemta-wildduck) plugin, otherwise the counters are not updated. -### GET /user/mailboxes - -Returns all mailbox names for the user - -Arguments - -- **username** is the username of the user to modify - -**Example** - -``` -curl "http://localhost:8080/user/mailboxes?username=testuser" -``` - -The response for successful operation should look like this: - -```json -{ - "success": true, - "username": "testuser", - "mailboxes": [ - { - "id": "58d8f2ae240366dfd5d8049c", - "path": "INBOX", - "special": "Inbox", - "messages": 100 - }, - { - "id": "58d8f2ae240366dfd5d8049d", - "path": "Sent Mail", - "special": "Sent", - "messages": 45 - }, - { - "id": "58d8f2ae240366dfd5d8049f", - "path": "Junk", - "special": "Junk", - "messages": 10 - }, - { - "id": "58d8f2ae240366dfd5d8049e", - "path": "Trash", - "special": "Trash", - "messages": 11 - } - ] -} -``` - -### GET /mailbox/:id - -List messages in a mailbox. - -Parameters - -- **id** is the mailbox ID -- **size** is optional number to limit the length of the messages array (defaults to 20) -- **before** is an optional paging number (see _next_ in response) -- **after** is an optional paging number (see _prev_ in response) - -Response includes the following fields - -- **mailbox** is an object that lists some metadata about the current mailbox - - - **id** is the mailbox ID - - **path** is the folder path - -- **next** is an URL fragment for retrieving the next page (or false if there are no more pages) - -- **prev** is an URL fragment for retrieving the previous page (or false if it is the first page) - -- **messages** is an array of messages in the mailbox - - - **id** is the message ID - - **date** is the date when this message was received - - **ha** is a boolean that indicates if this messages has attachments or not - - **intro** includes the first 256 characters from the message - - **subject** is the message title - - **from** is the From: field - - **to** is the To: field - - **cc** is the Cc: field - - **bcc** is the Bcc: field - -The response for successful listing should look like this: - -```json -{ - "success": true, - "mailbox": { - "id": "58dbf87fcff690a8c30470c7", - "path": "INBOX" - }, - "next": "/mailbox/58dbf87fcff690a8c30470c7?before=34&size=20", - "prev": false, - "messages": [ - { - "id": "58e25243ab71621c3890417e", - "date": "2017-04-03T13:46:44.226Z", - "ha": true, - "intro": "Welcome to Ryan Finnie's MIME torture test. This message was designed to introduce a couple of the newer features of MIME-aware MUAs, features that have come around since the days of the original MIME torture test. Just to be clear, this message SUPPLEMENT…", - "subject": "ryan finnie's mime torture test v1.0", - "from": "ryan finnie ", - "to": "bob@domain.dom" - } - ] -} -``` - -### GET /message/:id - -Retrieves message information - -Parameters - -- **id** is the MongoDB _id as a string for a message -- **mailbox** is optional Mailbox id. Use this to verify that the message is located at this mailbox - -**Example** - -``` -curl "http://localhost:8080/message/58d8299c5195c38e77c2daa5" -``` - -Response message includes the following fields - -- **id** is the id of the message -- **headers** is an array that lists all headers of the message. A header is an object: - - - **key** is the lowercase key of the header - - **value** is the header value in unicode (all encoded values are decoded to utf-8). The value is capped at around 800 characters. - -- **date** is the receive date (not header Date: field) - -- **mailbox** is the id of the mailbox this messages belongs to - -- **flags** is an array of IMAP flags for this message -- **text** is the plaintext version of the message (derived from html if not present in message source) -- **html** is the HTML version of the message (derived from plaintext if not present in message source). It is an array of strings, each array element corresponds to different MIME node and might have its own html header - -- **attachments** is an array of attachment objects. Attachments can be shared between messages. - - - **id** is the id of the attachment in the form of "ATT00001" - - **fileName** is the name of the attachment. Autogenerated from Content-Type if not set in source - - **contentType** is the MIME type of the message - - **disposition** defines Content-Disposition and is either 'inline', 'attachment' or _false_ - - **transferEncoding** defines Content-Transfer-Encoding - - **related** is a boolean value that states if the attachment should be hidden (_true_) or not. _Related_ attachments are usually embedded images - - **sizeKb** is the approximate size of the attachment in kilobytes - -#### Embedded images - -HTML content has embedded images linked with the following URL structure: - -``` -attachment:MESSAGE_ID/ATTACHMENT_ID -``` - -For example: - -``` - -``` - -To fetch the actual attachment contents for this image, use the following url: - -``` -http://localhost:8080/message/aaaaaa/attachment/bbbbbb -``` - -#### Example response - -The response for successful operation should look like this: - -```json -{ - "success": true, - "message": { - "id": "58d8299c5195c38e77c2daa5", - "mailbox": "58dbf87fcff690a8c30470c7", - "headers": [ - { - "key": "delivered-to", - "value": "andris@addrgw.com" - } - ], - "date": "2017-04-03T10:34:43.007Z", - "flags": ["\\Seen"], - "text": "Hello world!", - "html": ["

Hello world!

"], - "attachments": [ - { - "id": "ATT00001", - "fileName": "image.png", - "contentType": "image/png", - "disposition": "attachment", - "transferEncoding": "base64", - "related": true, - "sizeKb": 1 - } - ] - } -} -``` - -### GET /message/:mid/attachment/:aid - -Retrieves an attachment of the message - -Parameters - -- **mid** is the message ID -- **aid** is the attachment ID - -**Example** - -``` -curl "http://localhost:8080/message/58d8299c5195c38e77c2daa5/attachment/ATT00001" -``` - -### GET /message/:id/raw - -Retrieves RFC822 source of the message - -Parameters - -- **id** is the MongoDB _id as a string for a message -- **mailbox** is optional Mailbox id. Use this to verify that the message is located at this mailbox - -**Example** - -``` -curl "http://localhost:8080/message/58d8299c5195c38e77c2daa5/raw" -``` - -### DELETE /message/:id - -Deletes a message from a mailbox. - -Parameters - -- **id** is the MongoDB _id as a string for a message -- **mailbox** is an optional Mailbox id. Use this to verify that the message to be deleted is located at this mailbox - -**Example** - -``` -curl -XDELETE "http://localhost:8080/message/58d8299c5195c38e77c2daa5" -``` - -The response for successful operation should look like this: - -```json -{ - "success": true, - "message":{ - "id": "58d8299c5195c38e77c2daa5" - } -} -``` - -### Message filtering +## Message filtering Wild Duck has built-in message filtering in LMTP server. This is somewhat similar to Sieve even though the filters are not scripts. @@ -696,6 +438,14 @@ Filters are configuration objects stored in the `filters` array of the users obj **NB!** If you do not care about an action field then do not set it, otherwise matches from other filters do not apply +## Sharding + +Shard the following collections by these keys: + +* Collection: `messages`, key: `user` (by hash?) +* Collection: `attachment.files`, key: `_id` (by hash) +* Collection: `attachment.chunks`, key: `file_id` (by hash) + ## IMAP Protocol Differences This is a list of known differences from the IMAP specification. Listed differences are either intentional or are bugs that became features. diff --git a/api.js b/api.js index f60009c8..7d975c66 100644 --- a/api.js +++ b/api.js @@ -6,18 +6,13 @@ const log = require('npmlog'); const Joi = require('joi'); const bcrypt = require('bcryptjs'); const tools = require('./lib/tools'); -const MessageHandler = require('./lib/message-handler'); const UserHandler = require('./lib/user-handler'); const db = require('./lib/db'); -const ObjectID = require('mongodb').ObjectID; -const libqp = require('libqp'); -const libbase64 = require('libbase64'); const server = restify.createServer({ name: 'Wild Duck API' }); -let messageHandler; let userHandler; server.use(restify.plugins.queryParser()); @@ -599,700 +594,6 @@ server.get('/user', (req, res, next) => { }); }); -server.get('/user/mailboxes', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - username: Joi.string().alphanum().lowercase().min(3).max(30).required() - }); - - const result = Joi.validate( - { - username: req.query.username - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let username = result.value.username; - - db.database.collection('users').findOne({ - username - }, (err, userData) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - username - }); - return next(); - } - if (!userData) { - res.json({ - error: 'This user does not exist', - username - }); - return next(); - } - - db.database - .collection('mailboxes') - .find({ - user: userData._id - }) - .toArray((err, mailboxes) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - username - }); - return next(); - } - - if (!mailboxes) { - mailboxes = []; - } - - let priority = { - Inbox: 1, - Sent: 2, - Junk: 3, - Trash: 4 - }; - - res.json({ - success: true, - username, - mailboxes: mailboxes - .map(mailbox => ({ - id: mailbox._id.toString(), - path: mailbox.path, - special: mailbox.path === 'INBOX' ? 'Inbox' : mailbox.specialUse ? mailbox.specialUse.replace(/^\\/, '') : false - })) - .sort((a, b) => { - if (a.special && !b.special) { - return -1; - } - - if (b.special && !a.special) { - return 1; - } - - if (a.special && b.special) { - return (priority[a.special] || 5) - (priority[b.special] || 5); - } - - return a.path.localeCompare(b.path); - }) - }); - return next(); - }); - }); -}); - -// FIXME: if listing a page after the last one then there is no prev URL -// Probably should detect the last page the same way the first one is detected -server.get('/mailbox/:id', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - id: Joi.string().hex().lowercase().length(24).required(), - before: Joi.number().default(0), - after: Joi.number().default(0), - size: Joi.number().min(1).max(50).default(20) - }); - - const result = Joi.validate( - { - id: req.params.id, - before: req.params.before, - after: req.params.after, - size: req.params.size - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let id = result.value.id; - let before = result.value.before; - let after = result.value.after; - let size = result.value.size; - - db.database.collection('mailboxes').findOne({ - _id: new ObjectID(id) - }, (err, mailbox) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - if (!mailbox) { - res.json({ - error: 'This mailbox does not exist', - id - }); - return next(); - } - - let query = { - mailbox: mailbox._id - }; - let reverse = false; - let sort = [['uid', -1]]; - - if (req.params.before) { - query.uid = { - $lt: before - }; - } else if (req.params.after) { - query.uid = { - $gt: after - }; - sort = [['uid', 1]]; - reverse = true; - } - - db.database.collection('messages').findOne({ - mailbox: mailbox._id - }, { - fields: { - uid: true - }, - sort: [['uid', -1]] - }, (err, entry) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - - if (!entry) { - res.json({ - success: true, - mailbox: { - id: mailbox._id, - path: mailbox.path - }, - next: false, - prev: false, - messages: [] - }); - return next(); - } - - let newest = entry.uid; - - db.database.collection('messages').findOne({ - mailbox: mailbox._id - }, { - fields: { - uid: true - }, - sort: [['uid', 1]] - }, (err, entry) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - - if (!entry) { - res.json({ - error: 'Unexpected result' - }); - return next(); - } - - let oldest = entry.uid; - - db.database - .collection('messages') - .find(query, { - uid: true, - mailbox: true, - idate: true, - headers: true, - ha: true, - intro: true - }) - .sort(sort) - .limit(size) - .toArray((err, messages) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - - if (reverse) { - messages = messages.reverse(); - } - - let nextPage = false; - let prevPage = false; - - if (messages.length) { - if (after || before) { - prevPage = messages[0].uid; - if (prevPage >= newest) { - prevPage = false; - } - } - if (messages.length >= size) { - nextPage = messages[messages.length - 1].uid; - if (nextPage < oldest) { - nextPage = false; - } - } - } - - res.json({ - success: true, - mailbox: { - id: mailbox._id, - path: mailbox.path - }, - next: nextPage ? '/mailbox/' + id + '?before=' + nextPage + '&size=' + size : false, - prev: prevPage ? '/mailbox/' + id + '?after=' + prevPage + '&size=' + size : false, - messages: messages.map(message => { - let response = { - id: message._id, - date: message.idate, - ha: message.ha, - intro: message.intro - }; - - message.headers.forEach(entry => { - if (['subject', 'from', 'to', 'cc', 'bcc'].includes(entry.key)) { - response[entry.key] = entry.value; - } - }); - return response; - }) - }); - - return next(); - }); - }); - }); - }); -}); - -server.get('/message/:id', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - id: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).optional() - }); - - const result = Joi.validate( - { - id: req.params.id, - mailbox: req.params.mailbox - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let id = result.value.id; - let mailbox = result.value.mailbox; - - let query = { - _id: new ObjectID(id) - }; - - if (mailbox) { - query.mailbox = new ObjectID(mailbox); - } - - db.database.collection('messages').findOne(query, { - mailbox: true, - headers: true, - html: true, - text: true, - attachments: true, - idate: true, - flags: true - }, (err, message) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - if (!message) { - res.json({ - error: 'This message does not exist', - id - }); - return next(); - } - - res.json({ - success: true, - message: { - id, - mailbox: message.mailbox, - headers: message.headers, - date: message.idate, - flags: message.flags, - text: message.text, - html: message.html, - attachments: message.attachments - } - }); - - return next(); - }); -}); - -server.get('/message/:id/raw', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - id: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).optional() - }); - - const result = Joi.validate( - { - id: req.params.id, - mailbox: req.params.mailbox - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let id = result.value.id; - let mailbox = result.value.mailbox; - - let query = { - _id: new ObjectID(id) - }; - - if (mailbox) { - query.mailbox = new ObjectID(mailbox); - } - - db.database.collection('messages').findOne(query, { - mimeTree: true, - map: true, - size: true - }, (err, message) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - if (!message) { - res.json({ - error: 'This message does not exist', - id - }); - return next(); - } - - let response = messageHandler.indexer.rebuild(message.mimeTree); - if (!response || response.type !== 'stream' || !response.value) { - res.json({ - error: 'Can not fetch message', - id - }); - return next(); - } - - res.writeHead(200, { - 'Content-Type': 'message/rfc822' - }); - response.value.pipe(res); - }); -}); - -server.get('/message/:message/attachment/:attachment', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - message: Joi.string().hex().lowercase().length(24).required(), - attachment: Joi.string().regex(/^ATT\d+$/i).uppercase().required() - }); - - const result = Joi.validate( - { - message: req.params.message, - attachment: req.params.attachment - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let messageId = result.value.message; - let attachmentMid = result.value.attachment; - - db.database.collection('messages').findOne({ - _id: new ObjectID(messageId) - }, { - fields: { - map: true - } - }, (err, message) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - attachment: attachmentMid, - message: messageId - }); - return next(); - } - if (!message) { - res.json({ - error: 'This message does not exist', - attachment: attachmentMid, - message: messageId - }); - return next(); - } - - let attachmentId = message.map && message.map[attachmentMid]; - - if (!attachmentId) { - res.json({ - error: 'This attachment does not exist', - attachment: attachmentMid, - message: messageId - }); - return next(); - } - - db.database.collection('attachments.files').findOne({ - _id: new ObjectID(attachmentId) - }, (err, messageData) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - attachment: attachmentMid, - message: messageId - }); - return next(); - } - if (!messageData) { - res.json({ - error: 'This message does not exist', - attachment: attachmentMid, - message: messageId - }); - return next(); - } - - res.writeHead(200, { - 'Content-Type': messageData.contentType || 'application/octet-stream' - }); - - let attachmentStream = messageHandler.indexer.gridstore.openDownloadStream(messageData._id); - - attachmentStream.once('error', err => res.emit('error', err)); - - if (messageData.metadata.transferEncoding === 'base64') { - attachmentStream.pipe(new libbase64.Decoder()).pipe(res); - } else if (messageData.metadata.transferEncoding === 'quoted-printable') { - attachmentStream.pipe(new libqp.Decoder()).pipe(res); - } else { - attachmentStream.pipe(res); - } - }); - }); -}); - -server.get('/attachment/:attachment', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - attachment: Joi.string().hex().lowercase().length(24).required() - }); - - const result = Joi.validate( - { - attachment: req.params.attachment - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let attachmentId = result.value.attachment; - - db.database.collection('attachments.files').findOne({ - _id: new ObjectID(attachmentId) - }, (err, messageData) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - attachment: attachmentId - }); - return next(); - } - if (!messageData) { - res.json({ - error: 'This message does not exist', - attachment: attachmentId - }); - return next(); - } - - res.writeHead(200, { - 'Content-Type': messageData.contentType || 'application/octet-stream' - }); - - let attachmentStream = messageHandler.indexer.gridstore.openDownloadStream(messageData._id); - - attachmentStream.once('error', err => res.emit('error', err)); - - if (messageData.metadata.transferEncoding === 'base64') { - attachmentStream.pipe(new libbase64.Decoder()).pipe(res); - } else if (messageData.metadata.transferEncoding === 'quoted-printable') { - attachmentStream.pipe(new libqp.Decoder()).pipe(res); - } else { - attachmentStream.pipe(res); - } - }); -}); - -server.del('/message/:id', (req, res, next) => { - res.charSet('utf-8'); - - const schema = Joi.object().keys({ - id: Joi.string().hex().lowercase().length(24).required(), - mailbox: Joi.string().hex().lowercase().length(24).optional() - }); - - const result = Joi.validate( - { - id: req.params.id, - mailbox: req.params.mailbox - }, - schema, - { - abortEarly: false, - convert: true, - allowUnknown: true - } - ); - - if (result.error) { - res.json({ - error: result.error.message - }); - return next(); - } - - let id = result.value.id; - let mailbox = result.value.mailbox; - - let query = { - _id: new ObjectID(id) - }; - - if (mailbox) { - query.mailbox = new ObjectID(mailbox); - } - - messageHandler.del( - { - query - }, - (err, success) => { - if (err) { - res.json({ - error: 'MongoDB Error: ' + err.message, - id - }); - return next(); - } - - res.json({ - success, - id - }); - return next(); - } - ); -}); - module.exports = done => { if (!config.imap.enabled) { return setImmediate(() => done(null, false)); @@ -1300,7 +601,6 @@ module.exports = done => { let started = false; - messageHandler = new MessageHandler(db.database, db.redisConfig); userHandler = new UserHandler(db.database, db.redis); server.on('error', err => { diff --git a/imap-core/lib/commands/append.js b/imap-core/lib/commands/append.js index 1cc6db3a..15a4435b 100644 --- a/imap-core/lib/commands/append.js +++ b/imap-core/lib/commands/append.js @@ -124,5 +124,5 @@ function validateInternalDate(internaldate) { if (!internaldate || typeof internaldate !== 'string') { return false; } - return /^([ \d]\d)\-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ([\-+])(\d{2})(\d{2})$/i.test(internaldate); + return /^([ \d]\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ([-+])(\d{2})(\d{2})$/i.test(internaldate); } diff --git a/imap-core/lib/commands/store.js b/imap-core/lib/commands/store.js index fbb116a9..ed9b2103 100644 --- a/imap-core/lib/commands/store.js +++ b/imap-core/lib/commands/store.js @@ -78,7 +78,7 @@ module.exports = { return callback(new Error('Invalid sequence set for STORE')); } - if (!/^[\-+]?FLAGS$/.test(action)) { + if (!/^[-+]?FLAGS$/.test(action)) { return callback(new Error('Invalid message data item name for STORE')); } @@ -151,9 +151,10 @@ module.exports = { let response = { response: success === true ? 'OK' : 'NO', - code: typeof success === 'string' - ? success.toUpperCase() - : modified && modified.length ? 'MODIFIED ' + imapTools.packMessageRange(modified) : false, + code: + typeof success === 'string' + ? success.toUpperCase() + : modified && modified.length ? 'MODIFIED ' + imapTools.packMessageRange(modified) : false, message }; diff --git a/imap-core/lib/commands/uid-store.js b/imap-core/lib/commands/uid-store.js index e8409a04..8d146933 100644 --- a/imap-core/lib/commands/uid-store.js +++ b/imap-core/lib/commands/uid-store.js @@ -69,7 +69,7 @@ module.exports = { return callback(new Error('Invalid sequence set for UID STORE')); } - if (!/^[\-+]?FLAGS$/.test(action)) { + if (!/^[-+]?FLAGS$/.test(action)) { return callback(new Error('Invalid message data item name for UID STORE')); } @@ -131,9 +131,10 @@ module.exports = { callback(null, { response: success === true ? 'OK' : 'NO', - code: typeof success === 'string' - ? success.toUpperCase() - : modified && modified.length ? 'MODIFIED ' + imapTools.packMessageRange(modified) : false, + code: + typeof success === 'string' + ? success.toUpperCase() + : modified && modified.length ? 'MODIFIED ' + imapTools.packMessageRange(modified) : false, message }); } diff --git a/imap-core/lib/imap-tools.js b/imap-core/lib/imap-tools.js index aac8bdde..bbe4458e 100644 --- a/imap-core/lib/imap-tools.js +++ b/imap-core/lib/imap-tools.js @@ -334,7 +334,7 @@ module.exports.filterFolders = function(folders, query) { .replace(/\*\*+/g, '*') .replace(/%%+/g, '%') // escape special characters - .replace(/([\\^$+?!.():=\[\]|,\-])/g, '\\$1') + .replace(/([\\^$+?!.():=[\]|,-])/g, '\\$1') // setup * .replace(/[*]/g, '.*') // setup % diff --git a/imap-core/lib/indexer/body-structure.js b/imap-core/lib/indexer/body-structure.js index d9aab464..0e652503 100644 --- a/imap-core/lib/indexer/body-structure.js +++ b/imap-core/lib/indexer/body-structure.js @@ -194,8 +194,8 @@ class BodyStructure { } else { return ( data - // skip body MD5 from extension fields - .concat(this.getExtensionFields(node, options).slice(1)) + // skip body MD5 from extension fields + .concat(this.getExtensionFields(node, options).slice(1)) ); } } diff --git a/imap-core/lib/indexer/indexer.js b/imap-core/lib/indexer/indexer.js index 54d0e1b8..c5cdf815 100644 --- a/imap-core/lib/indexer/indexer.js +++ b/imap-core/lib/indexer/indexer.js @@ -725,11 +725,11 @@ class Indexer { } return ( formatHeaders(node.header) - .filter(line => { - let key = line.split(':').shift().toLowerCase().trim(); - return selector.headers.indexOf(key) >= 0; - }) - .join('\r\n') + '\r\n\r\n' + .filter(line => { + let key = line.split(':').shift().toLowerCase().trim(); + return selector.headers.indexOf(key) >= 0; + }) + .join('\r\n') + '\r\n\r\n' ); case 'header.fields.not': @@ -739,11 +739,11 @@ class Indexer { } return ( formatHeaders(node.header) - .filter(line => { - let key = line.split(':').shift().toLowerCase().trim(); - return selector.headers.indexOf(key) < 0; - }) - .join('\r\n') + '\r\n\r\n' + .filter(line => { + let key = line.split(':').shift().toLowerCase().trim(); + return selector.headers.indexOf(key) < 0; + }) + .join('\r\n') + '\r\n\r\n' ); case 'mime': diff --git a/imap-core/lib/indexer/parse-mime-tree.js b/imap-core/lib/indexer/parse-mime-tree.js index be897aa9..93f93d3e 100644 --- a/imap-core/lib/indexer/parse-mime-tree.js +++ b/imap-core/lib/indexer/parse-mime-tree.js @@ -184,7 +184,7 @@ class MIMEParser { // Do not touch headers that have strange looking keys, keep these // only in the unparsed array - if (/[^a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) { + if (/[^a-zA-Z0-9\-*]/.test(key) || key.length >= 100) { continue; } @@ -268,7 +268,7 @@ class MIMEParser { // Do not touch headers that have strange looking keys, keep these // only in the unparsed array - if (/[^a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) { + if (/[^a-zA-Z0-9\-*]/.test(key) || key.length >= 100) { return; } diff --git a/imap-core/lib/tls-options.js b/imap-core/lib/tls-options.js index e8372ada..0625fb1f 100644 --- a/imap-core/lib/tls-options.js +++ b/imap-core/lib/tls-options.js @@ -4,7 +4,8 @@ module.exports = getTLSOptions; const tlsDefaults = { - key: '-----BEGIN RSA PRIVATE KEY-----\n' + + key: + '-----BEGIN RSA PRIVATE KEY-----\n' + 'MIIEpAIBAAKCAQEA6Z5Qqhw+oWfhtEiMHE32Ht94mwTBpAfjt3vPpX8M7DMCTwHs\n' + '1xcXvQ4lQ3rwreDTOWdoJeEEy7gMxXqH0jw0WfBx+8IIJU69xstOyT7FRFDvA1yT\n' + 'RXY2yt9K5s6SKken/ebMfmZR+03ND4UFsDzkz0FfgcjrkXmrMF5Eh5UXX/+9YHeU\n' + @@ -31,7 +32,8 @@ const tlsDefaults = { 'wXOpdKrvkjZbT4AzcNrlGtRl3l7dEVXTu+dN7/ZieJRu7zaStlAQZkIyP9O3DdQ3\n' + 'rIcetQpfrJ1cAqz6Ng0pD0mh77vQ13WG1BBmDFa2A9BuzLoBituf4g==\n' + '-----END RSA PRIVATE KEY-----', - cert: '-----BEGIN CERTIFICATE-----\n' + + cert: + '-----BEGIN CERTIFICATE-----\n' + 'MIICpDCCAYwCCQCuVLVKVTXnAjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwls\n' + 'b2NhbGhvc3QwHhcNMTUwMjEyMTEzMjU4WhcNMjUwMjA5MTEzMjU4WjAUMRIwEAYD\n' + 'VQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDp\n' + diff --git a/imap-core/test/fixtures/mimetree.js b/imap-core/test/fixtures/mimetree.js index c84b053f..1928811b 100644 --- a/imap-core/test/fixtures/mimetree.js +++ b/imap-core/test/fixtures/mimetree.js @@ -2,7 +2,7 @@ module.exports.rfc822 = '' + - 'Subject: test\ r\ n ' + + 'Subject: test r n ' + 'Content-type: multipart/mixed; boundary=abc\r\n' + '\r\n' + '--abc\r\n' + diff --git a/imap-core/test/protocol-test.js b/imap-core/test/protocol-test.js index f2b2e1da..c27d2a2d 100644 --- a/imap-core/test/protocol-test.js +++ b/imap-core/test/protocol-test.js @@ -539,7 +539,7 @@ describe('IMAP Protocol integration tests', function() { let cmds = [ 'T1 LOGIN testuser pass', - 'T2 APPEND INBOX (\Seen $NotJunk NotJunk) "20-Oct-2015 09:57:08 +0300" {' + message.length + '}', + 'T2 APPEND INBOX (Seen $NotJunk NotJunk) "20-Oct-2015 09:57:08 +0300" {' + message.length + '}', lchunks, 'T3 LOGOUT' ]; @@ -781,7 +781,7 @@ describe('IMAP Protocol integration tests', function() { }, function(resp) { resp = resp.toString(); - expect(/^\* ID \("name\"/m.test(resp)).to.be.true; + expect(/^\* ID \("name"/m.test(resp)).to.be.true; expect(/^T1 OK/m.test(resp)).to.be.true; done(); } diff --git a/imap.js b/imap.js index 200c9653..3db92b6c 100644 --- a/imap.js +++ b/imap.js @@ -10,12 +10,14 @@ const imapHandler = IMAPServerModule.imapHandler; const ObjectID = require('mongodb').ObjectID; const Indexer = require('./imap-core/lib/indexer/indexer'); const imapTools = require('./imap-core/lib/imap-tools'); -const setupIndexes = require('./indexes.json'); const MessageHandler = require('./lib/message-handler'); const UserHandler = require('./lib/user-handler'); const db = require('./lib/db'); const RedFour = require('redfour'); const packageData = require('./package.json'); +const yaml = require('js-yaml'); +const fs = require('fs'); +const setupIndexes = yaml.safeLoad(fs.readFileSync(__dirname + '/indexes.yaml', 'utf8')).indexes; // home many modifications to cache before writing const BULK_BATCH_SIZE = 150; @@ -366,6 +368,7 @@ server.onDelete = function(path, session, callback) { [ { $match: { + user: session.user.id, mailbox: mailbox._id } }, @@ -394,6 +397,7 @@ server.onDelete = function(path, session, callback) { let storageUsed = (res && res[0] && res[0].storageUsed) || 0; db.database.collection('messages').deleteMany({ + user: session.user.id, mailbox: mailbox._id }, err => { if (err) { @@ -458,6 +462,7 @@ server.onOpen = function(path, session, callback) { db.database .collection('messages') .find({ + user: session.user.id, mailbox: mailbox._id }) .project({ @@ -499,6 +504,7 @@ server.onStatus = function(path, session, callback) { db.database .collection('messages') .find({ + user: session.user.id, mailbox: mailbox._id }) .count((err, total) => { @@ -508,6 +514,7 @@ server.onStatus = function(path, session, callback) { db.database .collection('messages') .find({ + user: session.user.id, mailbox: mailbox._id, seen: false }) @@ -648,11 +655,13 @@ server.onStore = function(path, update, session, callback) { } let query = { + user: session.user.id, mailbox: mailbox._id }; if (update.unchangedSince) { query = { + user: session.user.id, mailbox: mailbox._id, modseq: { $lte: update.unchangedSince @@ -884,7 +893,8 @@ server.onStore = function(path, update, session, callback) { updateEntries.push({ updateOne: { filter: { - _id: message._id + _id: message._id, + user: session.user.id }, update: flagsupdate } @@ -952,6 +962,7 @@ server.onExpunge = function(path, update, session, callback) { let cursor = db.database .collection('messages') .find({ + user: session.user.id, mailbox: mailbox._id, deleted: true }) @@ -1004,7 +1015,8 @@ server.onExpunge = function(path, update, session, callback) { } db.database.collection('messages').deleteOne({ - _id: message._id + _id: message._id, + user: session.user.id }, err => { if (err) { return updateQuota(() => cursor.close(() => callback(err))); @@ -1104,6 +1116,7 @@ server.onCopy = function(path, update, session, callback) { let cursor = db.database .collection('messages') .find({ + user: session.user.id, mailbox: mailbox._id, uid: { $in: update.messages @@ -1331,11 +1344,13 @@ server.onFetch = function(path, options, session, callback) { } let query = { + user: session.user.id, mailbox: mailbox._id }; if (options.changedSince) { query = { + user: session.user.id, mailbox: mailbox._id, modseq: { $gt: options.changedSince @@ -1443,7 +1458,8 @@ server.onFetch = function(path, options, session, callback) { updateEntries.push({ updateOne: { filter: { - _id: message._id + _id: message._id, + user: session.user.id }, update: { $addToSet: { @@ -1512,6 +1528,7 @@ server.onSearch = function(path, options, session, callback) { // prepare query let query = { + user: session.user.id, mailbox: mailbox._id }; @@ -2030,6 +2047,8 @@ function clearExpiredMessages() { return deleteOrphanedAttachments(() => done(null, true)); } + // TODO: check performance on sharded settings as the query + // does not use the shard key let cursor = db.database .collection('messages') .find({ @@ -2175,7 +2194,7 @@ module.exports = done => { return next(); } let index = setupIndexes[indexpos++]; - db.database.collection(index.collection).createIndexes(index.indexes, (err, r) => { + db.database.collection(index.collection).createIndexes([index.index], (err, r) => { if (err) { server.logger.error( { @@ -2184,7 +2203,7 @@ module.exports = done => { }, 'Failed creating index %s %s. %s', indexpos, - index.indexes.map(i => JSON.stringify(i.name)).join(', '), + JSON.stringify(index.index.name), err.message ); } else if (r.numIndexesAfter !== r.numIndexesBefore) { @@ -2194,7 +2213,7 @@ module.exports = done => { }, 'Created index %s %s', indexpos, - index.indexes.map(i => JSON.stringify(i.name)).join(', ') + JSON.stringify(index.index.name) ); } else { server.logger.debug( @@ -2203,7 +2222,7 @@ module.exports = done => { }, 'Skipped index %s %s: %s', indexpos, - index.indexes.map(i => JSON.stringify(i.name)).join(', '), + JSON.stringify(index.index.name), r.note || 'No index added' ); } @@ -2212,7 +2231,7 @@ module.exports = done => { }); }; - gcLock.acquireLock('db_indexes', 10 * 60 * 1000, (err, lock) => { + gcLock.acquireLock('db_indexes', 1 * 60 * 1000, (err, lock) => { if (err) { server.logger.error( { diff --git a/indexes.json b/indexes.json deleted file mode 100644 index 400d16d3..00000000 --- a/indexes.json +++ /dev/null @@ -1,427 +0,0 @@ -[ - { - "collection": "users", - "indexes": [ - { - "name": "users", - "key": { - "username": 1 - }, - "unique": true, - "background": true - } - ] - }, - { - "collection": "users", - "indexes": [ - { - "name": "show_new", - "key": { - "created": -1 - }, - "background": true - } - ] - }, - { - "collection": "addresses", - "indexes": [ - { - "name": "address", - "key": { - "address": 1 - }, - "unique": true, - "background": true - } - ] - }, - { - "collection": "addresses", - "indexes": [ - { - "name": "user", - "key": { - "user": 1 - }, - "background": true - } - ] - }, - { - "collection": "mailboxes", - "indexes": [ - { - "name": "find_by_user", - "key": { - "user": 1, - "path": 1 - }, - "background": true - } - ] - }, - { - "collection": "mailboxes", - "indexes": [ - { - "name": "user_subscribed", - "key": { - "user": 1, - "subscribed": 1 - }, - "background": true - } - ] - }, - { - "collection": "mailboxes", - "indexes": [ - { - "name": "find_by_type", - "key": { - "user": 1, - "specialUse": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_messages", - "key": { - "mailbox": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "user_messages_by_thread", - "key": { - "user": 1, - "thread": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_uid", - "key": { - "mailbox": 1, - "uid": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_modseq_uid", - "key": { - "mailbox": 1, - "modseq": 1, - "uid": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "newer_first", - "key": { - "mailbox": 1, - "uid": -1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_flags", - "key": { - "mailbox": 1, - "flags": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "by_modseq", - "key": { - "mailbox": 1, - "modseq": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "by_idate", - "key": { - "mailbox": 1, - "idate": 1, - "_id": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "by_idate_newer", - "key": { - "mailbox": 1, - "idate": -1, - "_id": -1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "by_hdate", - "key": { - "mailbox": 1, - "hdate": 1, - "msgid": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "by_size", - "key": { - "mailbox": 1, - "size": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "by_headers", - "key": { - "mailbox": 1, - "headers.key": 1, - "headers.value": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "fulltext", - "key": { - "mailbox": 1, - "subject": "text", - "text": "text" - }, - "weights": { - "subject": 10, - "text": 5 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_seen_flag", - "key": { - "mailbox": 1, - "seen": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_deleted_flag", - "key": { - "mailbox": 1, - "deleted": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_flagged_flag", - "key": { - "mailbox": 1, - "flagged": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "mailbox_draft_flag", - "key": { - "mailbox": 1, - "draft": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "has_attachment", - "key": { - "mailbox": 1, - "ha": 1 - }, - "background": true - } - ] - }, - { - "collection": "messages", - "indexes": [ - { - "name": "retention_time", - "partialFilterExpression": { - "exp": true - }, - "key": { - "exp": 1, - "rdate": 1 - }, - "background": true - } - ] - }, - { - "collection": "attachments.files", - "indexes": [ - { - "name": "attachment_hash", - "key": { - "metadata.h": 1 - }, - "background": true - } - ] - }, - { - "collection": "attachments.files", - "indexes": [ - { - "name": "related_attachments", - "key": { - "metadata.c": 1, - "metadata.m": 1 - }, - "background": true - } - ] - }, - { - "collection": "journal", - "indexes": [ - { - "name": "mailbox_modseq", - "key": { - "mailbox": 1, - "modseq": 1 - }, - "background": true - } - ] - }, - { - "collection": "journal", - "indexes": [ - { - "name": "autoexpire", - "expireAfterSeconds": 21600, - "key": { - "created": 1 - }, - "background": true - } - ] - }, - { - "collection": "threads", - "indexes": [ - { - "name": "thread", - "key": { - "user": 1, - "ids": 1 - }, - "background": true - } - ] - }, - { - "collection": "threads", - "indexes": [ - { - "name": "autoexpire", - "expireAfterSeconds": 31104000, - "key": { - "updated": 1 - }, - "background": true - } - ] - } -] diff --git a/indexes.yaml b/indexes.yaml new file mode 100644 index 00000000..b7323dab --- /dev/null +++ b/indexes.yaml @@ -0,0 +1,255 @@ +--- +indexes: + +# Indexes for the user collection + +- collection: users + index: + name: users + unique: true + key: + username: 1 +- collection: users + index: + name: show_new + key: + created: -1 + +# Indexes for the addresses collection + +- collection: addresses + index: + name: address + unique: true + key: + address: 1 +- collection: addresses + index: + name: user + key: + user: 1 + +# Indexes for the mailboxes collection + +- collection: mailboxes + index: + name: find_by_user + key: + user: 1 + path: 1 +- collection: mailboxes + index: + name: user_subscribed + key: + user: 1 + subscribed: 1 +- collection: mailboxes + index: + name: find_by_type + key: + user: 1 + specialUse: 1 + +# Indexes for the messages collection +# NB! this is a sharded collection and the shard +# key should be 'user' so keep this field as the first one +# in indexes + +- collection: messages + index: + name: mailbox_by_id + key: + _id: 1 + user: 1 +- collection: messages + index: + name: mailbox_messages + key: + user: 1 + mailbox: 1 +- collection: messages + index: + name: user_messages_by_thread + key: + user: 1 + thread: 1 +- collection: messages + index: + name: mailbox_uid + key: + user: 1 + mailbox: 1 + uid: 1 +- collection: messages + index: + name: mailbox_modseq_uid + key: + user: 1 + mailbox: 1 + modseq: 1 + uid: 1 +- collection: messages + index: + name: newer_first + key: + user: 1 + mailbox: 1 + uid: -1 +- collection: messages + index: + name: mailbox_flags + key: + user: 1 + mailbox: 1 + flags: 1 +- collection: messages + index: + name: by_modseq + key: + user: 1 + mailbox: 1 + modseq: 1 +- collection: messages + index: + name: by_idate + key: + user: 1 + mailbox: 1 + idate: 1 + _id: 1 +- collection: messages + index: + name: by_idate_newer + key: + user: 1 + mailbox: 1 + idate: -1 + _id: -1 +- collection: messages + index: + name: by_hdate + key: + user: 1 + mailbox: 1 + hdate: 1 + msgid: 1 +- collection: messages + index: + name: by_size + key: + user: 1 + mailbox: 1 + size: 1 +- collection: messages + index: + name: by_headers + key: + user: 1 + mailbox: 1 + headers.key: 1 + headers.value: 1 +- collection: messages + index: + # there can be only one $text index per collection, so in order to make + # account wide searches we do not use mailbox as compound key element here. + # IMAP TEXT and BODY searches might be slower though + name: fulltext + key: + user: 1 + subject: text + text: text + weights: + subject: 10 + text: 5 +- collection: messages + index: + name: mailbox_seen_flag + key: + user: 1 + mailbox: 1 + seen: 1 +- collection: messages + index: + name: mailbox_deleted_flag + key: + user: 1 + mailbox: 1 + deleted: 1 +- collection: messages + index: + name: mailbox_flagged_flag + key: + user: 1 + mailbox: 1 + flagged: 1 +- collection: messages + index: + name: mailbox_draft_flag + key: + user: 1 + mailbox: 1 + draft: 1 +- collection: messages + index: + name: has_attachment + key: + user: 1 + mailbox: 1 + ha: 1 +- collection: messages + index: + # This filter finds all messages that are expired and must be deleted. + # Not sure about performance though as it is a global query + name: retention_time + partialFilterExpression: + exp: true + key: + exp: 1 + rdate: 1 + +# Indexes for the attachments collection +# attachments.files collection should be sharded by _id (hash) +# attachments.chunks collection should be sharded by files_id (hash) + +- collection: attachments.files + index: + name: attachment_hash + key: + metadata.h: 1 +- collection: attachments.files + index: + name: related_attachments + key: + metadata.c: 1 + metadata.m: 1 + +# Indexes for the journal collection + +- collection: journal + index: + name: mailbox_modseq + key: + mailbox: 1 + modseq: 1 +- collection: journal + index: + name: autoexpire + expireAfterSeconds: 21600 + key: + created: 1 + +# Indexes for the threads collection + +- collection: threads + index: + name: thread + key: + user: 1 + ids: 1 +- collection: threads + index: + name: autoexpire + # autoremove thread indexes after 1 year of inactivity + expireAfterSeconds: 31104000 + key: + updated: 1 diff --git a/lib/imap-notifier.js b/lib/imap-notifier.js index 71f5bd64..0def551d 100644 --- a/lib/imap-notifier.js +++ b/lib/imap-notifier.js @@ -212,7 +212,8 @@ class ImapNotifier extends EventEmitter { this.database.collection('messages').updateMany({ _id: { $in: updated - } + }, + user: mailbox.user }, { // only update modseq if the new value is larger than old one $max: { diff --git a/lib/message-handler.js b/lib/message-handler.js index 67bb7f55..70aea1d1 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -93,6 +93,7 @@ class MessageHandler { } this.checkExistingMessage( + mailbox.user, mailbox._id, { hdate, @@ -314,9 +315,10 @@ class MessageHandler { }); } - checkExistingMessage(mailboxId, message, options, callback) { + checkExistingMessage(user, mailboxId, message, options, callback) { // if a similar message already exists then update existing one this.database.collection('messages').findOne({ + user, mailbox: mailboxId, hdate: message.hdate, msgid: message.msgid @@ -370,7 +372,8 @@ class MessageHandler { let modseq = mailbox.modifyIndex + 1; this.database.collection('messages').findOneAndUpdate({ - _id: existing._id + _id: existing._id, + user: mailbox.user }, { $set: { uid, @@ -452,114 +455,87 @@ class MessageHandler { } del(options, callback) { - let getMessage = next => { - if (options.message) { - return next(null, options.message); - } - this.database.collection('messages').findOne( - options.query, - { - fields: { - mailbox: true, - uid: true, - size: true, - map: true, - magic: true - } - }, - next - ); - }; + let message = options.message; + this.getMailbox( + { + mailbox: options.mailbox || message.mailbox + }, + (err, mailbox) => { + if (err) { + return callback(err); + } - getMessage((err, message) => { - if (err) { - return callback(err); - } - - if (!message) { - return callback(new Error('Message does not exist')); - } - - this.getMailbox( - { - mailbox: options.mailbox || message.mailbox - }, - (err, mailbox) => { + this.database.collection('messages').deleteOne({ + _id: message._id, + user: mailbox.user + }, err => { if (err) { return callback(err); } - this.database.collection('messages').deleteOne({ - _id: message._id - }, err => { - if (err) { - return callback(err); - } + this.updateQuota( + mailbox, + { + storageUsed: -message.size + }, + () => { + let updateAttachments = next => { + let attachments = Object.keys(message.map || {}).map(key => message.map[key]); + if (!attachments.length) { + return next(); + } - this.updateQuota( - mailbox, - { - storageUsed: -message.size - }, - () => { - let updateAttachments = next => { - let attachments = Object.keys(message.map || {}).map(key => message.map[key]); - if (!attachments.length) { - return next(); + // remove link to message from attachments (if any exist) + this.database.collection('attachments.files').updateMany({ + _id: { + $in: attachments } - - // remove link to message from attachments (if any exist) - this.database.collection('attachments.files').updateMany({ - _id: { - $in: attachments - } - }, { - $inc: { - 'metadata.c': -1, - 'metadata.m': -message.magic - } - }, { - multi: true, - w: 1 - }, err => { - if (err) { - // ignore as we don't really care if we have orphans or not - } - next(); - }); - }; - - updateAttachments(() => { - if (options.session && options.session.selected && options.session.selected.mailbox === mailbox.path) { - options.session.writeStream.write(options.session.formatResponse('EXPUNGE', message.uid)); + }, { + $inc: { + 'metadata.c': -1, + 'metadata.m': -message.magic } + }, { + multi: true, + w: 1 + }, err => { + if (err) { + // ignore as we don't really care if we have orphans or not + } + next(); + }); + }; - this.notifier.addEntries( - mailbox, - false, - { - command: 'EXPUNGE', - ignore: options.session && options.session.id, - uid: message.uid, - message: message._id - }, - () => { - this.notifier.fire(mailbox.user, mailbox.path); + updateAttachments(() => { + if (options.session && options.session.selected && options.session.selected.mailbox === mailbox.path) { + options.session.writeStream.write(options.session.formatResponse('EXPUNGE', message.uid)); + } - if (options.skipAttachments) { - return callback(null, true); - } + this.notifier.addEntries( + mailbox, + false, + { + command: 'EXPUNGE', + ignore: options.session && options.session.id, + uid: message.uid, + message: message._id + }, + () => { + this.notifier.fire(mailbox.user, mailbox.path); + if (options.skipAttachments) { return callback(null, true); } - ); - }); - } - ); - }); - } - ); - }); + + return callback(null, true); + } + ); + }); + } + ); + }); + } + ); } move(options, callback) { @@ -587,6 +563,7 @@ class MessageHandler { let cursor = this.database .collection('messages') .find({ + user: mailbox, mailbox: mailbox._id, uid: { $in: options.messages || [] @@ -683,7 +660,8 @@ class MessageHandler { // update message, change mailbox from old to new one this.database.collection('messages').findOneAndUpdate({ - _id: message._id + _id: message._id, + user: mailbox.user }, updateOptions, err => { if (err) { return cursor.close(() => done(err)); diff --git a/logo.txt b/logo.txt index 21f5447e..5077544d 100644 --- a/logo.txt +++ b/logo.txt @@ -1,4 +1,4 @@ -█ █░ ██▓ ██▓ ▓█████▄ ▓█████▄ █ ██ ▄████▄ ██ ▄█▀ + █ █░ ██▓ ██▓ ▓█████▄ ▓█████▄ █ ██ ▄████▄ ██ ▄█▀ ▓█░ █ ░█░▓██▒▓██▒ ▒██▀ ██▌ ▒██▀ ██▌ ██ ▓██▒▒██▀ ▀█ ██▄█▒ ▒█░ █ ░█ ▒██▒▒██░ ░██ █▌ ░██ █▌▓██ ▒██░▒▓█ ▄ ▓███▄░ ░█░ █ ░█ ░██░▒██░ ░▓█▄ ▌ ░▓█▄ ▌▓▓█ ░██░▒▓▓▄ ▄██▒▓██ █▄ diff --git a/package.json b/package.json index 0a6247bb..366aa88e 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "html-to-text": "^3.3.0", "iconv-lite": "^0.4.18", "joi": "^10.6.0", + "js-yaml": "^3.9.0", "libbase64": "^0.2.0", "libmime": "^3.1.0", "libqp": "^1.1.0", diff --git a/pop3.js b/pop3.js index accfe9ee..dca64240 100644 --- a/pop3.js +++ b/pop3.js @@ -84,6 +84,7 @@ const serverOptions = { db.database .collection('messages') .find({ + user: session.user.id, mailbox: mailbox._id }) .project({ @@ -121,7 +122,8 @@ const serverOptions = { onFetchMessage(id, session, callback) { db.database.collection('messages').findOne({ - _id: new ObjectID(id) + _id: new ObjectID(id), + user: session.user.id }, { mimeTree: true, size: true @@ -262,10 +264,11 @@ function markAsSeen(session, messages, callback) { } db.database.collection('messages').updateMany({ - mailbox: mailboxData._id, _id: { $in: ids }, + user: session.user.id, + mailbox: mailboxData._id, modseq: { $lt: mailboxData.modifyIndex }