diff --git a/api.js b/api.js index 49331d7..a3bf12c 100644 --- a/api.js +++ b/api.js @@ -9,11 +9,13 @@ const crypto = require('crypto'); const tools = require('./lib/tools'); const consts = require('./lib/consts'); const UserHandler = require('./lib/user-handler'); +const MailboxHandler = require('./lib/mailbox-handler'); const ImapNotifier = require('./lib/imap-notifier'); const db = require('./lib/db'); const MongoPaging = require('mongo-cursor-pagination'); const certs = require('./lib/certs').get('api'); const ObjectID = require('mongodb').ObjectID; +const imapTools = require('./imap-core/lib/imap-tools'); const serverOptions = { name: 'Wild Duck API', @@ -38,6 +40,7 @@ if (certs && config.api.secure) { const server = restify.createServer(serverOptions); let userHandler; +let mailboxHandler; let notifier; // disable compression for EventSource response @@ -1073,9 +1076,14 @@ server.get('/users/:user/mailboxes', (req, res, next) => { res.charSet('utf-8'); const schema = Joi.object().keys({ - user: Joi.string().hex().lowercase().length(24).required() + user: Joi.string().hex().lowercase().length(24).required(), + counters: Joi.boolean().truthy(['Y', 'true', 'yes', 1]).default(false) }); + if (req.query.counters) { + req.params.counters = req.query.counters; + } + const result = Joi.validate(req.params, schema, { abortEarly: false, convert: true @@ -1089,6 +1097,7 @@ server.get('/users/:user/mailboxes', (req, res, next) => { } let user = new ObjectID(result.value.user); + let counters = result.value.counters; db.users.collection('users').findOne({ _id: user @@ -1147,28 +1156,111 @@ server.get('/users/:user/mailboxes', (req, res, next) => { return a.path.localeCompare(b.path); }); - res.json({ - success: true, + let responses = []; + let position = 0; + let checkMailboxes = () => { + if (position >= mailboxes.length) { + res.json({ + success: true, + mailboxes: responses + }); - mailboxes: mailboxes.map(mailbox => { - let path = mailbox.path.split('/'); - let name = path.pop(); + return next(); + } - return { - id: mailbox._id, - name, - path: mailbox.path, - specialUse: mailbox.specialUse, - modifyIndex: mailbox.modifyIndex - }; - }) - }); + let mailbox = mailboxes[position++]; + let path = mailbox.path.split('/'); + let name = path.pop(); - return next(); + let response = { + id: mailbox._id, + name, + path: mailbox.path, + specialUse: mailbox.specialUse, + modifyIndex: mailbox.modifyIndex, + subscribed: mailbox.subscribed + }; + + if (!counters) { + responses.push(response); + return setImmediate(checkMailboxes); + } + + getMailboxCounter(mailbox._id, false, (err, total) => { + if (err) { + // ignore + } + getMailboxCounter(mailbox._id, 'unseen', (err, unseen) => { + if (err) { + // ignore + } + response.total = total; + response.unseen = unseen; + responses.push(response); + return setImmediate(checkMailboxes); + }); + }); + }; + checkMailboxes(); }); }); }); +server.post('/users/:user/mailboxes', (req, res, next) => { + res.charSet('utf-8'); + + const schema = Joi.object().keys({ + user: Joi.string().hex().lowercase().length(24).required(), + path: Joi.string().regex(/\/{2,}|\/$/g, { invert: true }).required(), + retention: Joi.number().min(0) + }); + + const result = Joi.validate(req.params, schema, { + abortEarly: false, + convert: true + }); + + if (result.error) { + res.json({ + error: result.error.message + }); + return next(); + } + + let user = new ObjectID(result.value.user); + let path = imapTools.normalizeMailbox(result.value.path); + let retention = result.value.retention; + + let opts = { + subscribed: true + }; + if (retention) { + opts.retention = retention; + } + + mailboxHandler.create(user, path, opts, (err, status, id) => { + if (err) { + res.json({ + error: err.message + }); + return next(); + } + + if (typeof status === 'string') { + res.json({ + error: 'Mailbox creation failed with code ' + status + }); + return next(); + } + + res.json({ + success: !!status, + id + }); + return next(); + }); +}); + server.get('/users/:user/mailboxes/:mailbox', (req, res, next) => { res.charSet('utf-8'); @@ -1247,6 +1339,7 @@ server.get('/users/:user/mailboxes/:mailbox', (req, res, next) => { path: mailboxData.path, specialUse: mailboxData.specialUse, modifyIndex: mailboxData.modifyIndex, + subscribed: mailboxData.subscribed, total, unseen }); @@ -1257,6 +1350,115 @@ server.get('/users/:user/mailboxes/:mailbox', (req, res, next) => { }); }); +server.put('/users/:user/mailboxes/:mailbox', (req, res, next) => { + res.charSet('utf-8'); + + const schema = Joi.object().keys({ + user: Joi.string().hex().lowercase().length(24).required(), + mailbox: Joi.string().hex().lowercase().length(24).required(), + path: Joi.string().regex(/\/{2,}|\/$/g, { invert: true }), + retention: Joi.number().min(0), + subscribed: Joi.boolean().truthy(['Y', 'true', 'yes', 1]) + }); + + const result = Joi.validate(req.params, schema, { + abortEarly: false, + convert: true + }); + + if (result.error) { + res.json({ + error: result.error.message + }); + return next(); + } + + let user = new ObjectID(result.value.user); + let mailbox = new ObjectID(result.value.mailbox); + + let updates = {}; + let update = false; + Object.keys(result.value || {}).forEach(key => { + if (!['user', 'mailbox'].includes(key)) { + updates[key] = result.value[key]; + update = true; + } + }); + + if (!update) { + res.json({ + error: 'Nothing was changed' + }); + return next(); + } + + mailboxHandler.update(user, mailbox, updates, (err, status) => { + if (err) { + res.json({ + error: err.message + }); + return next(); + } + + if (typeof status === 'string') { + res.json({ + error: 'Mailbox update failed with code ' + status + }); + return next(); + } + + res.json({ + success: true + }); + return next(); + }); +}); + +server.del('/users/:user/mailboxes/:mailbox', (req, res, next) => { + res.charSet('utf-8'); + + const schema = Joi.object().keys({ + user: Joi.string().hex().lowercase().length(24).required(), + mailbox: Joi.string().hex().lowercase().length(24).required() + }); + + const result = Joi.validate(req.params, schema, { + abortEarly: false, + convert: true + }); + + if (result.error) { + res.json({ + error: result.error.message + }); + return next(); + } + + let user = new ObjectID(result.value.user); + let mailbox = new ObjectID(result.value.mailbox); + + mailboxHandler.del(user, mailbox, (err, status) => { + if (err) { + res.json({ + error: err.message + }); + return next(); + } + + if (typeof status === 'string') { + res.json({ + error: 'Mailbox deletion failed with code ' + status + }); + return next(); + } + + res.json({ + success: true + }); + return next(); + }); +}); + server.get('/users/:user/updates', (req, res, next) => { res.charSet('utf-8'); @@ -1540,11 +1742,12 @@ module.exports = done => { let started = false; - userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis }); notifier = new ImapNotifier({ database: db.database, redis: db.redis }); + userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis }); + mailboxHandler = new MailboxHandler({ database: db.database, users: db.users, redis: db.redis, notifier }); server.on('error', err => { if (!started) { diff --git a/imap.js b/imap.js index 6bc7fc5..b23310a 100644 --- a/imap.js +++ b/imap.js @@ -8,6 +8,7 @@ const ImapNotifier = require('./lib/imap-notifier'); const Indexer = require('./imap-core/lib/indexer/indexer'); const MessageHandler = require('./lib/message-handler'); const UserHandler = require('./lib/user-handler'); +const MailboxHandler = require('./lib/mailbox-handler'); const db = require('./lib/db'); const consts = require('./lib/consts'); const RedFour = require('redfour'); @@ -79,6 +80,7 @@ const server = new IMAPServer(serverOptions); let messageHandler; let userHandler; +let mailboxHandler; let gcTimeout; let gcLock; @@ -281,9 +283,6 @@ module.exports = done => { gcTimeout.unref(); let start = () => { - messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, redis: db.redis }); - userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis }); - server.indexer = new Indexer({ database: db.database }); @@ -294,6 +293,10 @@ module.exports = done => { redis: db.redis }); + messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, redis: db.redis }); + userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis }); + mailboxHandler = new MailboxHandler({ database: db.database, users: db.users, redis: db.redis, notifier: server.notifier }); + let started = false; server.on('error', err => { @@ -325,9 +328,9 @@ module.exports = done => { server.onLsub = onLsub(server); server.onSubscribe = onSubscribe(server); server.onUnsubscribe = onUnsubscribe(server); - server.onCreate = onCreate(server); - server.onRename = onRename(server); - server.onDelete = onDelete(server); + server.onCreate = onCreate(server, mailboxHandler); + server.onRename = onRename(server, mailboxHandler); + server.onDelete = onDelete(server, mailboxHandler); server.onOpen = onOpen(server); server.onStatus = onStatus(server); server.onAppend = onAppend(server, messageHandler); diff --git a/lib/consts.js b/lib/consts.js index 071e4c4..f68c9e7 100644 --- a/lib/consts.js +++ b/lib/consts.js @@ -16,5 +16,12 @@ module.exports = { JUNK_RETENTION: 30 * 24 * 3600 * 1000, - MAILBOX_COUNTER_TTL: 24 * 3600 + MAILBOX_COUNTER_TTL: 24 * 3600, + + SCHEMA_VERSION: '1.0', + // how much plaintext to store. this is indexed with a fulltext index + MAX_PLAINTEXT_CONTENT: 2 * 1024, + + // how much HTML content to store. not indexed + MAX_HTML_CONTENT: 300 * 1024 }; diff --git a/lib/handlers/on-create.js b/lib/handlers/on-create.js index 2396373..cf43fb4 100644 --- a/lib/handlers/on-create.js +++ b/lib/handlers/on-create.js @@ -1,9 +1,7 @@ 'use strict'; -const db = require('../db'); - // CREATE "path/to/mailbox" -module.exports = server => (path, session, callback) => { +module.exports = (server, mailboxHandler) => (path, session, callback) => { server.logger.debug( { tnx: 'create', @@ -13,57 +11,5 @@ module.exports = server => (path, session, callback) => { session.id, path ); - db.database.collection('mailboxes').findOne({ - user: session.user.id, - path - }, (err, mailbox) => { - if (err) { - return callback(err); - } - if (mailbox) { - return callback(null, 'ALREADYEXISTS'); - } - - db.users.collection('users').findOne({ - _id: session.user.id - }, { - fields: { - retention: true - } - }, (err, user) => { - if (err) { - return callback(err); - } - - mailbox = { - user: session.user.id, - path, - uidValidity: Math.floor(Date.now() / 1000), - uidNext: 1, - modifyIndex: 0, - subscribed: true, - flags: [], - retention: user.retention - }; - - db.database.collection('mailboxes').insertOne(mailbox, (err, r) => { - if (err) { - return callback(err); - } - return server.notifier.addEntries( - session.user.id, - path, - { - command: 'CREATE', - mailbox: r.insertId, - name: path - }, - () => { - server.notifier.fire(session.user.id, path); - return callback(null, true); - } - ); - }); - }); - }); + mailboxHandler.create(session.user.id, path, { subscribed: true }, callback); }; diff --git a/lib/handlers/on-delete.js b/lib/handlers/on-delete.js index e00702f..1c4dfd4 100644 --- a/lib/handlers/on-delete.js +++ b/lib/handlers/on-delete.js @@ -3,7 +3,7 @@ const db = require('../db'); // DELETE "path/to/mailbox" -module.exports = server => (path, session, callback) => { +module.exports = (server, mailboxHandler) => (path, session, callback) => { server.logger.debug( { tnx: 'delete', @@ -13,6 +13,7 @@ module.exports = server => (path, session, callback) => { session.id, path ); + db.database.collection('mailboxes').findOne({ user: session.user.id, path @@ -23,91 +24,7 @@ module.exports = server => (path, session, callback) => { if (!mailbox) { return callback(null, 'NONEXISTENT'); } - if (mailbox.specialUse) { - return callback(null, 'CANNOT'); - } - server.notifier.addEntries( - session.user.id, - path, - { - command: 'DROP', - mailbox: mailbox._id - }, - () => { - db.database.collection('mailboxes').deleteOne({ - _id: mailbox._id - }, err => { - if (err) { - return callback(err); - } - - // calculate mailbox size by aggregating the size's of all messages - db.database - .collection('messages') - .aggregate( - [ - { - $match: { - mailbox: mailbox._id - } - }, - { - $group: { - _id: { - mailbox: '$mailbox' - }, - storageUsed: { - $sum: '$size' - } - } - } - ], - { - cursor: { - batchSize: 1 - } - } - ) - .toArray((err, res) => { - if (err) { - return callback(err); - } - - let storageUsed = (res && res[0] && res[0].storageUsed) || 0; - - db.database.collection('messages').deleteMany({ - mailbox: mailbox._id - }, err => { - if (err) { - return callback(err); - } - - let done = () => { - server.notifier.fire(session.user.id, path); - callback(null, true); - }; - - if (!storageUsed) { - return done(); - } - - // decrement quota counters - db.users.collection('users').findOneAndUpdate( - { - _id: mailbox.user - }, - { - $inc: { - storageUsed: -Number(storageUsed) || 0 - } - }, - done - ); - }); - }); - }); - } - ); + mailboxHandler.del(session.user.id, mailbox._id, callback); }); }; diff --git a/lib/handlers/on-rename.js b/lib/handlers/on-rename.js index a7c317d..4acdb23 100644 --- a/lib/handlers/on-rename.js +++ b/lib/handlers/on-rename.js @@ -4,7 +4,7 @@ const db = require('../db'); // RENAME "path/to/mailbox" "new/path" // NB! RENAME affects child and hierarchy mailboxes as well, this example does not do this -module.exports = server => (path, newname, session, callback) => { +module.exports = (server, mailboxHandler) => (path, newname, session, callback) => { server.logger.debug( { tnx: 'rename', @@ -15,45 +15,18 @@ module.exports = server => (path, newname, session, callback) => { path, newname ); + db.database.collection('mailboxes').findOne({ user: session.user.id, - path: newname + path }, (err, mailbox) => { if (err) { return callback(err); } - if (mailbox) { - return callback(null, 'ALREADYEXISTS'); + if (!mailbox) { + return callback(null, 'NONEXISTENT'); } - return server.notifier.addEntries( - session.user.id, - path, - { - command: 'RENAME', - name: newname - }, - () => { - db.database.collection('mailboxes').findOneAndUpdate({ - user: session.user.id, - path - }, { - $set: { - path: newname - } - }, {}, (err, item) => { - if (err) { - return callback(err); - } - if (!item || !item.value) { - // was not able to acquire a lock - return callback(null, 'NONEXISTENT'); - } - - server.notifier.fire(session.user.id, path); - return callback(null, true); - }); - } - ); + mailboxHandler.rename(session.user.id, mailbox._id, newname, false, callback); }); }; diff --git a/lib/mailbox-handler.js b/lib/mailbox-handler.js new file mode 100644 index 0000000..4fff3bd --- /dev/null +++ b/lib/mailbox-handler.js @@ -0,0 +1,293 @@ +'use strict'; + +const ObjectID = require('mongodb').ObjectID; +const ImapNotifier = require('./imap-notifier'); + +class MailboxHandler { + constructor(options) { + this.database = options.database; + this.users = options.users || options.database; + this.redis = options.redis; + this.notifier = + options.notifier || + new ImapNotifier({ + database: options.database, + redis: this.redis, + pushOnly: true + }); + } + + create(user, path, opts, callback) { + this.database.collection('mailboxes').findOne({ + user, + path + }, (err, mailbox) => { + if (err) { + return callback(err); + } + if (mailbox) { + return callback(null, 'ALREADYEXISTS'); + } + + this.users.collection('users').findOne({ + _id: user + }, { + fields: { + retention: true + } + }, (err, userData) => { + if (err) { + return callback(err); + } + + if (!userData) { + return callback(new Error('User not found')); + } + + mailbox = { + _id: new ObjectID(), + user, + path, + uidValidity: Math.floor(Date.now() / 1000), + uidNext: 1, + modifyIndex: 0, + subscribed: true, + flags: [], + retention: userData.retention + }; + + Object.keys(opts || {}).forEach(key => { + if (!['_id', 'user', 'path'].includes(key)) { + mailbox[key] = opts[key]; + } + }); + + this.database.collection('mailboxes').insertOne(mailbox, (err, r) => { + if (err) { + return callback(err); + } + return this.notifier.addEntries( + user, + path, + { + command: 'CREATE', + mailbox: r.insertedId, + path + }, + () => { + this.notifier.fire(user, path); + return callback(null, true, mailbox._id); + } + ); + }); + }); + }); + } + + rename(user, mailbox, newname, opts, callback) { + this.database.collection('mailboxes').findOne({ + _id: mailbox, + user + }, (err, mailboxData) => { + if (err) { + return callback(err); + } + if (!mailboxData) { + return callback(null, 'NONEXISTENT'); + } + this.database.collection('mailboxes').findOne({ + user: mailboxData.user, + path: newname + }, (err, existing) => { + if (err) { + return callback(err); + } + if (existing) { + return callback(null, 'ALREADYEXISTS'); + } + + let $set = { path: newname }; + + Object.keys(opts || {}).forEach(key => { + if (!['_id', 'user', 'path'].includes(key)) { + $set[key] = opts[key]; + } + }); + + this.database.collection('mailboxes').findOneAndUpdate({ + _id: mailbox + }, { + $set + }, {}, (err, item) => { + if (err) { + return callback(err); + } + + if (!item || !item.value) { + // was not able to acquire a lock + return callback(null, 'NONEXISTENT'); + } + this.notifier.addEntries( + mailboxData, + false, + { + command: 'RENAME', + path: newname + }, + () => { + this.notifier.fire(mailboxData.user, mailboxData.path); + return callback(null, true); + } + ); + }); + }); + }); + } + + del(user, mailbox, callback) { + this.database.collection('mailboxes').findOne({ + _id: mailbox, + user + }, (err, mailboxData) => { + if (err) { + return callback(err); + } + if (!mailboxData) { + return callback(null, 'NONEXISTENT'); + } + if (mailboxData.specialUse) { + return callback(null, 'CANNOT'); + } + + this.database.collection('mailboxes').deleteOne({ + _id: mailbox + }, err => { + if (err) { + return callback(err); + } + + this.notifier.addEntries( + mailboxData, + false, + { + command: 'DROP', + mailbox + }, + () => { + // calculate mailbox size by aggregating the size's of all messages + this.database + .collection('messages') + .aggregate( + [ + { + $match: { + mailbox + } + }, + { + $group: { + _id: { + mailbox: '$mailbox' + }, + storageUsed: { + $sum: '$size' + } + } + } + ], + { + cursor: { + batchSize: 1 + } + } + ) + .toArray((err, res) => { + if (err) { + return callback(err); + } + + let storageUsed = (res && res[0] && res[0].storageUsed) || 0; + + this.database.collection('messages').deleteMany({ + mailbox: mailbox._id + }, err => { + if (err) { + return callback(err); + } + + let done = () => { + this.notifier.fire(mailboxData.user, mailboxData.path); + callback(null, true); + }; + + if (!storageUsed) { + return done(); + } + + // decrement quota counters + this.users.collection('users').findOneAndUpdate( + { + _id: mailbox.user + }, + { + $inc: { + storageUsed: -Number(storageUsed) || 0 + } + }, + done + ); + }); + }); + } + ); + }); + }); + } + + update(user, mailbox, updates, callback) { + if (!updates) { + return callback(null, false); + } + + this.database.collection('mailboxes').findOne({ + _id: mailbox + }, (err, mailboxData) => { + if (err) { + return callback(err); + } + if (!mailboxData) { + return callback(null, 'NONEXISTENT'); + } + if (updates.path !== mailboxData.path) { + return this.rename(user, mailbox, updates.path, updates, callback); + } + + let $set = {}; + + Object.keys(updates || {}).forEach(key => { + if (!['_id', 'user', 'path'].includes(key)) { + $set[key] = updates[key]; + } + }); + + this.database.collection('mailboxes').findOneAndUpdate({ + _id: mailbox + }, { + $set + }, {}, (err, item) => { + if (err) { + return callback(err); + } + + if (!item || !item.value) { + // was not able to acquire a lock + return callback(null, 'NONEXISTENT'); + } + + return callback(null, true); + }); + }); + } +} + +module.exports = MailboxHandler; diff --git a/lib/message-handler.js b/lib/message-handler.js index f231e2f..543d3dd 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -7,17 +7,10 @@ const Indexer = require('../imap-core/lib/indexer/indexer'); const ImapNotifier = require('./imap-notifier'); const libmime = require('libmime'); const counters = require('./counters'); +const consts = require('./consts'); const tools = require('./tools'); const parseDate = require('../imap-core/lib/parse-date'); -// how many modifications to cache before writing -const BULK_BATCH_SIZE = 150; -const SCHEMA_VERSION = '1.0'; -// how much plaintext to store. this is indexed with a fulltext index -const MAX_PLAINTEXT_CONTENT = 2 * 1024; -// how much HTML content to store. not indexed -const MAX_HTML_CONTENT = 300 * 1024; - // index only the following headers for SEARCH const INDEXED_HEADERS = ['to', 'cc', 'subject', 'from', 'sender', 'reply-to', 'message-id', 'thread-index']; @@ -136,7 +129,7 @@ class MessageHandler { // should be kept when COPY'ing or MOVE'ing root: id, - v: SCHEMA_VERSION, + v: consts.SCHEMA_VERSION, // if true then expires after rdate + retention exp: !!mailbox.retention, @@ -181,7 +174,8 @@ class MessageHandler { if (maildata.text) { message.text = maildata.text.replace(/\r\n/g, '\n').trim(); // text is indexed with a fulltext index, so only store the beginning of it - message.text = message.text.length <= MAX_PLAINTEXT_CONTENT ? message.text : message.text.substr(0, MAX_PLAINTEXT_CONTENT); + message.text = + message.text.length <= consts.MAX_PLAINTEXT_CONTENT ? message.text : message.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); message.intro = message.text.replace(/\s+/g, ' ').trim(); if (message.intro.length > 128) { let intro = message.intro.substr(0, 128); @@ -197,16 +191,16 @@ class MessageHandler { let htmlSize = 0; message.html = maildata.html .map(html => { - if (htmlSize >= MAX_HTML_CONTENT || !html) { + if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { return ''; } - if (htmlSize + Buffer.byteLength(html) <= MAX_HTML_CONTENT) { + if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) { htmlSize += Buffer.byteLength(html); return html; } - html = html.substr(0, htmlSize + Buffer.byteLength(html) - MAX_HTML_CONTENT); + html = html.substr(0, htmlSize + Buffer.byteLength(html) - consts.MAX_HTML_CONTENT); htmlSize += Buffer.byteLength(html); return html; }) @@ -701,7 +695,7 @@ class MessageHandler { unseen: message.unseen }); - if (existsEntries.length >= BULK_BATCH_SIZE) { + if (existsEntries.length >= consts.BULK_BATCH_SIZE) { // mark messages as deleted from old mailbox return this.notifier.addEntries(mailbox, false, removeEntries, () => { // mark messages as added to new mailbox