nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2024-11-10 17:47:07 +08:00
Code Issues Projects Releases Packages Wiki Activity

Started with QUOTA handling

Browse source
This commit is contained in:
Andris Reinman 2017-03-26 23:58:05 +03:00
parent 31dd36b78b
commit 845ea959be
7 changed files with 303 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
README.md
View file

@ -67,9 +67,7 @@ Actual update data (information about new and deleted messages, flag updates and
1. Add interoperability with current servers, for example by fetching authentication data from MySQL
2. Maybe allow some kind of message manipulation through plugins? This would allow to turn Wild Duck for example into an encrypted mail server – mail data would be encrypted using users public key before storing it to DB and decrypted with users private key whenever the user logs in and FETCHes or SEARCHes messages. Private key would be protected by users password. For the user the encryption layer would be invisible while guaranteeing that if the user is currently not logged in then there would be no way to read the messages as the private key is locked.
3. Add quota handling. Every time a user gets a new message added to storage, the quota counter should increase. If only a single quota root would be used per account then implementing rfc2087 should be fairly easy. What is not so easy is keeping count on copied and deleted messages (there's a great technique for this described in the [mail.ru blog](https://team.mail.ru/efficient-storage-how-we-went-down-from-50-pb-to-32-pb/)).
The problem with quota counters is that the actions (_store message + increment counter for mailbox_ or _delete message + decrement counter for mailbox_) are not transactional, so if something fails, the counter might end up in an invalid state. A possible fix would be to use fake transactions - set up a transaction with mailbox and counter data by storing a transaction entry, then process required actions and finally remove the transaction entry. If something fails and transaction is not completed, then the mailbox would be marked for reindexing which would mean that the mailbox quota is entirely re-calculated and quota counters are reset.
3. Add proper quota handling. Every time a user gets a new message added to storage, the quota counter should increase. If only a single quota root would be used per account then implementing rfc2087 should be fairly easy. Currently Wild Duck keeps a count on storage used by an user but this is not exposed to the client.
## Usage
@ -124,6 +122,7 @@ Arguments
- **username** is the username of the user. This is not an email address but authentication username, use only letters and numbers
- **password** is the password for the user
- **storage** (optional) is the maximum storage in bytes allowed for this user
**Example**

15
api.js
View file

@ -23,7 +23,8 @@ server.use(restify.bodyParser({
server.post('/user/create', (req, res, next) => {
const schema = Joi.object().keys({
username: Joi.string().alphanum().lowercase().min(3).max(30).required(),
password: Joi.string().min(3).max(100).required()
password: Joi.string().min(3).max(100).required(),
storage: Joi.number().default(0)
});
const result = Joi.validate({
@ -44,6 +45,7 @@ server.post('/user/create', (req, res, next) => {
let username = result.value.username;
let password = result.value.password;
let storage = result.value.storage;
database.collection('users').findOne({
username
@ -69,6 +71,9 @@ server.post('/user/create', (req, res, next) => {
username,
password: hash,
address: false,
storageUsed: 0,
messages: 0,
storage,
created: new Date()
}, (err, result) => {
if (err) {
@ -89,6 +94,8 @@ server.post('/user/create', (req, res, next) => {
uidValidity,
uidNext: 1,
modifyIndex: 0,
storageUsed: 0,
messages: 0,
subscribed: true
}, {
user,
@ -97,6 +104,8 @@ server.post('/user/create', (req, res, next) => {
uidValidity,
uidNext: 1,
modifyIndex: 0,
storageUsed: 0,
messages: 0,
subscribed: true
}, {
user,
@ -105,6 +114,8 @@ server.post('/user/create', (req, res, next) => {
uidValidity,
uidNext: 1,
modifyIndex: 0,
storageUsed: 0,
messages: 0,
subscribed: true
}, {
user,
@ -113,6 +124,8 @@ server.post('/user/create', (req, res, next) => {
uidValidity,
uidNext: 1,
modifyIndex: 0,
storageUsed: 0,
messages: 0,
subscribed: true
}], {
w: 1,

95
examples/push-message.js
View file

@ -13,53 +13,72 @@ const config = require('config');
const nodemailer = require('nodemailer');
const transporter = nodemailer.createTransport({
pool: true,
maxConnections: 10,
host: 'localhost',
port: config.smtp.port,
logger: false,
debug: false
});
transporter.sendMail({
envelope: {
let sent = 0;
let total = 1;
let startTime = Date.now();
function send() {
transporter.sendMail({
envelope: {
from: 'andris@kreata.ee',
to: [recipient]
},
from: 'andris@kreata.ee',
to: [recipient]
},
from: 'andris@kreata.ee',
to: recipient,
subject: 'Test ööö message [' + Date.now() + ']',
text: 'Hello world! Current time is ' + new Date().toString() + ' <img src="cid:note@example.com"/>',
html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>',
attachments: [
to: recipient,
subject: 'Test ööö message [' + Date.now() + ']',
text: 'Hello world! Current time is ' + new Date().toString() + ' <img src="cid:note@example.com"/>',
html: '<p>Hello world! Current time is <em>' + new Date().toString() + '</em></p>',
attachments: [
// attachment as plaintext
{
filename: 'notes.txt',
content: 'Some notes about this e-mail',
contentType: 'text/plain' // optional, would be detected from the filename
},
// attachment as plaintext
{
filename: 'notes.txt',
content: 'Some notes about this e-mail',
contentType: 'text/plain' // optional, would be detected from the filename
},
// Small Binary Buffer attachment, should be kept with message
{
filename: 'image.png',
content: new Buffer('iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' +
'//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' +
'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', 'base64'),
// Small Binary Buffer attachment, should be kept with message
{
filename: 'image.png',
content: new Buffer('iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' +
'//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' +
'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', 'base64'),
cid: 'note@example.com' // should be as unique as possible
},
cid: 'note@example.com' // should be as unique as possible
},
// Large Binary Buffer attachment, should be kept separately
{
path: __dirname + '/swan.jpg',
filename: 'swän.jpg'
// Large Binary Buffer attachment, should be kept separately
{
path: __dirname + '/swan.jpg',
filename: 'swän.jpg'
}
]
}, (err, info) => {
if (err && err.response) {
console.log('Message failed: %s', err.response);
} else if (err) {
console.log(err);
} else {
console.log(info);
}
]
}, (err, info) => {
if (err && err.response) {
console.log('Message failed: %s', err.response);
} else if (err) {
console.log(err);
} else {
console.log(info);
}
});
sent++;
if (sent >= total) {
console.log('Sent %s messages in %s s', sent, (Date.now() - startTime) / 1000);
return transporter.close();
}
});
}
for (let i = 0; i < total; i++) {
send();
}

2
imap-core/lib/imap-tools.js
View file

@ -468,6 +468,8 @@ module.exports.formatInternalDate = function (date) {
*/
module.exports.getQueryResponse = function (query, message, options) {
options = options || {};
// for optimization purposes try to use cached mimeTree etc. if available
// If these values are missing then generate these when first time required
// So if the query is for (UID FLAGS) then mimeTree is never generated

2
imap-core/test/protocol-test.js
View file

@ -21,6 +21,8 @@ describe('IMAP Protocol integration tests', function () {
logger: false // remove to print IMAP traffic to console
});
server.acceptUTF8Enabled = false;
server.listen(function () {
port = server.server.address().port;
done();

185
imap.js
View file

@ -158,6 +158,8 @@ server.onCreate = function (path, session, callback) {
uidValidity: Math.floor(Date.now() / 1000),
uidNext: 1,
modifyIndex: 0,
storageUsed: 0,
messages: 0,
subscribed: true
};
@ -231,21 +233,30 @@ server.onDelete = function (path, session, callback) {
return callback(err);
}
database.collection('journal').deleteMany({
database.collection('messages').deleteMany({
mailbox: mailbox._id
}, err => {
if (err) {
return callback(err);
}
database.collection('messages').deleteMany({
mailbox: mailbox._id
}, err => {
if (err) {
return callback(err);
// decrement quota counters
database.collection('users').findOneAndUpdate({
_id: mailbox.user
}, {
$inc: {
storageUsed: -Number(mailbox.storageUsed) || 0,
messages: -Number(mailbox.messages) || 0
}
callback(null, true);
}, () => {
database.collection('journal').deleteMany({
mailbox: mailbox._id
}, err => {
if (err) {
return callback(err);
}
callback(null, true);
});
});
});
});
@ -327,25 +338,41 @@ server.onStatus = function (path, session, callback) {
// APPEND mailbox (flags) date message
server.onAppend = function (path, flags, date, raw, session, callback) {
this.logger.debug('[%s] Appending message to "%s"', session.id, path);
messageHandler.add({
user: session.user.id,
path,
meta: {
source: 'IMAP',
to: session.user.username,
time: Date.now()
},
date,
flags,
raw
}, (err, status, data) => {
database.collection('users').findOne({
_id: session.user.id
}, (err, user) => {
if (err) {
if (err.imapResponse) {
return callback(null, err.imapResponse);
}
return callback(err);
}
callback(null, status, data);
if (!user) {
return callback(new Error('User not found'));
}
if (user.storage && user.storageUsed + raw.length > user.storage) {
return callback(false, 'OVERQUOTA');
}
messageHandler.add({
user: session.user.id,
path,
meta: {
source: 'IMAP',
to: session.user.username,
time: Date.now()
},
date,
flags,
raw
}, (err, status, data) => {
if (err) {
if (err.imapResponse) {
return callback(null, err.imapResponse);
}
return callback(err);
}
callback(null, status, data);
});
});
};
@ -513,31 +540,61 @@ server.onExpunge = function (path, update, session, callback) {
flags: '\\Deleted'
}).project({
_id: true,
uid: true
uid: true,
size: true
}).sort([
['uid', 1]
]);
let deletedMessages = 0;
let deletedStorage = 0;
let updateQuota = next => {
if (!deletedMessages) {
return next();
}
database.collection('mailboxes').findOneAndUpdate({
_id: mailbox._id
}, {
$inc: {
storageUsed: -deletedStorage,
messages: -deletedMessages
}
}, () => {
database.collection('users').findOneAndUpdate({
_id: mailbox.user
}, {
$inc: {
storageUsed: -deletedStorage,
messages: -deletedMessages
}
}, next);
});
};
let processNext = () => {
cursor.next((err, message) => {
if (err) {
return callback(err);
return updateQuota(() => callback(err));
}
if (!message) {
return cursor.close(() => {
this.notifier.fire(session.user.id, path);
updateQuota(() => {
this.notifier.fire(session.user.id, path);
// delete all attachments that do not have any active links to message objects
database.collection('attachments.files').deleteMany({
'metadata.messages': {
$size: 0
}
}, err => {
if (err) {
// ignore as we don't really care if we have orphans or not
}
// delete all attachments that do not have any active links to message objects
database.collection('attachments.files').deleteMany({
'metadata.messages': {
$size: 0
}
}, err => {
if (err) {
// ignore as we don't really care if we have orphans or not
}
return callback(null, true);
return callback(null, true);
});
});
});
}
@ -550,9 +607,12 @@ server.onExpunge = function (path, update, session, callback) {
_id: message._id
}, err => {
if (err) {
return cursor.close(() => callback(err));
return updateQuota(() => cursor.close(() => callback(err)));
}
deletedMessages++;
deletedStorage += Number(message.size) || 0;
// remove link to message from attachments (if any exist)
database.collection('attachments.files').updateMany({
'metadata.messages': message._id
@ -614,20 +674,48 @@ server.onCopy = function (path, update, session, callback) {
}
}); // no projection as we need to copy the entire message
let copiedMessages = 0;
let copiedStorage = 0;
let updateQuota = next => {
if (!copiedMessages) {
return next();
}
database.collection('mailboxes').findOneAndUpdate({
_id: target._id
}, {
$inc: {
storageUsed: copiedStorage,
messages: copiedMessages
}
}, () => {
database.collection('users').findOneAndUpdate({
_id: mailbox.user
}, {
$inc: {
storageUsed: copiedStorage,
messages: copiedMessages
}
}, next);
});
};
let sourceUid = [];
let destinationUid = [];
let processNext = () => {
cursor.next((err, message) => {
if (err) {
return callback(err);
return updateQuota(() => callback(err));
}
if (!message) {
return cursor.close(() => {
this.notifier.fire(session.user.id, target.path);
return callback(null, true, {
uidValidity: target.uidValidity,
sourceUid,
destinationUid
updateQuota(() => {
this.notifier.fire(session.user.id, target.path);
return callback(null, true, {
uidValidity: target.uidValidity,
sourceUid,
destinationUid
});
});
});
}
@ -645,12 +733,12 @@ server.onCopy = function (path, update, session, callback) {
uidNext: true
}, (err, item) => {
if (err) {
return callback(err);
return updateQuota(() => callback(err));
}
if (!item || !item.value) {
// was not able to acquire a lock
return callback(null, 'TRYCREATE');
return updateQuota(() => callback(null, 'TRYCREATE'));
}
let uidNext = item.value.uidNext;
@ -667,9 +755,12 @@ server.onCopy = function (path, update, session, callback) {
database.collection('messages').insertOne(message, err => {
if (err) {
return callback(err);
return updateQuota(() => callback(err));
}
copiedMessages++;
copiedStorage += Number(message.size) || 0;
// remove link to message from attachments (if any exist)
database.collection('attachments.files').updateMany({
'metadata.messages': sourceId

143
lib/message-handler.js
View file

@ -83,76 +83,109 @@ class MessageHandler {
return callback(new Error('The user you are trying to contact is receiving mail at a rate that prevents additional messages from being delivered. Please resend your message at a later time'));
}
// acquire new UID+MODSEQ
this.database.collection('mailboxes').findOneAndUpdate({
_id: mailbox._id
this.database.collection('users').findOneAndUpdate({
_id: mailbox.user
}, {
$inc: {
// allocate bot UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1
storageUsed: size,
messages: 1
}
}, (err, item) => {
}, err => {
if (err) {
this.redlock.releaseLock(lock, () => false);
return callback(err);
}
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
this.redlock.releaseLock(lock, () => false);
return callback(err);
}
let mailbox = item.value;
let internaldate = options.date && new Date(options.date) || new Date();
let headerdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
if (!headerdate || headerdate.toString() === 'Invalid Date') {
headerdate = internaldate;
}
let message = {
_id: id,
mailbox: mailbox._id,
uid: mailbox.uidNext,
internaldate,
headerdate,
flags: [].concat(options.flags || []),
size,
meta: options.meta || {},
modseq: mailbox.modifyIndex + 1,
mimeTree,
envelope,
bodystructure,
messageId
let rollback = err => {
this.database.collection('users').findOneAndUpdate({
_id: mailbox.user
}, {
$inc: {
storageUsed: -size,
messages: -1
}
}, () => {
this.redlock.releaseLock(lock, () => callback(err));
});
};
this.database.collection('messages').insertOne(message, err => {
// acquire new UID+MODSEQ
this.database.collection('mailboxes').findOneAndUpdate({
_id: mailbox._id
}, {
$inc: {
// allocate bot UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1,
storageUsed: size,
messages: 1
}
}, (err, item) => {
if (err) {
this.redlock.releaseLock(lock, () => false);
return callback(err);
return rollback(err);
}
let uidValidity = mailbox.uidValidity;
let uid = message.uid;
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
return rollback(err);
}
this.notifier.addEntries(mailbox, false, {
command: 'EXISTS',
uid: message.uid,
message: message._id,
modseq: message.modseq
}, () => {
let mailbox = item.value;
this.redlock.releaseLock(lock, () => {
this.notifier.fire(mailbox.user, mailbox.path);
return callback(null, true, {
uidValidity,
uid
let internaldate = options.date && new Date(options.date) || new Date();
let headerdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
if (!headerdate || headerdate.toString() === 'Invalid Date') {
headerdate = internaldate;
}
let message = {
_id: id,
mailbox: mailbox._id,
uid: mailbox.uidNext,
internaldate,
headerdate,
flags: [].concat(options.flags || []),
size,
meta: options.meta || {},
modseq: mailbox.modifyIndex + 1,
mimeTree,
envelope,
bodystructure,
messageId
};
this.database.collection('messages').insertOne(message, err => {
if (err) {
return this.database.collection('mailboxes').findOneAndUpdate({
_id: mailbox._id
}, {
$inc: {
storageUsed: -size,
messages: -1
}
}, () => rollback(err));
}
let uidValidity = mailbox.uidValidity;
let uid = message.uid;
this.notifier.addEntries(mailbox, false, {
command: 'EXISTS',
uid: message.uid,
message: message._id,
modseq: message.modseq
}, () => {
this.redlock.releaseLock(lock, () => {
this.notifier.fire(mailbox.user, mailbox.path);
return callback(null, true, {
uidValidity,
uid
});
});
});
});