nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2025-11-07 06:56:13 +08:00
Code Issues Projects Releases Packages Wiki Activity

Added method to delete users

Browse source
This commit is contained in:
Andris Reinman 2017-11-17 13:37:53 +02:00
parent ba3beb3138
commit b54e110020
14 changed files with 1378 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -261,8 +261,10 @@ Shard the following collections by these keys:
sh.enableSharding('wildduck');
// consider using mailbox:hashed for messages only with large shard chunk size
sh.shardCollection('wildduck.messages', { mailbox: 1, uid: 1 });
sh.shardCollection('wildduck.archived', { user: 1, _id: 1 });
sh.shardCollection('wildduck.threads', { user: 'hashed' });
sh.shardCollection('wildduck.messagelog', { id: 'hashed' });
sh.shardCollection('wildduck.authlog', { user: 'hashed' });
// attachment _id is a sha256 hash of attachment contents
sh.shardCollection('wildduck.attachments.files', { _id: 'hashed' });
sh.shardCollection('wildduck.attachments.chunks', { files_id: 'hashed' });

153
docs/api.md
View file

@ -334,6 +334,30 @@ Response for a successful operation:
}
```
### Delete user
#### DELETE /users/{user}
Deletes user data form database. Messages are not immediately deleted but marked to be deleted in 2 days. Various relate dlog entries also expire naturally and are not touched by this call
**Parameters**
- **user** is the ID of the user
**Example**
```
curl -XDELETE "http://localhost:8080/users/5970860fcdb513ce633407a1"
```
Response for a successful operation:
```json
{
"success": true
}
```
### Log out user from all IMAP sessions
#### PUT /users/{user}/logout
@ -1476,6 +1500,7 @@ Response for a successful operation:
"success": true
}
```
### Delete a message
#### DELETE /users/{user}/mailboxes/{mailbox}/messages/{message}
@ -1871,6 +1896,134 @@ Response for a successful operation:
}
```
## Archive
Deleted messages are moved to temporary archive from where these are purged after configured delay (defaults to 2 weeks). During that window it is possible to list and restore archived messages. Restoring an archived message resets the UID of a message. Archived messages do not count against user quota.
### List archived messages
#### GET /user/{user}/archived
Lists archived messages for an user. This is similar to listing mailbox messages, major difference being that archived message IDs are not numeric but hex strings
**Parameters**
- **user** (required) is the ID of the user
- **order** optional message ordering, either "asc" or "desc". Defaults to "desc" (newer first)
**Example**
```
curl "http://localhost:8080/users/59467f27535f8f0f067ba8e6/archived"
```
Response for a successful operation:
```json
{
"success": true,
"total": 1,
"page": 1,
"previousCursor": false,
"nextCursor": false,
"specialUse": null,
"results": [
{
"id": "5a0d7baa221311cf2d8f145e",
"mailbox": "59467f27535f8f0f067ba8e6",
"thread": "5971da7754cfdc7f0983bbde",
"from": {
"address": "sender@example.com",
"name": "Sender Name"
},
"subject": "Subject line",
"date": "2011-11-02T19:19:08.000Z",
"intro": "Beginning text in the message…",
"attachments": false,
"seen": true,
"deleted": false,
"flagged": false,
"draft": false
}
]
}
```
### Get archived message details
#### GET /users/{user}/archived/{message}
Returns data about a specific message. This is similar to listing mailbox message, major difference being that archived message ID is not numeric but a hex strings
**Parameters**
- **user** (required) is the ID of the user
- **message** (required) is the ID of the message
**Example**
```
curl "http://localhost:8080/users/59467f27535f8f0f067ba8e6/archived/5a0d7baa221311cf2d8f145e"
```
Response for a successful operation:
```json
{
"success": true,
"id": "5a0d7baa221311cf2d8f145e",
"from": {
"address": "sender@example.com",
"name": "Sender Name"
},
"to": [
{
"address": "testuser@example.com",
"name": "Test User"
}
],
"subject": "Subject line",
"messageId": "<FA472D2A-092E-44BC-9D38-AFACE48AB98E@example.com>",
"date": "2011-11-02T19:19:08.000Z",
"seen": true,
"deleted": false,
"flagged": false,
"draft": false,
"html": [
"Notice that the HTML content is an array of HTML strings"
],
"attachments": []
}
```
### Restore archived message
#### POST /users/{user}/archived/{message}/restore
Restores archived message
**Parameters**
- **user** (required) is the ID of the user
- **message** (required) is the ID of the message
- **mailbox** is an optional ID of the destination mailbox. By default the message is restored to the mailbox it was deleted from or to INBOX if the source mailbox does not exist anymore
**Example**
```
curl -XPOST "http://localhost:8080/users/59467f27535f8f0f067ba8e6/archived/5a0d7baa221311cf2d8f145e/restore" -H 'content-type: application/json' -d '{}'
```
Response for a successful operation includes the mailbox ID the message was restored to and the updated UID of the message:
```json
{
"success": true,
"mailbox": "5a05ad49484b251f07951b22",
"uid": 40
}
```
## Quota
### Recalculate user quota

278
imap.js
View file

@ -80,11 +80,24 @@ function clearExpiredMessages() {
gcTimeout.unref();
return;
} else if (!lock.success) {
logger.debug(
{
tnx: 'gc'
},
'Lock already acquired'
);
gcTimeout = setTimeout(clearExpiredMessages, consts.GC_INTERVAL);
gcTimeout.unref();
return;
}
logger.debug(
{
tnx: 'gc'
},
'Got lock for garbage collector'
);
let done = () => {
gcLock.releaseLock(lock, err => {
if (err) {
@ -102,36 +115,49 @@ function clearExpiredMessages() {
});
};
if (!config.imap.disableRetention) {
if (config.imap.disableRetention) {
// delete all attachments that do not have any active links to message objects
// do not touch expired messages
return messageHandler.attachmentStorage.deleteOrphaned(() => done(null, true));
}
// find and delete all messages that are expired
// NB! scattered query, searches over all mailboxes and thus over all shards
let cursor = db.database
.collection('messages')
.find({
exp: true,
rdate: {
$lte: Date.now()
}
})
.project({
_id: true,
mailbox: true,
uid: true,
size: true,
'mimeTree.attachmentMap': true,
magic: true,
unseen: true
let deleteOrphaned = next => {
// delete all attachments that do not have any active links to message objects
messageHandler.attachmentStorage.deleteOrphaned(() => {
next(null, true);
});
};
let deleted = 0;
let clear = () =>
cursor.close(() => {
// delete all attachments that do not have any active links to message objects
messageHandler.attachmentStorage.deleteOrphaned(() => {
let archiveExpiredMessages = next => {
logger.debug(
{
tnx: 'gc'
},
'Archiving expired messages'
);
// find and delete all messages that are expired
// NB! scattered query, searches over all mailboxes and thus over all shards
let cursor = db.database
.collection('messages')
.find({
exp: true,
rdate: {
$lte: Date.now()
}
})
.project({
_id: true,
mailbox: true,
uid: true,
size: true,
'mimeTree.attachmentMap': true,
magic: true,
unseen: true
});
let deleted = 0;
let clear = () =>
cursor.close(() => {
if (deleted) {
logger.debug(
{
@ -141,56 +167,176 @@ function clearExpiredMessages() {
deleted
);
}
done(null, true);
return deleteOrphaned(next);
});
});
let processNext = () => {
if (Date.now() - startTime > consts.GC_INTERVAL * 0.8) {
// deleting expired messages has taken too long time, cancel
return clear();
}
cursor.next((err, message) => {
if (err) {
return done(err);
}
if (!message) {
let processNext = () => {
if (Date.now() - startTime > consts.GC_INTERVAL * 0.8) {
// deleting expired messages has taken too long time, cancel
return clear();
}
logger.info(
{
tnx: 'gc',
err
},
'Deleting expired message id=%s',
message._id
);
gcTimeout = setTimeout(clearExpiredMessages, consts.GC_INTERVAL);
messageHandler.del(
{
message,
skipAttachments: true
},
err => {
if (err) {
return cursor.close(() => done(err));
}
deleted++;
if (consts.GC_DELAY_DELETE) {
setTimeout(processNext, consts.GC_DELAY_DELETE);
} else {
setImmediate(processNext);
}
cursor.next((err, messageData) => {
if (err) {
return done(err);
}
);
});
if (!messageData) {
return clear();
}
messageHandler.del(
{
messageData,
// do not archive messages of deleted users
archive: !messageData.userDeleted
},
err => {
if (err) {
logger.error(
{
tnx: 'gc',
err
},
'Failed to delete expired message id=%s. %s',
messageData._id,
err.message
);
return cursor.close(() => done(err));
}
logger.debug(
{
tnx: 'gc',
err
},
'Deleted expired message id=%s',
messageData._id
);
deleted++;
if (consts.GC_DELAY_DELETE) {
setTimeout(processNext, consts.GC_DELAY_DELETE);
} else {
setImmediate(processNext);
}
}
);
});
};
processNext();
};
processNext();
let purgeExpiredMessages = next => {
logger.debug(
{
tnx: 'gc'
},
'Purging archived messages'
);
// find and delete all messages that are expired
// NB! scattered query, searches over all mailboxes and thus over all shards
let cursor = db.database
.collection('archived')
.find({
exp: true,
rdate: {
$lte: Date.now()
}
})
.project({
_id: true,
mailbox: true,
uid: true,
size: true,
'mimeTree.attachmentMap': true,
magic: true,
unseen: true
});
let deleted = 0;
let clear = () =>
cursor.close(() => {
if (deleted) {
logger.debug(
{
tnx: 'gc'
},
'Purged %s messages',
deleted
);
}
return deleteOrphaned(next);
});
let processNext = () => {
if (Date.now() - startTime > consts.GC_INTERVAL * 0.8) {
// deleting expired messages has taken too long time, cancel
return clear();
}
cursor.next((err, messageData) => {
if (err) {
return done(err);
}
if (!messageData) {
return clear();
}
db.database.collection('archived').deleteOne({ _id: messageData._id }, err => {
if (err) {
//failed to delete
logger.error(
{
tnx: 'gc',
err
},
'Failed to delete archived message id=%s. %s',
messageData._id,
err.message
);
return cursor.close(() => done(err));
}
logger.debug(
{
tnx: 'gc'
},
'Deleted archived message id=%s',
messageData._id
);
let attachmentIds = Object.keys(messageData.mimeTree.attachmentMap || {}).map(key => messageData.mimeTree.attachmentMap[key]);
if (!attachmentIds.length) {
// no stored attachments
deleted++;
if (consts.GC_DELAY_DELETE) {
setTimeout(processNext, consts.GC_DELAY_DELETE);
} else {
setImmediate(processNext);
}
return;
}
messageHandler.attachmentStorage.updateMany(attachmentIds, -1, -messageData.magic, err => {
if (err) {
// should we care about this error?
}
deleted++;
if (consts.GC_DELAY_DELETE) {
setTimeout(processNext, consts.GC_DELAY_DELETE);
} else {
setImmediate(processNext);
}
});
});
});
};
processNext();
};
archiveExpiredMessages(() => purgeExpiredMessages(done));
});
}

37
indexes.yaml
View file

@ -86,9 +86,19 @@ indexes:
- collection: asps
type: users # index applies to users database
index:
name: user
name: asps_user
key:
user: 1
active: 1
expires: 1
- collection: asps
index:
name: asps_autoexpire
# autoremove expired asps entries after 180 days
expireAfterSeconds: 15552000
key:
expires: 1
# Indexes for the authentication log collection
- collection: authlog
@ -304,6 +314,31 @@ indexes:
key:
size: 1
# indexes for deleted messages
- collection: archived
index:
name: user_messages
key:
user: 1
_id: 1
# indexes for deleted messages
- collection: archived
index:
name: user_messages_desc
key:
user: 1
_id: -1
- collection: archived
index:
name: retention_time
partialFilterExpression:
exp: true
key:
exp: 1
rdate: 1
# Indexes for the attachments collection
# attachments.files collection should be sharded by _id (hash)
# attachments.chunks collection should be sharded by files_id (hash)

11
lib/api/asps.js
View file

@ -56,7 +56,16 @@ module.exports = (db, server, userHandler) => {
db.users
.collection('asps')
.find({
user
user,
active: true,
$or: [
{
expires: false
},
{
expires: { $gt: new Date() }
}
]
})
.sort({ _id: 1 })
.toArray((err, asps) => {

541
lib/api/messages.js
View file

@ -1433,6 +1433,538 @@ module.exports = (db, server, messageHandler) => {
});
});
});
server.get({ name: 'archived', path: '/users/:user/archived' }, (req, res, next) => {
res.charSet('utf-8');
const schema = Joi.object().keys({
user: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
limit: Joi.number()
.empty('')
.default(20)
.min(1)
.max(250),
next: Joi.string()
.empty('')
.alphanum()
.max(1024),
previous: Joi.string()
.empty('')
.alphanum()
.max(1024),
order: Joi.any()
.empty('')
.allow(['asc', 'desc'])
.default('desc'),
page: Joi.number()
.empty('')
.default(1)
});
req.query.user = req.params.user;
const result = Joi.validate(req.query, schema, {
abortEarly: false,
convert: true,
allowUnknown: true
});
if (result.error) {
res.json({
error: result.error.message
});
return next();
}
let user = new ObjectID(result.value.user);
let limit = result.value.limit;
let page = result.value.page;
let pageNext = result.value.next;
let pagePrevious = result.value.previous;
let sortAscending = result.value.order === 'asc';
getArchivedMessageCount(db, user, (err, total) => {
if (err) {
res.json({
error: err.message
});
return next();
}
let opts = {
limit,
query: { user },
fields: {
_id: true,
mailbox: true,
uid: true,
'meta.from': true,
hdate: true,
subject: true,
'mimeTree.parsedHeader.from': true,
'mimeTree.parsedHeader.sender': true,
'mimeTree.parsedHeader.content-type': true,
ha: true,
intro: true,
unseen: true,
undeleted: true,
flagged: true,
draft: true,
thread: true
},
paginatedField: '_id',
sortAscending
};
if (pageNext) {
opts.next = pageNext;
} else if (pagePrevious) {
opts.previous = pagePrevious;
}
MongoPaging.find(db.database.collection('archived'), opts, (err, result) => {
if (err) {
res.json({
error: result.error.message
});
return next();
}
if (!result.hasPrevious) {
page = 1;
}
let response = {
success: true,
total,
page,
previousCursor: result.hasPrevious ? result.previous : false,
nextCursor: result.hasNext ? result.next : false,
results: (result.results || []).map(messageData => {
let parsedHeader = (messageData.mimeTree && messageData.mimeTree.parsedHeader) || {};
let from = parsedHeader.from ||
parsedHeader.sender || [
{
name: '',
address: (messageData.meta && messageData.meta.from) || ''
}
];
tools.decodeAddresses(from);
let response = {
id: messageData._id,
mailbox: messageData.mailbox,
thread: messageData.thread,
from: from && from[0],
subject: messageData.subject,
date: messageData.hdate.toISOString(),
intro: messageData.intro,
attachments: !!messageData.ha,
seen: !messageData.unseen,
deleted: !messageData.undeleted,
flagged: messageData.flagged,
draft: messageData.draft,
url: server.router.render('archived_message', { user, message: messageData._id })
};
let parsedContentType = parsedHeader['content-type'];
if (parsedContentType) {
response.contentType = {
value: parsedContentType.value
};
if (parsedContentType.hasParams) {
response.contentType.params = parsedContentType.params;
}
if (parsedContentType.subtype === 'encrypted') {
response.encrypted = true;
}
}
return response;
})
};
res.json(response);
return next();
});
});
});
server.get({ name: 'archived_message', path: '/users/:user/archived/:message' }, (req, res, next) => {
res.charSet('utf-8');
const schema = Joi.object().keys({
user: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
message: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
replaceCidLinks: Joi.boolean()
.truthy(['Y', 'true', 'yes', 1])
.default(false)
});
if (req.query.replaceCidLinks) {
req.params.replaceCidLinks = req.query.replaceCidLinks;
}
const result = Joi.validate(req.params, schema, {
abortEarly: false,
convert: true
});
if (result.error) {
res.json({
error: result.error.message
});
return next();
}
let user = new ObjectID(result.value.user);
let message = new ObjectID(result.value.message);
let replaceCidLinks = result.value.replaceCidLinks;
db.database.collection('archived').findOne({
_id: message,
user
}, {
fields: {
_id: true,
mailbox: true,
user: true,
thread: true,
'meta.from': true,
'meta.to': true,
hdate: true,
'mimeTree.parsedHeader': true,
subject: true,
msgid: true,
exp: true,
rdate: true,
ha: true,
unseen: true,
undeleted: true,
flagged: true,
draft: true,
attachments: true,
html: true,
text: true,
textFooter: true,
forwardTargets: true
}
}, (err, messageData) => {
if (err) {
res.json({
error: 'MongoDB Error: ' + err.message
});
return next();
}
if (!messageData || messageData.user.toString() !== user.toString()) {
res.json({
error: 'This message does not exist'
});
return next();
}
let parsedHeader = (messageData.mimeTree && messageData.mimeTree.parsedHeader) || {};
let from = parsedHeader.from ||
parsedHeader.sender || [
{
name: '',
address: (messageData.meta && messageData.meta.from) || ''
}
];
tools.decodeAddresses(from);
let replyTo = parsedHeader['reply-to'];
if (replyTo) {
tools.decodeAddresses(replyTo);
}
let to = parsedHeader.to;
if (to) {
tools.decodeAddresses(to);
}
let cc = parsedHeader.cc;
if (cc) {
tools.decodeAddresses(cc);
}
let list;
if (parsedHeader['list-id'] || parsedHeader['list-unsubscribe']) {
let listId = parsedHeader['list-id'];
if (listId) {
listId = addressparser(listId.toString());
tools.decodeAddresses(listId);
listId = listId.shift();
}
let listUnsubscribe = parsedHeader['list-unsubscribe'];
if (listUnsubscribe) {
listUnsubscribe = addressparser(listUnsubscribe.toString());
tools.decodeAddresses(listUnsubscribe);
}
list = {
id: listId,
unsubscribe: listUnsubscribe
};
}
let expires;
if (messageData.exp) {
expires = new Date(messageData.rdate).toISOString();
}
messageData.text = (messageData.text || '') + (messageData.textFooter || '');
if (replaceCidLinks) {
messageData.html = (messageData.html || []).map(html =>
html.replace(/attachment:([a-f0-9]+)\/(ATT\d+)/g, (str, mid, aid) =>
server.router.render('archived_attachment', { user, message, attachment: aid })
)
);
messageData.text = messageData.text.replace(/attachment:([a-f0-9]+)\/(ATT\d+)/g, (str, mid, aid) =>
server.router.render('archived_attachment', { user, message, attachment: aid })
);
}
let response = {
success: true,
id: message,
mailbox: messageData.mailbox,
from: from[0],
replyTo,
to,
cc,
subject: messageData.subject,
messageId: messageData.msgid,
date: messageData.hdate.toISOString(),
list,
expires,
seen: !messageData.unseen,
deleted: !messageData.undeleted,
flagged: messageData.flagged,
draft: messageData.draft,
html: messageData.html,
text: messageData.text,
forwardTargets: messageData.forwardTargets,
attachments: (messageData.attachments || []).map(attachment => {
attachment.url = server.router.render('archived_attachment', { user, message, attachment: attachment.id });
return attachment;
})
};
let parsedContentType = parsedHeader['content-type'];
if (parsedContentType) {
response.contentType = {
value: parsedContentType.value
};
if (parsedContentType.hasParams) {
response.contentType.params = parsedContentType.params;
}
if (parsedContentType.subtype === 'encrypted') {
response.encrypted = true;
}
}
res.json(response);
return next();
});
});
server.get({ name: 'archived_attachment', path: '/users/:user/archived/:message/attachments/:attachment' }, (req, res, next) => {
const schema = Joi.object().keys({
user: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
message: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
attachment: Joi.string()
.regex(/^ATT\d+$/i)
.uppercase()
.required()
});
const result = Joi.validate(req.params, schema, {
abortEarly: false,
convert: true
});
if (result.error) {
res.json({
error: result.error.message
});
return next();
}
let user = new ObjectID(result.value.user);
let message = new ObjectID(result.value.message);
let attachment = result.value.attachment;
db.database.collection('archived').findOne({
user,
_id: message
}, {
fields: {
_id: true,
user: true,
attachments: true,
'mimeTree.attachmentMap': true
}
}, (err, messageData) => {
if (err) {
res.json({
error: 'MongoDB Error: ' + err.message
});
return next();
}
if (!messageData || messageData.user.toString() !== user.toString()) {
res.json({
error: 'This message does not exist'
});
return next();
}
let attachmentId = messageData.mimeTree.attachmentMap && messageData.mimeTree.attachmentMap[attachment];
if (!attachmentId) {
res.json({
error: 'This attachment does not exist'
});
return next();
}
messageHandler.attachmentStorage.get(attachmentId, (err, attachmentData) => {
if (err) {
res.json({
error: err.message
});
return next();
}
res.writeHead(200, {
'Content-Type': attachmentData.contentType || 'application/octet-stream'
});
let decode = true;
if (attachmentData.metadata.decoded) {
attachmentData.metadata.decoded = false;
decode = false;
}
let attachmentStream = messageHandler.attachmentStorage.createReadStream(attachmentId, attachmentData);
attachmentStream.once('error', err => res.emit('error', err));
if (!decode) {
return attachmentStream.pipe(res);
}
if (attachmentData.transferEncoding === 'base64') {
attachmentStream.pipe(new libbase64.Decoder()).pipe(res);
} else if (attachmentData.transferEncoding === 'quoted-printable') {
attachmentStream.pipe(new libqp.Decoder()).pipe(res);
} else {
attachmentStream.pipe(res);
}
});
});
});
server.post({ name: 'archived_restore', path: '/users/:user/archived/:message/restore' }, (req, res, next) => {
res.charSet('utf-8');
const schema = Joi.object().keys({
user: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
message: Joi.string()
.hex()
.lowercase()
.length(24)
.required(),
mailbox: Joi.string()
.hex()
.lowercase()
.length(24)
});
const result = Joi.validate(req.params, schema, {
abortEarly: false,
convert: true
});
if (result.error) {
res.json({
error: result.error.message
});
return next();
}
let user = new ObjectID(result.value.user);
let message = new ObjectID(result.value.message);
let mailbox = result.value.mailbox ? new ObjectID(result.value.mailbox) : false;
db.database.collection('archived').findOne({
_id: message,
user
}, (err, messageData) => {
if (err) {
res.json({
error: 'MongoDB Error: ' + err.message
});
return next();
}
if (!messageData || messageData.user.toString() !== user.toString()) {
res.json({
error: 'This message does not exist'
});
return next();
}
messageData.mailbox = mailbox || messageData.mailbox;
messageHandler.put(messageData, (err, response) => {
if (err) {
res.json({
error: err.message
});
} else if (!response) {
res.json({
succese: false,
error: 'Failed to restore message'
});
} else {
response.success = true;
res.json(response);
return db.database.collection('archived').deleteOne({ _id: messageData._id }, () => next());
}
return next();
});
});
});
};
function getFilteredMessageCount(db, filter, done) {
@ -1449,6 +1981,15 @@ function getFilteredMessageCount(db, filter, done) {
});
}
function getArchivedMessageCount(db, user, done) {
db.database.collection('archived').count({ user }, (err, total) => {
if (err) {
return done(err);
}
done(null, total);
});
}
function leftPad(val, chr, len) {
return chr.repeat(len - val.toString().length) + val;
}

38
lib/api/users.js
View file

@ -783,6 +783,44 @@ module.exports = (db, server, userHandler) => {
return next();
});
});
server.del('/users/:user', (req, res, next) => {
res.charSet('utf-8');
const schema = Joi.object().keys({
user: Joi.string()
.hex()
.lowercase()
.length(24)
.required()
});
const result = Joi.validate(req.params, schema, {
abortEarly: false,
convert: true
});
if (result.error) {
res.json({
error: result.error.message
});
return next();
}
let user = new ObjectID(result.value.user);
userHandler.delete(user, (err, status) => {
if (err) {
res.json({
error: err.message
});
return next();
}
res.json({
success: status
});
return next();
});
});
};
function getKeyInfo(pubKey) {

8
lib/consts.js
View file

@ -11,8 +11,9 @@ module.exports = {
// artificail delay between deleting next expired message in ms
// set to 0 to disable
GC_DELAY_DELETE: 80,
GC_DELAY_DELETE: 0,
// default
MAX_STORAGE: 1 * (1024 * 1024 * 1024),
MAX_RECIPIENTS: 2000,
MAX_FORWARDS: 2000,
@ -49,5 +50,8 @@ module.exports = {
SCOPES: ['imap', 'pop3', 'smtp'],
// Refuse to process messages larger than 64 MB. Allowing larger messages might cause jumbo chunks in MongoDB
MAX_ALLOWE_MESSAGE_SIZE: 64 * 1024 * 1024
MAX_ALLOWE_MESSAGE_SIZE: 64 * 1024 * 1024,
// how long to keep deleted messages around before purgeing
ARCHIVE_TIME: 2 * 7 * 24 * 3600 * 1000
};

80
lib/handlers/on-expunge.js
View file

@ -1,9 +1,10 @@
'use strict';
const consts = require('../consts');
const db = require('../db');
// EXPUNGE deletes all messages in selected mailbox marked with \Delete
module.exports = (server, messageHandler) => (path, update, session, callback) => {
module.exports = server => (path, update, session, callback) => {
server.logger.debug(
{
tnx: 'expunge',
@ -36,14 +37,6 @@ module.exports = (server, messageHandler) => (path, update, session, callback) =
$lt: mailboxData.uidNext
}
})
.project({
_id: true,
uid: true,
size: true,
'mimeTree.attachmentMap': true,
magic: true,
unseen: true
})
.sort([['uid', 1]]);
let deletedMessages = 0;
@ -68,66 +61,61 @@ module.exports = (server, messageHandler) => (path, update, session, callback) =
};
let processNext = () => {
cursor.next((err, message) => {
cursor.next((err, messageData) => {
if (err) {
return updateQuota(() => callback(err));
}
if (!message) {
if (!messageData) {
return cursor.close(() => {
updateQuota(() => {
server.notifier.fire(session.user.id, path);
session.writeStream.write({
tag: '*',
command: String(session.selected.uidList.length),
attributes: [
{
type: 'atom',
value: 'EXISTS'
}
]
});
return callback(null, true);
});
});
}
if (!update.silent) {
session.writeStream.write(session.formatResponse('EXPUNGE', message.uid));
}
db.database.collection('messages').deleteOne({
_id: message._id,
mailbox: mailboxData._id,
uid: message.uid
}, err => {
messageData.exp = true;
messageData.rdate = Date.now() + consts.ARCHIVE_TIME;
db.database.collection('archived').insertOne(messageData, err => {
if (err) {
return updateQuota(() => cursor.close(() => callback(err)));
}
deletedMessages++;
deletedStorage += Number(message.size) || 0;
if (!update.silent) {
session.writeStream.write(session.formatResponse('EXPUNGE', messageData.uid));
}
let attachmentIds = Object.keys(message.mimeTree.attachmentMap || {}).map(key => message.mimeTree.attachmentMap[key]);
db.database.collection('messages').deleteOne({
_id: messageData._id,
mailbox: mailboxData._id,
uid: messageData.uid
}, err => {
if (err) {
return updateQuota(() => cursor.close(() => callback(err)));
}
deletedMessages++;
deletedStorage += Number(messageData.size) || 0;
if (!attachmentIds.length) {
// not stored attachments
return server.notifier.addEntries(
session.user.id,
path,
{
command: 'EXPUNGE',
ignore: session.id,
uid: message.uid,
message: message._id,
unseen: message.unseen
},
processNext
);
}
messageHandler.attachmentStorage.updateMany(attachmentIds, -1, -message.magic, err => {
if (err) {
// should we care about this error?
}
server.notifier.addEntries(
session.user.id,
path,
{
command: 'EXPUNGE',
ignore: session.id,
uid: message.uid,
message: message._id,
unseen: message.unseen
uid: messageData.uid,
message: messageData._id,
unseen: messageData.unseen
},
processNext
);

3
lib/handlers/on-move.js
View file

@ -28,7 +28,8 @@ module.exports = (server, messageHandler) => (path, update, session, callback) =
},
session,
// list of UIDs to move
messages: update.messages
messages: update.messages,
showExpunged: true
},
(...args) => {
if (args[0]) {

98
lib/mailbox-handler.js
View file

@ -150,6 +150,9 @@ class MailboxHandler {
});
}
/**
* Deletes a mailbox. Does not immediatelly release quota as the messages get deleted after a while
*/
del(user, mailbox, callback) {
this.database.collection('mailboxes').findOne({
_id: mailbox,
@ -180,78 +183,33 @@ class MailboxHandler {
mailbox
},
() => {
// calculate mailbox size by aggregating the size's of all messages
this.database
.collection('messages')
.aggregate(
[
{
$match: {
mailbox,
uid: {
$gt: 0,
$lt: mailboxData.uidNext + 100
}
}
},
{
$group: {
_id: {
mailbox: '$mailbox'
},
storageUsed: {
$sum: '$size'
}
}
}
],
{
cursor: {
batchSize: 1
}
}
)
.toArray((err, res) => {
if (err) {
return callback(err);
}
this.database.collection('messages').updateMany({
mailbox,
uid: {
$gt: 0,
$lt: mailboxData.uidNext + 100
}
}, {
$set: {
exp: true,
// make sure the messages are in top of the expire queue
rdate: Date.now() - 24 * 3600 * 1000
}
}, {
multi: true,
w: 1
}, err => {
if (err) {
return callback(err);
}
let storageUsed = (res && res[0] && res[0].storageUsed) || 0;
let done = () => {
this.notifier.fire(mailboxData.user, mailboxData.path);
callback(null, true);
};
this.database.collection('messages').deleteMany({
mailbox,
uid: {
$gt: 0,
$lt: mailboxData.uidNext + 100
}
}, err => {
if (err) {
return callback(err);
}
let done = () => {
this.notifier.fire(mailboxData.user, mailboxData.path);
callback(null, true);
};
if (!storageUsed) {
return done();
}
// decrement quota counters
this.users.collection('users').findOneAndUpdate(
{
_id: mailbox.user
},
{
$inc: {
storageUsed: -Number(storageUsed) || 0
}
},
done
);
});
});
return done();
});
}
);
});

267
lib/message-handler.js
View file

@ -45,21 +45,24 @@ class MessageHandler {
}
getMailbox(options, callback) {
let query = {};
if (options.mailbox) {
if (typeof options.mailbox === 'object' && options.mailbox._id) {
return setImmediate(() => callback(null, options.mailbox));
}
query._id = options.mailbox;
if (options.user) {
query.user = options.user;
}
} else {
query.user = options.user;
if (options.specialUse) {
query.specialUse = options.specialUse;
let query = options.query;
if (!query) {
query = {};
if (options.mailbox) {
if (typeof options.mailbox === 'object' && options.mailbox._id) {
return setImmediate(() => callback(null, options.mailbox));
}
query._id = options.mailbox;
if (options.user) {
query.user = options.user;
}
} else {
query.path = options.path;
query.user = options.user;
if (options.specialUse) {
query.specialUse = options.specialUse;
} else {
query.path = options.path;
}
}
}
@ -154,6 +157,9 @@ class MessageHandler {
exp: !!mailboxData.retention,
rdate: Date.now() + (mailboxData.retention || 0),
// make sure the field exists. it is set to true when user is deleted
userDeleted: false,
idate,
hdate,
flags,
@ -582,68 +588,91 @@ class MessageHandler {
}
del(options, callback) {
let messageData = options.message;
let messageData = options.messageData;
this.getMailbox(
options.mailbox || {
mailbox: messageData.mailbox
},
(err, mailboxData) => {
if (err) {
if (err && !err.imapResponse) {
return callback(err);
}
this.database.collection('messages').deleteOne({
_id: messageData._id,
mailbox: mailboxData._id,
uid: messageData.uid
}, err => {
if (err) {
return callback(err);
let pushToArchive = next => {
if (!options.archive) {
return next();
}
messageData.exp = true;
messageData.rdate = Date.now() + consts.ARCHIVE_TIME;
this.database.collection('archived').insertOne(messageData, err => {
if (err) {
return callback(err);
}
return next();
});
};
this.updateQuota(
mailboxData._id,
{
storageUsed: -messageData.size
},
() => {
let updateAttachments = next => {
let attachmentIds = Object.keys(messageData.mimeTree.attachmentMap || {}).map(key => messageData.mimeTree.attachmentMap[key]);
if (!attachmentIds.length) {
return next();
pushToArchive(() => {
this.database.collection('messages').deleteOne({
_id: messageData._id,
mailbox: messageData.mailbox,
uid: messageData.uid
}, err => {
if (err) {
return callback(err);
}
this.updateQuota(
messageData.mailbox,
{
storageUsed: -messageData.size
},
() => {
if (!mailboxData) {
// deleted an orphan message
return callback(null, true);
}
this.attachmentStorage.deleteMany(attachmentIds, messageData.magic, next);
};
let updateAttachments = next => {
if (options.archive) {
// archived messages still need the attachments
return next();
}
updateAttachments(() => {
if (options.session && options.session.selected && options.session.selected.mailbox === mailboxData.path) {
options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageData.uid));
}
let attachmentIds = Object.keys(messageData.mimeTree.attachmentMap || {}).map(
key => messageData.mimeTree.attachmentMap[key]
);
if (!attachmentIds.length) {
return next();
}
this.notifier.addEntries(
mailboxData._id,
false,
{
command: 'EXPUNGE',
ignore: options.session && options.session.id,
uid: messageData.uid,
message: messageData._id,
unseen: messageData.unseen
},
() => {
this.notifier.fire(mailboxData.user, mailboxData.path);
this.attachmentStorage.deleteMany(attachmentIds, messageData.magic, next);
};
if (options.skipAttachments) {
updateAttachments(() => {
if (options.session && options.session.selected && options.session.selected.mailbox === mailboxData.path) {
options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageData.uid));
}
this.notifier.addEntries(
messageData.mailbox,
false,
{
command: 'EXPUNGE',
ignore: options.session && options.session.id,
uid: messageData.uid,
message: messageData._id,
unseen: messageData.unseen
},
() => {
this.notifier.fire(mailboxData.user, mailboxData.path);
return callback(null, true);
}
return callback(null, true);
}
);
});
}
);
);
});
}
);
});
});
}
);
@ -700,6 +729,19 @@ class MessageHandler {
});
};
if (sourceUid.length && options.showExpunged) {
options.session.writeStream.write({
tag: '*',
command: String(options.session.selected.uidList.length),
attributes: [
{
type: 'atom',
value: 'EXISTS'
}
]
});
}
if (existsEntries.length) {
// mark messages as deleted from old mailbox
return this.notifier.addEntries(mailboxData, false, removeEntries, () => {
@ -711,6 +753,7 @@ class MessageHandler {
});
});
}
next();
};
@ -872,6 +915,10 @@ class MessageHandler {
unseen
});
if (options.showExpunged) {
options.session.writeStream.write(options.session.formatResponse('EXPUNGE', messageUid));
}
let entry = {
command: 'EXISTS',
uid: uidNext,
@ -909,6 +956,108 @@ class MessageHandler {
});
}
put(messageData, callback) {
let getMailbox = next => {
this.getMailbox({ mailbox: messageData.mailbox }, (err, mailboxData) => {
if (err && !err.imapResponse) {
return callback(err);
}
if (mailboxData) {
return next(null, mailboxData);
}
this.getMailbox(
{
query: {
user: messageData.user,
path: 'INBOX'
}
},
callback
);
});
};
getMailbox((err, mailboxData) => {
if (err) {
return callback(err);
}
this.database.collection('mailboxes').findOneAndUpdate({
_id: mailboxData._id
}, {
$inc: {
uidNext: 1
}
}, {
uidNext: true
}, (err, item) => {
if (err) {
return callback(err);
}
if (!item || !item.value) {
return callback(new Error('Mailbox disappeared'));
}
let uidNext = item.value.uidNext;
// set new mailbox
messageData.mailbox = mailboxData._id;
// new mailbox means new UID
messageData.uid = uidNext;
// this will be changed later by the notification system
messageData.modseq = 0;
// retention settings
messageData.exp = !!mailboxData.retention;
messageData.rdate = Date.now() + (mailboxData.retention || 0);
if (['\\Junk', '\\Trash'].includes(mailboxData.specialUse) || !mailboxData.undeleted) {
delete messageData.searchable;
} else {
messageData.searchable = true;
}
let junk = false;
if (mailboxData.specialUse === '\\Junk' && !messageData.junk) {
messageData.junk = true;
junk = 1;
} else if (mailboxData.specialUse !== '\\Trash' && messageData.junk) {
delete messageData.junk;
junk = -1;
}
this.database.collection('messages').insertOne(messageData, (err, r) => {
if (err) {
return callback(err);
}
let insertId = r.insertedId;
let entry = {
command: 'EXISTS',
uid: uidNext,
message: insertId,
unseen: messageData.unseen
};
if (junk) {
entry.junk = junk;
}
// mark messages as added to new mailbox
this.notifier.addEntries(mailboxData, false, entry, () => {
this.notifier.fire(mailboxData.user, mailboxData.path);
return callback(null, {
mailbox: mailboxData._id,
uid: uidNext
});
});
});
});
});
}
generateIndexedHeaders(headersArray, options) {
// allow configuring extra header keys that are indexed
let indexedHeaders = options && options.indexedHeaders;

102
lib/user-handler.js
View file

@ -284,7 +284,16 @@ class UserHandler {
this.users
.collection('asps')
.find({
user: userData._id
user: userData._id,
active: true,
$or: [
{
expires: false
},
{
expires: { $gt: new Date() }
}
]
})
.toArray((err, asps) => {
if (err) {
@ -423,6 +432,8 @@ class UserHandler {
scopes,
password: bcrypt.hashSync(password, consts.BCRYPT_ROUNDS),
prefix,
active: true,
expires: data.expires || false,
created: new Date()
};
@ -467,15 +478,20 @@ class UserHandler {
}
deleteASP(user, asp, data, callback) {
this.users.collection('asps').deleteOne({
return this.users.collection('asps').findOneAndUpdate({
_id: asp,
user
}, (err, r) => {
}, {
$set: {
active: false,
expires: new Date()
}
}, (err, result) => {
if (err) {
return callback(err);
}
if (!r.deletedCount) {
if (!result || !result.value) {
return callback(new Error('Application Specific Password was not found'));
}
@ -1741,6 +1757,8 @@ class UserHandler {
if (user) {
entry.user = user;
} else {
entry.user = entry.user || new ObjectID('000000000000000000000000');
}
entry.action = entry.action || 'authentication';
@ -1782,6 +1800,82 @@ class UserHandler {
return callback(null, true);
});
}
// This method deletes non expireing records from database
delete(user, callback) {
this.database.collection('messages').updateMany({ user }, {
$set: {
exp: true,
rdate: new Date(Date.now() + 2 * 24 * 3600 * 1000),
userDeleted: true
}
}, err => {
if (err) {
log.error('USERDEL', 'Failed to delete messages for id=%s error=%s', user, err.message);
return callback(err);
}
let tryCount = 0;
let tryDelete = err => {
if (tryCount++ > 10) {
return callback(err);
}
this.database.collection('mailboxes').deleteMany({ user }, err => {
if (err) {
log.error('USERDEL', 'Failed to delete mailboxes for id=%s error=%s', user, err.message);
if (tryCount > 2) {
return setTimeout(() => tryDelete(err), 100);
}
}
this.users.collection('addresses').deleteMany({ user }, err => {
if (err) {
log.error('USERDEL', 'Failed to delete addresses for id=%s error=%s', user, err.message);
if (tryCount > 4) {
return setTimeout(() => tryDelete(err), 100);
}
}
this.users.collection('users').deleteOne({ _id: user }, err => {
if (err) {
log.error('USERDEL', 'Failed to delete user id=%s error=%s', user, err.message);
return setTimeout(() => tryDelete(err), 100);
}
this.users.collection('asps').deleteMany({ user }, err => {
if (err) {
log.error('USERDEL', 'Failed to delete asps for id=%s error=%s', user, err.message);
}
this.users.collection('filters').deleteMany({ user }, err => {
if (err) {
log.error('USERDEL', 'Failed to delete filters for id=%s error=%s', user, err.message);
}
this.users.collection('autoreplies').deleteMany({ user }, err => {
if (err) {
log.error('USERDEL', 'Failed to delete autoreplies for id=%s error=%s', user, err.message);
}
return this.logAuthEvent(
user,
{
action: 'delete user',
result: 'success'
},
() => callback(null, true)
);
});
});
});
});
});
});
};
setImmediate(tryDelete);
});
}
}
module.exports = UserHandler;

10
test/api-test.js
View file

@ -151,6 +151,16 @@ frisby
expect(response).to.exist;
expect(response.success).to.be.true;
expect(response.results.length).to.equal(2);
frisby
.create('DELETE users/{id}')
.delete(URL + '/users/' + userId)
.expectStatus(200)
.afterJSON(response => {
expect(response).to.exist;
expect(response.success).to.be.true;
})
.toss();
})
.toss();
})