mirror of
https://github.com/nodemailer/wildduck.git
synced 2024-09-20 07:16:05 +08:00
Added user field to indexes and message queries to enable sharding
This commit is contained in:
parent
da967e13a6
commit
3f82ba0be6
272
README.md
272
README.md
|
@ -166,6 +166,8 @@ Actual update data (information about new and deleted messages, flag updates and
|
|||
|
||||
## HTTP API
|
||||
|
||||
> **NB!** The HTTP API is being re-designed, do not build apps against the current API for now
|
||||
|
||||
Users, mailboxes and messages can be managed with HTTP requests against Wild Duck API
|
||||
|
||||
TODO:
|
||||
|
@ -378,267 +380,7 @@ Where
|
|||
|
||||
Recipient limits assume that messages are sent using ZoneMTA with [zonemta-wildduck](https://github.com/wildduck-email/zonemta-wildduck) plugin, otherwise the counters are not updated.
|
||||
|
||||
### GET /user/mailboxes
|
||||
|
||||
Returns all mailbox names for the user
|
||||
|
||||
Arguments
|
||||
|
||||
- **username** is the username of the user to modify
|
||||
|
||||
**Example**
|
||||
|
||||
```
|
||||
curl "http://localhost:8080/user/mailboxes?username=testuser"
|
||||
```
|
||||
|
||||
The response for successful operation should look like this:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"username": "testuser",
|
||||
"mailboxes": [
|
||||
{
|
||||
"id": "58d8f2ae240366dfd5d8049c",
|
||||
"path": "INBOX",
|
||||
"special": "Inbox",
|
||||
"messages": 100
|
||||
},
|
||||
{
|
||||
"id": "58d8f2ae240366dfd5d8049d",
|
||||
"path": "Sent Mail",
|
||||
"special": "Sent",
|
||||
"messages": 45
|
||||
},
|
||||
{
|
||||
"id": "58d8f2ae240366dfd5d8049f",
|
||||
"path": "Junk",
|
||||
"special": "Junk",
|
||||
"messages": 10
|
||||
},
|
||||
{
|
||||
"id": "58d8f2ae240366dfd5d8049e",
|
||||
"path": "Trash",
|
||||
"special": "Trash",
|
||||
"messages": 11
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### GET /mailbox/:id
|
||||
|
||||
List messages in a mailbox.
|
||||
|
||||
Parameters
|
||||
|
||||
- **id** is the mailbox ID
|
||||
- **size** is optional number to limit the length of the messages array (defaults to 20)
|
||||
- **before** is an optional paging number (see _next_ in response)
|
||||
- **after** is an optional paging number (see _prev_ in response)
|
||||
|
||||
Response includes the following fields
|
||||
|
||||
- **mailbox** is an object that lists some metadata about the current mailbox
|
||||
|
||||
- **id** is the mailbox ID
|
||||
- **path** is the folder path
|
||||
|
||||
- **next** is an URL fragment for retrieving the next page (or false if there are no more pages)
|
||||
|
||||
- **prev** is an URL fragment for retrieving the previous page (or false if it is the first page)
|
||||
|
||||
- **messages** is an array of messages in the mailbox
|
||||
|
||||
- **id** is the message ID
|
||||
- **date** is the date when this message was received
|
||||
- **ha** is a boolean that indicates if this messages has attachments or not
|
||||
- **intro** includes the first 256 characters from the message
|
||||
- **subject** is the message title
|
||||
- **from** is the From: field
|
||||
- **to** is the To: field
|
||||
- **cc** is the Cc: field
|
||||
- **bcc** is the Bcc: field
|
||||
|
||||
The response for successful listing should look like this:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"mailbox": {
|
||||
"id": "58dbf87fcff690a8c30470c7",
|
||||
"path": "INBOX"
|
||||
},
|
||||
"next": "/mailbox/58dbf87fcff690a8c30470c7?before=34&size=20",
|
||||
"prev": false,
|
||||
"messages": [
|
||||
{
|
||||
"id": "58e25243ab71621c3890417e",
|
||||
"date": "2017-04-03T13:46:44.226Z",
|
||||
"ha": true,
|
||||
"intro": "Welcome to Ryan Finnie's MIME torture test. This message was designed to introduce a couple of the newer features of MIME-aware MUAs, features that have come around since the days of the original MIME torture test. Just to be clear, this message SUPPLEMENT…",
|
||||
"subject": "ryan finnie's mime torture test v1.0",
|
||||
"from": "ryan finnie <rfinnie@domain.dom>",
|
||||
"to": "bob@domain.dom"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### GET /message/:id
|
||||
|
||||
Retrieves message information
|
||||
|
||||
Parameters
|
||||
|
||||
- **id** is the MongoDB _id as a string for a message
|
||||
- **mailbox** is optional Mailbox id. Use this to verify that the message is located at this mailbox
|
||||
|
||||
**Example**
|
||||
|
||||
```
|
||||
curl "http://localhost:8080/message/58d8299c5195c38e77c2daa5"
|
||||
```
|
||||
|
||||
Response message includes the following fields
|
||||
|
||||
- **id** is the id of the message
|
||||
- **headers** is an array that lists all headers of the message. A header is an object:
|
||||
|
||||
- **key** is the lowercase key of the header
|
||||
- **value** is the header value in unicode (all encoded values are decoded to utf-8). The value is capped at around 800 characters.
|
||||
|
||||
- **date** is the receive date (not header Date: field)
|
||||
|
||||
- **mailbox** is the id of the mailbox this messages belongs to
|
||||
|
||||
- **flags** is an array of IMAP flags for this message
|
||||
- **text** is the plaintext version of the message (derived from html if not present in message source)
|
||||
- **html** is the HTML version of the message (derived from plaintext if not present in message source). It is an array of strings, each array element corresponds to different MIME node and might have its own html header
|
||||
|
||||
- **attachments** is an array of attachment objects. Attachments can be shared between messages.
|
||||
|
||||
- **id** is the id of the attachment in the form of "ATT00001"
|
||||
- **fileName** is the name of the attachment. Autogenerated from Content-Type if not set in source
|
||||
- **contentType** is the MIME type of the message
|
||||
- **disposition** defines Content-Disposition and is either 'inline', 'attachment' or _false_
|
||||
- **transferEncoding** defines Content-Transfer-Encoding
|
||||
- **related** is a boolean value that states if the attachment should be hidden (_true_) or not. _Related_ attachments are usually embedded images
|
||||
- **sizeKb** is the approximate size of the attachment in kilobytes
|
||||
|
||||
#### Embedded images
|
||||
|
||||
HTML content has embedded images linked with the following URL structure:
|
||||
|
||||
```
|
||||
attachment:MESSAGE_ID/ATTACHMENT_ID
|
||||
```
|
||||
|
||||
For example:
|
||||
|
||||
```
|
||||
<img src="attachment:aaaaaa/bbbbbb">
|
||||
```
|
||||
|
||||
To fetch the actual attachment contents for this image, use the following url:
|
||||
|
||||
```
|
||||
http://localhost:8080/message/aaaaaa/attachment/bbbbbb
|
||||
```
|
||||
|
||||
#### Example response
|
||||
|
||||
The response for successful operation should look like this:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message": {
|
||||
"id": "58d8299c5195c38e77c2daa5",
|
||||
"mailbox": "58dbf87fcff690a8c30470c7",
|
||||
"headers": [
|
||||
{
|
||||
"key": "delivered-to",
|
||||
"value": "andris@addrgw.com"
|
||||
}
|
||||
],
|
||||
"date": "2017-04-03T10:34:43.007Z",
|
||||
"flags": ["\\Seen"],
|
||||
"text": "Hello world!",
|
||||
"html": ["<p>Hello world!</p>"],
|
||||
"attachments": [
|
||||
{
|
||||
"id": "ATT00001",
|
||||
"fileName": "image.png",
|
||||
"contentType": "image/png",
|
||||
"disposition": "attachment",
|
||||
"transferEncoding": "base64",
|
||||
"related": true,
|
||||
"sizeKb": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### GET /message/:mid/attachment/:aid
|
||||
|
||||
Retrieves an attachment of the message
|
||||
|
||||
Parameters
|
||||
|
||||
- **mid** is the message ID
|
||||
- **aid** is the attachment ID
|
||||
|
||||
**Example**
|
||||
|
||||
```
|
||||
curl "http://localhost:8080/message/58d8299c5195c38e77c2daa5/attachment/ATT00001"
|
||||
```
|
||||
|
||||
### GET /message/:id/raw
|
||||
|
||||
Retrieves RFC822 source of the message
|
||||
|
||||
Parameters
|
||||
|
||||
- **id** is the MongoDB _id as a string for a message
|
||||
- **mailbox** is optional Mailbox id. Use this to verify that the message is located at this mailbox
|
||||
|
||||
**Example**
|
||||
|
||||
```
|
||||
curl "http://localhost:8080/message/58d8299c5195c38e77c2daa5/raw"
|
||||
```
|
||||
|
||||
### DELETE /message/:id
|
||||
|
||||
Deletes a message from a mailbox.
|
||||
|
||||
Parameters
|
||||
|
||||
- **id** is the MongoDB _id as a string for a message
|
||||
- **mailbox** is an optional Mailbox id. Use this to verify that the message to be deleted is located at this mailbox
|
||||
|
||||
**Example**
|
||||
|
||||
```
|
||||
curl -XDELETE "http://localhost:8080/message/58d8299c5195c38e77c2daa5"
|
||||
```
|
||||
|
||||
The response for successful operation should look like this:
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"message":{
|
||||
"id": "58d8299c5195c38e77c2daa5"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Message filtering
|
||||
## Message filtering
|
||||
|
||||
Wild Duck has built-in message filtering in LMTP server. This is somewhat similar to Sieve even though the filters are not scripts.
|
||||
|
||||
|
@ -696,6 +438,14 @@ Filters are configuration objects stored in the `filters` array of the users obj
|
|||
|
||||
**NB!** If you do not care about an action field then do not set it, otherwise matches from other filters do not apply
|
||||
|
||||
## Sharding
|
||||
|
||||
Shard the following collections by these keys:
|
||||
|
||||
* Collection: `messages`, key: `user` (by hash?)
|
||||
* Collection: `attachment.files`, key: `_id` (by hash)
|
||||
* Collection: `attachment.chunks`, key: `file_id` (by hash)
|
||||
|
||||
## IMAP Protocol Differences
|
||||
|
||||
This is a list of known differences from the IMAP specification. Listed differences are either intentional or are bugs that became features.
|
||||
|
|
700
api.js
700
api.js
|
@ -6,18 +6,13 @@ const log = require('npmlog');
|
|||
const Joi = require('joi');
|
||||
const bcrypt = require('bcryptjs');
|
||||
const tools = require('./lib/tools');
|
||||
const MessageHandler = require('./lib/message-handler');
|
||||
const UserHandler = require('./lib/user-handler');
|
||||
const db = require('./lib/db');
|
||||
const ObjectID = require('mongodb').ObjectID;
|
||||
const libqp = require('libqp');
|
||||
const libbase64 = require('libbase64');
|
||||
|
||||
const server = restify.createServer({
|
||||
name: 'Wild Duck API'
|
||||
});
|
||||
|
||||
let messageHandler;
|
||||
let userHandler;
|
||||
|
||||
server.use(restify.plugins.queryParser());
|
||||
|
@ -599,700 +594,6 @@ server.get('/user', (req, res, next) => {
|
|||
});
|
||||
});
|
||||
|
||||
server.get('/user/mailboxes', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
username: Joi.string().alphanum().lowercase().min(3).max(30).required()
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
username: req.query.username
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let username = result.value.username;
|
||||
|
||||
db.database.collection('users').findOne({
|
||||
username
|
||||
}, (err, userData) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
username
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!userData) {
|
||||
res.json({
|
||||
error: 'This user does not exist',
|
||||
username
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
db.database
|
||||
.collection('mailboxes')
|
||||
.find({
|
||||
user: userData._id
|
||||
})
|
||||
.toArray((err, mailboxes) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
username
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
if (!mailboxes) {
|
||||
mailboxes = [];
|
||||
}
|
||||
|
||||
let priority = {
|
||||
Inbox: 1,
|
||||
Sent: 2,
|
||||
Junk: 3,
|
||||
Trash: 4
|
||||
};
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
username,
|
||||
mailboxes: mailboxes
|
||||
.map(mailbox => ({
|
||||
id: mailbox._id.toString(),
|
||||
path: mailbox.path,
|
||||
special: mailbox.path === 'INBOX' ? 'Inbox' : mailbox.specialUse ? mailbox.specialUse.replace(/^\\/, '') : false
|
||||
}))
|
||||
.sort((a, b) => {
|
||||
if (a.special && !b.special) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (b.special && !a.special) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (a.special && b.special) {
|
||||
return (priority[a.special] || 5) - (priority[b.special] || 5);
|
||||
}
|
||||
|
||||
return a.path.localeCompare(b.path);
|
||||
})
|
||||
});
|
||||
return next();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// FIXME: if listing a page after the last one then there is no prev URL
|
||||
// Probably should detect the last page the same way the first one is detected
|
||||
server.get('/mailbox/:id', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
id: Joi.string().hex().lowercase().length(24).required(),
|
||||
before: Joi.number().default(0),
|
||||
after: Joi.number().default(0),
|
||||
size: Joi.number().min(1).max(50).default(20)
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
id: req.params.id,
|
||||
before: req.params.before,
|
||||
after: req.params.after,
|
||||
size: req.params.size
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let id = result.value.id;
|
||||
let before = result.value.before;
|
||||
let after = result.value.after;
|
||||
let size = result.value.size;
|
||||
|
||||
db.database.collection('mailboxes').findOne({
|
||||
_id: new ObjectID(id)
|
||||
}, (err, mailbox) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!mailbox) {
|
||||
res.json({
|
||||
error: 'This mailbox does not exist',
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let query = {
|
||||
mailbox: mailbox._id
|
||||
};
|
||||
let reverse = false;
|
||||
let sort = [['uid', -1]];
|
||||
|
||||
if (req.params.before) {
|
||||
query.uid = {
|
||||
$lt: before
|
||||
};
|
||||
} else if (req.params.after) {
|
||||
query.uid = {
|
||||
$gt: after
|
||||
};
|
||||
sort = [['uid', 1]];
|
||||
reverse = true;
|
||||
}
|
||||
|
||||
db.database.collection('messages').findOne({
|
||||
mailbox: mailbox._id
|
||||
}, {
|
||||
fields: {
|
||||
uid: true
|
||||
},
|
||||
sort: [['uid', -1]]
|
||||
}, (err, entry) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
if (!entry) {
|
||||
res.json({
|
||||
success: true,
|
||||
mailbox: {
|
||||
id: mailbox._id,
|
||||
path: mailbox.path
|
||||
},
|
||||
next: false,
|
||||
prev: false,
|
||||
messages: []
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let newest = entry.uid;
|
||||
|
||||
db.database.collection('messages').findOne({
|
||||
mailbox: mailbox._id
|
||||
}, {
|
||||
fields: {
|
||||
uid: true
|
||||
},
|
||||
sort: [['uid', 1]]
|
||||
}, (err, entry) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
if (!entry) {
|
||||
res.json({
|
||||
error: 'Unexpected result'
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let oldest = entry.uid;
|
||||
|
||||
db.database
|
||||
.collection('messages')
|
||||
.find(query, {
|
||||
uid: true,
|
||||
mailbox: true,
|
||||
idate: true,
|
||||
headers: true,
|
||||
ha: true,
|
||||
intro: true
|
||||
})
|
||||
.sort(sort)
|
||||
.limit(size)
|
||||
.toArray((err, messages) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
if (reverse) {
|
||||
messages = messages.reverse();
|
||||
}
|
||||
|
||||
let nextPage = false;
|
||||
let prevPage = false;
|
||||
|
||||
if (messages.length) {
|
||||
if (after || before) {
|
||||
prevPage = messages[0].uid;
|
||||
if (prevPage >= newest) {
|
||||
prevPage = false;
|
||||
}
|
||||
}
|
||||
if (messages.length >= size) {
|
||||
nextPage = messages[messages.length - 1].uid;
|
||||
if (nextPage < oldest) {
|
||||
nextPage = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
mailbox: {
|
||||
id: mailbox._id,
|
||||
path: mailbox.path
|
||||
},
|
||||
next: nextPage ? '/mailbox/' + id + '?before=' + nextPage + '&size=' + size : false,
|
||||
prev: prevPage ? '/mailbox/' + id + '?after=' + prevPage + '&size=' + size : false,
|
||||
messages: messages.map(message => {
|
||||
let response = {
|
||||
id: message._id,
|
||||
date: message.idate,
|
||||
ha: message.ha,
|
||||
intro: message.intro
|
||||
};
|
||||
|
||||
message.headers.forEach(entry => {
|
||||
if (['subject', 'from', 'to', 'cc', 'bcc'].includes(entry.key)) {
|
||||
response[entry.key] = entry.value;
|
||||
}
|
||||
});
|
||||
return response;
|
||||
})
|
||||
});
|
||||
|
||||
return next();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
server.get('/message/:id', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
id: Joi.string().hex().lowercase().length(24).required(),
|
||||
mailbox: Joi.string().hex().lowercase().length(24).optional()
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
id: req.params.id,
|
||||
mailbox: req.params.mailbox
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let id = result.value.id;
|
||||
let mailbox = result.value.mailbox;
|
||||
|
||||
let query = {
|
||||
_id: new ObjectID(id)
|
||||
};
|
||||
|
||||
if (mailbox) {
|
||||
query.mailbox = new ObjectID(mailbox);
|
||||
}
|
||||
|
||||
db.database.collection('messages').findOne(query, {
|
||||
mailbox: true,
|
||||
headers: true,
|
||||
html: true,
|
||||
text: true,
|
||||
attachments: true,
|
||||
idate: true,
|
||||
flags: true
|
||||
}, (err, message) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!message) {
|
||||
res.json({
|
||||
error: 'This message does not exist',
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: {
|
||||
id,
|
||||
mailbox: message.mailbox,
|
||||
headers: message.headers,
|
||||
date: message.idate,
|
||||
flags: message.flags,
|
||||
text: message.text,
|
||||
html: message.html,
|
||||
attachments: message.attachments
|
||||
}
|
||||
});
|
||||
|
||||
return next();
|
||||
});
|
||||
});
|
||||
|
||||
server.get('/message/:id/raw', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
id: Joi.string().hex().lowercase().length(24).required(),
|
||||
mailbox: Joi.string().hex().lowercase().length(24).optional()
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
id: req.params.id,
|
||||
mailbox: req.params.mailbox
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let id = result.value.id;
|
||||
let mailbox = result.value.mailbox;
|
||||
|
||||
let query = {
|
||||
_id: new ObjectID(id)
|
||||
};
|
||||
|
||||
if (mailbox) {
|
||||
query.mailbox = new ObjectID(mailbox);
|
||||
}
|
||||
|
||||
db.database.collection('messages').findOne(query, {
|
||||
mimeTree: true,
|
||||
map: true,
|
||||
size: true
|
||||
}, (err, message) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!message) {
|
||||
res.json({
|
||||
error: 'This message does not exist',
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let response = messageHandler.indexer.rebuild(message.mimeTree);
|
||||
if (!response || response.type !== 'stream' || !response.value) {
|
||||
res.json({
|
||||
error: 'Can not fetch message',
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'message/rfc822'
|
||||
});
|
||||
response.value.pipe(res);
|
||||
});
|
||||
});
|
||||
|
||||
server.get('/message/:message/attachment/:attachment', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
message: Joi.string().hex().lowercase().length(24).required(),
|
||||
attachment: Joi.string().regex(/^ATT\d+$/i).uppercase().required()
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
message: req.params.message,
|
||||
attachment: req.params.attachment
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let messageId = result.value.message;
|
||||
let attachmentMid = result.value.attachment;
|
||||
|
||||
db.database.collection('messages').findOne({
|
||||
_id: new ObjectID(messageId)
|
||||
}, {
|
||||
fields: {
|
||||
map: true
|
||||
}
|
||||
}, (err, message) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
attachment: attachmentMid,
|
||||
message: messageId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!message) {
|
||||
res.json({
|
||||
error: 'This message does not exist',
|
||||
attachment: attachmentMid,
|
||||
message: messageId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let attachmentId = message.map && message.map[attachmentMid];
|
||||
|
||||
if (!attachmentId) {
|
||||
res.json({
|
||||
error: 'This attachment does not exist',
|
||||
attachment: attachmentMid,
|
||||
message: messageId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
db.database.collection('attachments.files').findOne({
|
||||
_id: new ObjectID(attachmentId)
|
||||
}, (err, messageData) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
attachment: attachmentMid,
|
||||
message: messageId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!messageData) {
|
||||
res.json({
|
||||
error: 'This message does not exist',
|
||||
attachment: attachmentMid,
|
||||
message: messageId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Type': messageData.contentType || 'application/octet-stream'
|
||||
});
|
||||
|
||||
let attachmentStream = messageHandler.indexer.gridstore.openDownloadStream(messageData._id);
|
||||
|
||||
attachmentStream.once('error', err => res.emit('error', err));
|
||||
|
||||
if (messageData.metadata.transferEncoding === 'base64') {
|
||||
attachmentStream.pipe(new libbase64.Decoder()).pipe(res);
|
||||
} else if (messageData.metadata.transferEncoding === 'quoted-printable') {
|
||||
attachmentStream.pipe(new libqp.Decoder()).pipe(res);
|
||||
} else {
|
||||
attachmentStream.pipe(res);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
server.get('/attachment/:attachment', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
attachment: Joi.string().hex().lowercase().length(24).required()
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
attachment: req.params.attachment
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let attachmentId = result.value.attachment;
|
||||
|
||||
db.database.collection('attachments.files').findOne({
|
||||
_id: new ObjectID(attachmentId)
|
||||
}, (err, messageData) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
attachment: attachmentId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!messageData) {
|
||||
res.json({
|
||||
error: 'This message does not exist',
|
||||
attachment: attachmentId
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Type': messageData.contentType || 'application/octet-stream'
|
||||
});
|
||||
|
||||
let attachmentStream = messageHandler.indexer.gridstore.openDownloadStream(messageData._id);
|
||||
|
||||
attachmentStream.once('error', err => res.emit('error', err));
|
||||
|
||||
if (messageData.metadata.transferEncoding === 'base64') {
|
||||
attachmentStream.pipe(new libbase64.Decoder()).pipe(res);
|
||||
} else if (messageData.metadata.transferEncoding === 'quoted-printable') {
|
||||
attachmentStream.pipe(new libqp.Decoder()).pipe(res);
|
||||
} else {
|
||||
attachmentStream.pipe(res);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
server.del('/message/:id', (req, res, next) => {
|
||||
res.charSet('utf-8');
|
||||
|
||||
const schema = Joi.object().keys({
|
||||
id: Joi.string().hex().lowercase().length(24).required(),
|
||||
mailbox: Joi.string().hex().lowercase().length(24).optional()
|
||||
});
|
||||
|
||||
const result = Joi.validate(
|
||||
{
|
||||
id: req.params.id,
|
||||
mailbox: req.params.mailbox
|
||||
},
|
||||
schema,
|
||||
{
|
||||
abortEarly: false,
|
||||
convert: true,
|
||||
allowUnknown: true
|
||||
}
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
res.json({
|
||||
error: result.error.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
let id = result.value.id;
|
||||
let mailbox = result.value.mailbox;
|
||||
|
||||
let query = {
|
||||
_id: new ObjectID(id)
|
||||
};
|
||||
|
||||
if (mailbox) {
|
||||
query.mailbox = new ObjectID(mailbox);
|
||||
}
|
||||
|
||||
messageHandler.del(
|
||||
{
|
||||
query
|
||||
},
|
||||
(err, success) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: 'MongoDB Error: ' + err.message,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
res.json({
|
||||
success,
|
||||
id
|
||||
});
|
||||
return next();
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
module.exports = done => {
|
||||
if (!config.imap.enabled) {
|
||||
return setImmediate(() => done(null, false));
|
||||
|
@ -1300,7 +601,6 @@ module.exports = done => {
|
|||
|
||||
let started = false;
|
||||
|
||||
messageHandler = new MessageHandler(db.database, db.redisConfig);
|
||||
userHandler = new UserHandler(db.database, db.redis);
|
||||
|
||||
server.on('error', err => {
|
||||
|
|
|
@ -124,5 +124,5 @@ function validateInternalDate(internaldate) {
|
|||
if (!internaldate || typeof internaldate !== 'string') {
|
||||
return false;
|
||||
}
|
||||
return /^([ \d]\d)\-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ([\-+])(\d{2})(\d{2})$/i.test(internaldate);
|
||||
return /^([ \d]\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ([-+])(\d{2})(\d{2})$/i.test(internaldate);
|
||||
}
|
||||
|
|
|
@ -78,7 +78,7 @@ module.exports = {
|
|||
return callback(new Error('Invalid sequence set for STORE'));
|
||||
}
|
||||
|
||||
if (!/^[\-+]?FLAGS$/.test(action)) {
|
||||
if (!/^[-+]?FLAGS$/.test(action)) {
|
||||
return callback(new Error('Invalid message data item name for STORE'));
|
||||
}
|
||||
|
||||
|
@ -151,7 +151,8 @@ module.exports = {
|
|||
|
||||
let response = {
|
||||
response: success === true ? 'OK' : 'NO',
|
||||
code: typeof success === 'string'
|
||||
code:
|
||||
typeof success === 'string'
|
||||
? success.toUpperCase()
|
||||
: modified && modified.length ? 'MODIFIED ' + imapTools.packMessageRange(modified) : false,
|
||||
message
|
||||
|
|
|
@ -69,7 +69,7 @@ module.exports = {
|
|||
return callback(new Error('Invalid sequence set for UID STORE'));
|
||||
}
|
||||
|
||||
if (!/^[\-+]?FLAGS$/.test(action)) {
|
||||
if (!/^[-+]?FLAGS$/.test(action)) {
|
||||
return callback(new Error('Invalid message data item name for UID STORE'));
|
||||
}
|
||||
|
||||
|
@ -131,7 +131,8 @@ module.exports = {
|
|||
|
||||
callback(null, {
|
||||
response: success === true ? 'OK' : 'NO',
|
||||
code: typeof success === 'string'
|
||||
code:
|
||||
typeof success === 'string'
|
||||
? success.toUpperCase()
|
||||
: modified && modified.length ? 'MODIFIED ' + imapTools.packMessageRange(modified) : false,
|
||||
message
|
||||
|
|
|
@ -334,7 +334,7 @@ module.exports.filterFolders = function(folders, query) {
|
|||
.replace(/\*\*+/g, '*')
|
||||
.replace(/%%+/g, '%')
|
||||
// escape special characters
|
||||
.replace(/([\\^$+?!.():=\[\]|,\-])/g, '\\$1')
|
||||
.replace(/([\\^$+?!.():=[\]|,-])/g, '\\$1')
|
||||
// setup *
|
||||
.replace(/[*]/g, '.*')
|
||||
// setup %
|
||||
|
|
|
@ -184,7 +184,7 @@ class MIMEParser {
|
|||
|
||||
// Do not touch headers that have strange looking keys, keep these
|
||||
// only in the unparsed array
|
||||
if (/[^a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) {
|
||||
if (/[^a-zA-Z0-9\-*]/.test(key) || key.length >= 100) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -268,7 +268,7 @@ class MIMEParser {
|
|||
|
||||
// Do not touch headers that have strange looking keys, keep these
|
||||
// only in the unparsed array
|
||||
if (/[^a-zA-Z0-9\-\*]/.test(key) || key.length >= 100) {
|
||||
if (/[^a-zA-Z0-9\-*]/.test(key) || key.length >= 100) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
@ -4,7 +4,8 @@
|
|||
module.exports = getTLSOptions;
|
||||
|
||||
const tlsDefaults = {
|
||||
key: '-----BEGIN RSA PRIVATE KEY-----\n' +
|
||||
key:
|
||||
'-----BEGIN RSA PRIVATE KEY-----\n' +
|
||||
'MIIEpAIBAAKCAQEA6Z5Qqhw+oWfhtEiMHE32Ht94mwTBpAfjt3vPpX8M7DMCTwHs\n' +
|
||||
'1xcXvQ4lQ3rwreDTOWdoJeEEy7gMxXqH0jw0WfBx+8IIJU69xstOyT7FRFDvA1yT\n' +
|
||||
'RXY2yt9K5s6SKken/ebMfmZR+03ND4UFsDzkz0FfgcjrkXmrMF5Eh5UXX/+9YHeU\n' +
|
||||
|
@ -31,7 +32,8 @@ const tlsDefaults = {
|
|||
'wXOpdKrvkjZbT4AzcNrlGtRl3l7dEVXTu+dN7/ZieJRu7zaStlAQZkIyP9O3DdQ3\n' +
|
||||
'rIcetQpfrJ1cAqz6Ng0pD0mh77vQ13WG1BBmDFa2A9BuzLoBituf4g==\n' +
|
||||
'-----END RSA PRIVATE KEY-----',
|
||||
cert: '-----BEGIN CERTIFICATE-----\n' +
|
||||
cert:
|
||||
'-----BEGIN CERTIFICATE-----\n' +
|
||||
'MIICpDCCAYwCCQCuVLVKVTXnAjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwls\n' +
|
||||
'b2NhbGhvc3QwHhcNMTUwMjEyMTEzMjU4WhcNMjUwMjA5MTEzMjU4WjAUMRIwEAYD\n' +
|
||||
'VQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDp\n' +
|
||||
|
|
2
imap-core/test/fixtures/mimetree.js
vendored
2
imap-core/test/fixtures/mimetree.js
vendored
|
@ -2,7 +2,7 @@
|
|||
|
||||
module.exports.rfc822 =
|
||||
'' +
|
||||
'Subject: test\ r\ n ' +
|
||||
'Subject: test r n ' +
|
||||
'Content-type: multipart/mixed; boundary=abc\r\n' +
|
||||
'\r\n' +
|
||||
'--abc\r\n' +
|
||||
|
|
|
@ -539,7 +539,7 @@ describe('IMAP Protocol integration tests', function() {
|
|||
|
||||
let cmds = [
|
||||
'T1 LOGIN testuser pass',
|
||||
'T2 APPEND INBOX (\Seen $NotJunk NotJunk) "20-Oct-2015 09:57:08 +0300" {' + message.length + '}',
|
||||
'T2 APPEND INBOX (Seen $NotJunk NotJunk) "20-Oct-2015 09:57:08 +0300" {' + message.length + '}',
|
||||
lchunks,
|
||||
'T3 LOGOUT'
|
||||
];
|
||||
|
@ -781,7 +781,7 @@ describe('IMAP Protocol integration tests', function() {
|
|||
},
|
||||
function(resp) {
|
||||
resp = resp.toString();
|
||||
expect(/^\* ID \("name\"/m.test(resp)).to.be.true;
|
||||
expect(/^\* ID \("name"/m.test(resp)).to.be.true;
|
||||
expect(/^T1 OK/m.test(resp)).to.be.true;
|
||||
done();
|
||||
}
|
||||
|
|
37
imap.js
37
imap.js
|
@ -10,12 +10,14 @@ const imapHandler = IMAPServerModule.imapHandler;
|
|||
const ObjectID = require('mongodb').ObjectID;
|
||||
const Indexer = require('./imap-core/lib/indexer/indexer');
|
||||
const imapTools = require('./imap-core/lib/imap-tools');
|
||||
const setupIndexes = require('./indexes.json');
|
||||
const MessageHandler = require('./lib/message-handler');
|
||||
const UserHandler = require('./lib/user-handler');
|
||||
const db = require('./lib/db');
|
||||
const RedFour = require('redfour');
|
||||
const packageData = require('./package.json');
|
||||
const yaml = require('js-yaml');
|
||||
const fs = require('fs');
|
||||
const setupIndexes = yaml.safeLoad(fs.readFileSync(__dirname + '/indexes.yaml', 'utf8')).indexes;
|
||||
|
||||
// home many modifications to cache before writing
|
||||
const BULK_BATCH_SIZE = 150;
|
||||
|
@ -366,6 +368,7 @@ server.onDelete = function(path, session, callback) {
|
|||
[
|
||||
{
|
||||
$match: {
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
}
|
||||
},
|
||||
|
@ -394,6 +397,7 @@ server.onDelete = function(path, session, callback) {
|
|||
let storageUsed = (res && res[0] && res[0].storageUsed) || 0;
|
||||
|
||||
db.database.collection('messages').deleteMany({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
}, err => {
|
||||
if (err) {
|
||||
|
@ -458,6 +462,7 @@ server.onOpen = function(path, session, callback) {
|
|||
db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
})
|
||||
.project({
|
||||
|
@ -499,6 +504,7 @@ server.onStatus = function(path, session, callback) {
|
|||
db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
})
|
||||
.count((err, total) => {
|
||||
|
@ -508,6 +514,7 @@ server.onStatus = function(path, session, callback) {
|
|||
db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id,
|
||||
seen: false
|
||||
})
|
||||
|
@ -648,11 +655,13 @@ server.onStore = function(path, update, session, callback) {
|
|||
}
|
||||
|
||||
let query = {
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
};
|
||||
|
||||
if (update.unchangedSince) {
|
||||
query = {
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id,
|
||||
modseq: {
|
||||
$lte: update.unchangedSince
|
||||
|
@ -884,7 +893,8 @@ server.onStore = function(path, update, session, callback) {
|
|||
updateEntries.push({
|
||||
updateOne: {
|
||||
filter: {
|
||||
_id: message._id
|
||||
_id: message._id,
|
||||
user: session.user.id
|
||||
},
|
||||
update: flagsupdate
|
||||
}
|
||||
|
@ -952,6 +962,7 @@ server.onExpunge = function(path, update, session, callback) {
|
|||
let cursor = db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id,
|
||||
deleted: true
|
||||
})
|
||||
|
@ -1004,7 +1015,8 @@ server.onExpunge = function(path, update, session, callback) {
|
|||
}
|
||||
|
||||
db.database.collection('messages').deleteOne({
|
||||
_id: message._id
|
||||
_id: message._id,
|
||||
user: session.user.id
|
||||
}, err => {
|
||||
if (err) {
|
||||
return updateQuota(() => cursor.close(() => callback(err)));
|
||||
|
@ -1104,6 +1116,7 @@ server.onCopy = function(path, update, session, callback) {
|
|||
let cursor = db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id,
|
||||
uid: {
|
||||
$in: update.messages
|
||||
|
@ -1331,11 +1344,13 @@ server.onFetch = function(path, options, session, callback) {
|
|||
}
|
||||
|
||||
let query = {
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
};
|
||||
|
||||
if (options.changedSince) {
|
||||
query = {
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id,
|
||||
modseq: {
|
||||
$gt: options.changedSince
|
||||
|
@ -1443,7 +1458,8 @@ server.onFetch = function(path, options, session, callback) {
|
|||
updateEntries.push({
|
||||
updateOne: {
|
||||
filter: {
|
||||
_id: message._id
|
||||
_id: message._id,
|
||||
user: session.user.id
|
||||
},
|
||||
update: {
|
||||
$addToSet: {
|
||||
|
@ -1512,6 +1528,7 @@ server.onSearch = function(path, options, session, callback) {
|
|||
// prepare query
|
||||
|
||||
let query = {
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
};
|
||||
|
||||
|
@ -2030,6 +2047,8 @@ function clearExpiredMessages() {
|
|||
return deleteOrphanedAttachments(() => done(null, true));
|
||||
}
|
||||
|
||||
// TODO: check performance on sharded settings as the query
|
||||
// does not use the shard key
|
||||
let cursor = db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
|
@ -2175,7 +2194,7 @@ module.exports = done => {
|
|||
return next();
|
||||
}
|
||||
let index = setupIndexes[indexpos++];
|
||||
db.database.collection(index.collection).createIndexes(index.indexes, (err, r) => {
|
||||
db.database.collection(index.collection).createIndexes([index.index], (err, r) => {
|
||||
if (err) {
|
||||
server.logger.error(
|
||||
{
|
||||
|
@ -2184,7 +2203,7 @@ module.exports = done => {
|
|||
},
|
||||
'Failed creating index %s %s. %s',
|
||||
indexpos,
|
||||
index.indexes.map(i => JSON.stringify(i.name)).join(', '),
|
||||
JSON.stringify(index.index.name),
|
||||
err.message
|
||||
);
|
||||
} else if (r.numIndexesAfter !== r.numIndexesBefore) {
|
||||
|
@ -2194,7 +2213,7 @@ module.exports = done => {
|
|||
},
|
||||
'Created index %s %s',
|
||||
indexpos,
|
||||
index.indexes.map(i => JSON.stringify(i.name)).join(', ')
|
||||
JSON.stringify(index.index.name)
|
||||
);
|
||||
} else {
|
||||
server.logger.debug(
|
||||
|
@ -2203,7 +2222,7 @@ module.exports = done => {
|
|||
},
|
||||
'Skipped index %s %s: %s',
|
||||
indexpos,
|
||||
index.indexes.map(i => JSON.stringify(i.name)).join(', '),
|
||||
JSON.stringify(index.index.name),
|
||||
r.note || 'No index added'
|
||||
);
|
||||
}
|
||||
|
@ -2212,7 +2231,7 @@ module.exports = done => {
|
|||
});
|
||||
};
|
||||
|
||||
gcLock.acquireLock('db_indexes', 10 * 60 * 1000, (err, lock) => {
|
||||
gcLock.acquireLock('db_indexes', 1 * 60 * 1000, (err, lock) => {
|
||||
if (err) {
|
||||
server.logger.error(
|
||||
{
|
||||
|
|
427
indexes.json
427
indexes.json
|
@ -1,427 +0,0 @@
|
|||
[
|
||||
{
|
||||
"collection": "users",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "users",
|
||||
"key": {
|
||||
"username": 1
|
||||
},
|
||||
"unique": true,
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "users",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "show_new",
|
||||
"key": {
|
||||
"created": -1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "addresses",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "address",
|
||||
"key": {
|
||||
"address": 1
|
||||
},
|
||||
"unique": true,
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "addresses",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "user",
|
||||
"key": {
|
||||
"user": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "mailboxes",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "find_by_user",
|
||||
"key": {
|
||||
"user": 1,
|
||||
"path": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "mailboxes",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "user_subscribed",
|
||||
"key": {
|
||||
"user": 1,
|
||||
"subscribed": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "mailboxes",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "find_by_type",
|
||||
"key": {
|
||||
"user": 1,
|
||||
"specialUse": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_messages",
|
||||
"key": {
|
||||
"mailbox": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "user_messages_by_thread",
|
||||
"key": {
|
||||
"user": 1,
|
||||
"thread": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_uid",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"uid": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_modseq_uid",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"modseq": 1,
|
||||
"uid": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "newer_first",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"uid": -1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_flags",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"flags": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "by_modseq",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"modseq": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "by_idate",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"idate": 1,
|
||||
"_id": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "by_idate_newer",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"idate": -1,
|
||||
"_id": -1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "by_hdate",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"hdate": 1,
|
||||
"msgid": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "by_size",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"size": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "by_headers",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"headers.key": 1,
|
||||
"headers.value": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "fulltext",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"subject": "text",
|
||||
"text": "text"
|
||||
},
|
||||
"weights": {
|
||||
"subject": 10,
|
||||
"text": 5
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_seen_flag",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"seen": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_deleted_flag",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"deleted": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_flagged_flag",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"flagged": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_draft_flag",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"draft": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "has_attachment",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"ha": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "messages",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "retention_time",
|
||||
"partialFilterExpression": {
|
||||
"exp": true
|
||||
},
|
||||
"key": {
|
||||
"exp": 1,
|
||||
"rdate": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "attachments.files",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "attachment_hash",
|
||||
"key": {
|
||||
"metadata.h": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "attachments.files",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "related_attachments",
|
||||
"key": {
|
||||
"metadata.c": 1,
|
||||
"metadata.m": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "journal",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "mailbox_modseq",
|
||||
"key": {
|
||||
"mailbox": 1,
|
||||
"modseq": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "journal",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "autoexpire",
|
||||
"expireAfterSeconds": 21600,
|
||||
"key": {
|
||||
"created": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "threads",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "thread",
|
||||
"key": {
|
||||
"user": 1,
|
||||
"ids": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"collection": "threads",
|
||||
"indexes": [
|
||||
{
|
||||
"name": "autoexpire",
|
||||
"expireAfterSeconds": 31104000,
|
||||
"key": {
|
||||
"updated": 1
|
||||
},
|
||||
"background": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
255
indexes.yaml
Normal file
255
indexes.yaml
Normal file
|
@ -0,0 +1,255 @@
|
|||
---
|
||||
indexes:
|
||||
|
||||
# Indexes for the user collection
|
||||
|
||||
- collection: users
|
||||
index:
|
||||
name: users
|
||||
unique: true
|
||||
key:
|
||||
username: 1
|
||||
- collection: users
|
||||
index:
|
||||
name: show_new
|
||||
key:
|
||||
created: -1
|
||||
|
||||
# Indexes for the addresses collection
|
||||
|
||||
- collection: addresses
|
||||
index:
|
||||
name: address
|
||||
unique: true
|
||||
key:
|
||||
address: 1
|
||||
- collection: addresses
|
||||
index:
|
||||
name: user
|
||||
key:
|
||||
user: 1
|
||||
|
||||
# Indexes for the mailboxes collection
|
||||
|
||||
- collection: mailboxes
|
||||
index:
|
||||
name: find_by_user
|
||||
key:
|
||||
user: 1
|
||||
path: 1
|
||||
- collection: mailboxes
|
||||
index:
|
||||
name: user_subscribed
|
||||
key:
|
||||
user: 1
|
||||
subscribed: 1
|
||||
- collection: mailboxes
|
||||
index:
|
||||
name: find_by_type
|
||||
key:
|
||||
user: 1
|
||||
specialUse: 1
|
||||
|
||||
# Indexes for the messages collection
|
||||
# NB! this is a sharded collection and the shard
|
||||
# key should be 'user' so keep this field as the first one
|
||||
# in indexes
|
||||
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_by_id
|
||||
key:
|
||||
_id: 1
|
||||
user: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_messages
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: user_messages_by_thread
|
||||
key:
|
||||
user: 1
|
||||
thread: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_uid
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
uid: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_modseq_uid
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
modseq: 1
|
||||
uid: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: newer_first
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
uid: -1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_flags
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
flags: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: by_modseq
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
modseq: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: by_idate
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
idate: 1
|
||||
_id: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: by_idate_newer
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
idate: -1
|
||||
_id: -1
|
||||
- collection: messages
|
||||
index:
|
||||
name: by_hdate
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
hdate: 1
|
||||
msgid: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: by_size
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
size: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: by_headers
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
headers.key: 1
|
||||
headers.value: 1
|
||||
- collection: messages
|
||||
index:
|
||||
# there can be only one $text index per collection, so in order to make
|
||||
# account wide searches we do not use mailbox as compound key element here.
|
||||
# IMAP TEXT and BODY searches might be slower though
|
||||
name: fulltext
|
||||
key:
|
||||
user: 1
|
||||
subject: text
|
||||
text: text
|
||||
weights:
|
||||
subject: 10
|
||||
text: 5
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_seen_flag
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
seen: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_deleted_flag
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
deleted: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_flagged_flag
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
flagged: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: mailbox_draft_flag
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
draft: 1
|
||||
- collection: messages
|
||||
index:
|
||||
name: has_attachment
|
||||
key:
|
||||
user: 1
|
||||
mailbox: 1
|
||||
ha: 1
|
||||
- collection: messages
|
||||
index:
|
||||
# This filter finds all messages that are expired and must be deleted.
|
||||
# Not sure about performance though as it is a global query
|
||||
name: retention_time
|
||||
partialFilterExpression:
|
||||
exp: true
|
||||
key:
|
||||
exp: 1
|
||||
rdate: 1
|
||||
|
||||
# Indexes for the attachments collection
|
||||
# attachments.files collection should be sharded by _id (hash)
|
||||
# attachments.chunks collection should be sharded by files_id (hash)
|
||||
|
||||
- collection: attachments.files
|
||||
index:
|
||||
name: attachment_hash
|
||||
key:
|
||||
metadata.h: 1
|
||||
- collection: attachments.files
|
||||
index:
|
||||
name: related_attachments
|
||||
key:
|
||||
metadata.c: 1
|
||||
metadata.m: 1
|
||||
|
||||
# Indexes for the journal collection
|
||||
|
||||
- collection: journal
|
||||
index:
|
||||
name: mailbox_modseq
|
||||
key:
|
||||
mailbox: 1
|
||||
modseq: 1
|
||||
- collection: journal
|
||||
index:
|
||||
name: autoexpire
|
||||
expireAfterSeconds: 21600
|
||||
key:
|
||||
created: 1
|
||||
|
||||
# Indexes for the threads collection
|
||||
|
||||
- collection: threads
|
||||
index:
|
||||
name: thread
|
||||
key:
|
||||
user: 1
|
||||
ids: 1
|
||||
- collection: threads
|
||||
index:
|
||||
name: autoexpire
|
||||
# autoremove thread indexes after 1 year of inactivity
|
||||
expireAfterSeconds: 31104000
|
||||
key:
|
||||
updated: 1
|
|
@ -212,7 +212,8 @@ class ImapNotifier extends EventEmitter {
|
|||
this.database.collection('messages').updateMany({
|
||||
_id: {
|
||||
$in: updated
|
||||
}
|
||||
},
|
||||
user: mailbox.user
|
||||
}, {
|
||||
// only update modseq if the new value is larger than old one
|
||||
$max: {
|
||||
|
|
|
@ -93,6 +93,7 @@ class MessageHandler {
|
|||
}
|
||||
|
||||
this.checkExistingMessage(
|
||||
mailbox.user,
|
||||
mailbox._id,
|
||||
{
|
||||
hdate,
|
||||
|
@ -314,9 +315,10 @@ class MessageHandler {
|
|||
});
|
||||
}
|
||||
|
||||
checkExistingMessage(mailboxId, message, options, callback) {
|
||||
checkExistingMessage(user, mailboxId, message, options, callback) {
|
||||
// if a similar message already exists then update existing one
|
||||
this.database.collection('messages').findOne({
|
||||
user,
|
||||
mailbox: mailboxId,
|
||||
hdate: message.hdate,
|
||||
msgid: message.msgid
|
||||
|
@ -370,7 +372,8 @@ class MessageHandler {
|
|||
let modseq = mailbox.modifyIndex + 1;
|
||||
|
||||
this.database.collection('messages').findOneAndUpdate({
|
||||
_id: existing._id
|
||||
_id: existing._id,
|
||||
user: mailbox.user
|
||||
}, {
|
||||
$set: {
|
||||
uid,
|
||||
|
@ -452,34 +455,7 @@ class MessageHandler {
|
|||
}
|
||||
|
||||
del(options, callback) {
|
||||
let getMessage = next => {
|
||||
if (options.message) {
|
||||
return next(null, options.message);
|
||||
}
|
||||
this.database.collection('messages').findOne(
|
||||
options.query,
|
||||
{
|
||||
fields: {
|
||||
mailbox: true,
|
||||
uid: true,
|
||||
size: true,
|
||||
map: true,
|
||||
magic: true
|
||||
}
|
||||
},
|
||||
next
|
||||
);
|
||||
};
|
||||
|
||||
getMessage((err, message) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (!message) {
|
||||
return callback(new Error('Message does not exist'));
|
||||
}
|
||||
|
||||
let message = options.message;
|
||||
this.getMailbox(
|
||||
{
|
||||
mailbox: options.mailbox || message.mailbox
|
||||
|
@ -490,7 +466,8 @@ class MessageHandler {
|
|||
}
|
||||
|
||||
this.database.collection('messages').deleteOne({
|
||||
_id: message._id
|
||||
_id: message._id,
|
||||
user: mailbox.user
|
||||
}, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
|
@ -559,7 +536,6 @@ class MessageHandler {
|
|||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
move(options, callback) {
|
||||
|
@ -587,6 +563,7 @@ class MessageHandler {
|
|||
let cursor = this.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: mailbox,
|
||||
mailbox: mailbox._id,
|
||||
uid: {
|
||||
$in: options.messages || []
|
||||
|
@ -683,7 +660,8 @@ class MessageHandler {
|
|||
|
||||
// update message, change mailbox from old to new one
|
||||
this.database.collection('messages').findOneAndUpdate({
|
||||
_id: message._id
|
||||
_id: message._id,
|
||||
user: mailbox.user
|
||||
}, updateOptions, err => {
|
||||
if (err) {
|
||||
return cursor.close(() => done(err));
|
||||
|
|
|
@ -27,6 +27,7 @@
|
|||
"html-to-text": "^3.3.0",
|
||||
"iconv-lite": "^0.4.18",
|
||||
"joi": "^10.6.0",
|
||||
"js-yaml": "^3.9.0",
|
||||
"libbase64": "^0.2.0",
|
||||
"libmime": "^3.1.0",
|
||||
"libqp": "^1.1.0",
|
||||
|
|
7
pop3.js
7
pop3.js
|
@ -84,6 +84,7 @@ const serverOptions = {
|
|||
db.database
|
||||
.collection('messages')
|
||||
.find({
|
||||
user: session.user.id,
|
||||
mailbox: mailbox._id
|
||||
})
|
||||
.project({
|
||||
|
@ -121,7 +122,8 @@ const serverOptions = {
|
|||
|
||||
onFetchMessage(id, session, callback) {
|
||||
db.database.collection('messages').findOne({
|
||||
_id: new ObjectID(id)
|
||||
_id: new ObjectID(id),
|
||||
user: session.user.id
|
||||
}, {
|
||||
mimeTree: true,
|
||||
size: true
|
||||
|
@ -262,10 +264,11 @@ function markAsSeen(session, messages, callback) {
|
|||
}
|
||||
|
||||
db.database.collection('messages').updateMany({
|
||||
mailbox: mailboxData._id,
|
||||
_id: {
|
||||
$in: ids
|
||||
},
|
||||
user: session.user.id,
|
||||
mailbox: mailboxData._id,
|
||||
modseq: {
|
||||
$lt: mailboxData.modifyIndex
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue