use special fields for common flags

This commit is contained in:
Andris Reinman 2017-04-04 14:46:51 +03:00
parent a6309be9ab
commit ad486b9df8
6 changed files with 101 additions and 35 deletions

View file

@ -13,12 +13,13 @@ Wild Duck is a distributed IMAP server built with Node.js, MongoDB and Redis. No
3. Provide Gmail-like features like pushing sent messages automatically to Sent Mail folder or notifying about messages moved to Junk folder so these could be marked as spam
4. Provide parsed mailbox and message data over HTTP. This should make creating webmail interfaces super easy, no need to parse RFC822 messages to get text content or attachments
## Similar alterntives
## Alterntives
Here's a list of Email/IMAP servers that use database for storing email messages
- [DBMail](http://www.dbmail.org/)
- [Archiveopteryx](http://archiveopteryx.org/)
- [ElasticInbox](http://www.elasticinbox.com/)
## Supported features
@ -66,12 +67,11 @@ You can see an example mail entry [here](https://gist.github.com/andris9/520d530
### Is the server scalable?
Not yet exactly. Even though on some parts Wild Duck is already fast, there are still some important improvements that need to be done:
Somewhat yes. Even though on some parts Wild Duck is already fast, there are still some important improvements that need to be done:
1. Optimize SEARCH queries to use MongoDB queries. Currently only simple stuff (flag, internaldate, not flag, modseq) is included in query and more complex comparisons are handled by the application but this means that too much data must be loaded from database (unless it is a very simple query like "SEARCH UNSEEN" that is already optimized)
2. Optimize FETCH queries to load only partial data for BODY subparts
3. Parse incoming message into the mime tree as a stream. Currently the entire message is buffered in memory before being parsed.
4. CPU usage seems a bit too high, there is probably a ton of profiling to do
1. Optimize FETCH queries to load only partial data for BODY subparts
2. Parse incoming message into the mime tree as a stream. Currently the entire message is buffered in memory before being parsed.
3. CPU usage seems a bit too high, there is probably a ton of profiling to do
### How does it work?

4
api.js
View file

@ -733,7 +733,6 @@ server.get('/mailbox/:id', (req, res, next) => {
};
let reverse = false;
let sort = [
['mailbox', 1],
['uid', -1]
];
@ -746,7 +745,6 @@ server.get('/mailbox/:id', (req, res, next) => {
$gt: after
};
sort = [
['mailbox', 1],
['uid', 1]
];
reverse = true;
@ -759,7 +757,6 @@ server.get('/mailbox/:id', (req, res, next) => {
uid: true
},
sort: [
['mailbox', 1],
['uid', -1]
]
}, (err, entry) => {
@ -794,7 +791,6 @@ server.get('/mailbox/:id', (req, res, next) => {
uid: true
},
sort: [
['mailbox', 1],
['uid', 1]
]
}, (err, entry) => {

90
imap.js
View file

@ -319,7 +319,6 @@ server.onOpen = function (path, session, callback) {
}).project({
uid: true
}).sort([
['mailbox', 1],
['uid', 1]
]).toArray((err, messages) => {
if (err) {
@ -477,7 +476,6 @@ server.onStore = function (path, update, session, callback) {
uid: true,
flags: true
}).sort([
['mailbox', 1],
['uid', 1]
]);
@ -538,7 +536,11 @@ server.onStore = function (path, update, session, callback) {
if (updated) {
flagsupdate = {
$set: {
flags: message.flags
flags: message.flags,
seen: message.flags.includes('\\Seen'),
flagged: message.flags.includes('\\Flagged'),
deleted: message.flags.includes('\\Deleted')
}
};
}
@ -565,6 +567,25 @@ server.onStore = function (path, update, session, callback) {
}
}
};
if (newFlags.includes('\\Seen') || newFlags.includes('\\Flagged') || newFlags.includes('\\Deleted')) {
flagsupdate.$set = {};
if (newFlags.includes('\\Seen')) {
flagsupdate.$set = {
seen: true
};
}
if (newFlags.includes('\\Flagged')) {
flagsupdate.$set = {
flagged: true
};
}
if (newFlags.includes('\\Deleted')) {
flagsupdate.$set = {
deleted: true
};
}
}
}
break;
}
@ -592,6 +613,24 @@ server.onStore = function (path, update, session, callback) {
}
}
};
if (oldFlags.includes('\\Seen') || oldFlags.includes('\\Flagged') || oldFlags.includes('\\Deleted')) {
flagsupdate.$set = {};
if (oldFlags.includes('\\Seen')) {
flagsupdate.$set = {
seen: false
};
}
if (oldFlags.includes('\\Flagged')) {
flagsupdate.$set = {
flagged: false
};
}
if (oldFlags.includes('\\Deleted')) {
flagsupdate.$set = {
deleted: false
};
}
}
}
break;
}
@ -663,7 +702,6 @@ server.onExpunge = function (path, update, session, callback) {
uid: true,
size: true
}).sort([
['mailbox', 1],
['uid', 1]
]);
@ -784,7 +822,6 @@ server.onCopy = function (path, update, session, callback) {
$in: update.messages
}
}).sort([
['mailbox', 1],
['uid', 1]
]); // no projection as we need to copy the entire message
@ -929,7 +966,6 @@ server.onMove = function (path, update, session, callback) {
}).project({
uid: 1
}).sort([
['mailbox', 1],
['uid', 1]
]);
@ -1069,7 +1105,6 @@ server.onFetch = function (path, options, session, callback) {
}
let cursor = db.database.collection('messages').find(query).project(projection).sort([
['mailbox', 1],
['uid', 1]
]);
@ -1253,18 +1288,34 @@ server.onSearch = function (path, options, session, callback) {
case 'flag':
{
if (term.exists) {
parent.push({
flags: {
[!ne ? '$eq' : '$ne']: term.value
switch (term.value) {
case '\\Seen':
case '\\Deleted':
case '\\Flagged':
if (term.exists) {
parent.push({
[term.value.toLowerCase().substr(1)]: !ne
});
} else {
parent.push({
[term.value.toLowerCase().substr(1)]: ne
});
}
});
} else {
parent.push({
flags: {
[!ne ? '$ne' : '$eq']: term.value
break;
default:
if (term.exists) {
parent.push({
flags: {
[!ne ? '$eq' : '$ne']: term.value
}
});
} else {
parent.push({
flags: {
[!ne ? '$ne' : '$eq']: term.value
}
});
}
});
}
}
break;
@ -1423,11 +1474,6 @@ server.onSearch = function (path, options, session, callback) {
uid: true,
modseq: true
});
/*.
sort([
['mailbox', 1],
['uid', 1]
]);*/
let highestModseq = 0;
let uidList = [];

View file

@ -121,6 +121,24 @@
"mailbox": 1,
"text": "text"
}
}, {
"name": "mailbox_seen_flag",
"key": {
"mailbox": 1,
"seen": 1
}
}, {
"name": "mailbox_deleted_flag",
"key": {
"mailbox": 1,
"deleted": 1
}
}, {
"name": "mailbox_flagged_flag",
"key": {
"mailbox": 1,
"flagged": 1
}
}]
}, {
"collection": "attachment.files",

View file

@ -280,7 +280,6 @@ class ImapNotifier extends EventEmitter {
$gt: modifyIndex
}
}).sort([
['mailbox', 1],
['modseq', 1]
]).toArray(callback);
});

View file

@ -169,6 +169,8 @@ class MessageHandler {
let internaldate = options.date && new Date(options.date) || new Date();
let headerdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
let flags = [].concat(options.flags || []);
if (!headerdate || headerdate.toString() === 'Invalid Date') {
headerdate = internaldate;
}
@ -184,7 +186,7 @@ class MessageHandler {
internaldate,
headerdate,
flags: [].concat(options.flags || []),
flags,
size,
meta: options.meta || {},
@ -193,7 +195,12 @@ class MessageHandler {
mimeTree,
envelope,
bodystructure,
messageId
messageId,
// use boolean for more common flags
seen: flags.includes('\\Seen'),
flagged: flags.includes('\\Flagged'),
deleted: flags.includes('\\Deleted')
};
if (maildata.attachments && maildata.attachments.length) {