nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2024-12-27 02:10:52 +08:00
Code Issues Projects Releases Packages Wiki Activity

do not return blank content type values in bodystructure

Browse source
This commit is contained in:
Andris Reinman 2017-12-04 16:52:20 +02:00
parent ed5e6037a3
commit 4e44084f17
14 changed files with 381 additions and 322 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
config/imap.toml
View file

@ -13,13 +13,13 @@ maxMB=25
# delete messages from \Trash and \Junk after retention days
retention=30
# TODO: Max donwload bandwith per day
maxDownloadMB=3000
# Default max donwload bandwith per day in megabytes
maxDownloadMB=10000
# TODO: Max upload bandwith per day
maxUploadMB=1000
# Default max upload bandwith per day in megabytes
maxUploadMB=10000
# TODO: Max concurrent connections per service per client
# Default max concurrent connections per service per client
maxConnections=15
# if `true` then do not autodelete expired messages

4
config/pop3.toml
View file

@ -15,8 +15,8 @@ disableVersionString=false
# POP3 server never lists all messages but only a limited length list
maxMessages=250
# TODO: Max donwload bandwith per day
maxDownloadMB=3000
# Max donwload bandwith per day in megabytes
maxDownloadMB=10000
# If true, then expect HAProxy PROXY header as the first line of data
useProxy=false

2
docs/api_data.js
View file

File diff suppressed because one or more lines are too long

2
docs/api_data.json
View file

File diff suppressed because one or more lines are too long

2
docs/api_project.js
View file

@ -1 +1 @@
define({ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs. Under construction, see old docs here: https://github.com/nodemailer/wildduck/blob/master/docs/api.md", "title": "WildDuck API", "url": "http://localhost:8080", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2017-12-01T13:21:34.058Z", "url": "http://apidocjs.com", "version": "0.17.6" } });
define({ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs. Under construction, see old docs here: https://github.com/nodemailer/wildduck/blob/master/docs/api.md", "title": "WildDuck API", "url": "http://localhost:8080", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2017-12-04T08:40:27.968Z", "url": "http://apidocjs.com", "version": "0.17.6" } });

2
docs/api_project.json
View file

@ -1 +1 @@
{ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs. Under construction, see old docs here: https://github.com/nodemailer/wildduck/blob/master/docs/api.md", "title": "WildDuck API", "url": "http://localhost:8080", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2017-12-01T13:21:34.058Z", "url": "http://apidocjs.com", "version": "0.17.6" } }
{ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs. Under construction, see old docs here: https://github.com/nodemailer/wildduck/blob/master/docs/api.md", "title": "WildDuck API", "url": "http://localhost:8080", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2017-12-04T08:40:27.968Z", "url": "http://apidocjs.com", "version": "0.17.6" } }

11
imap-core/lib/indexer/body-structure.js
View file

@ -60,6 +60,17 @@ class BodyStructure {
let bodySubtype = (node.parsedHeader['content-type'] && node.parsedHeader['content-type'].subtype) || null;
let contentTransfer = node.parsedHeader['content-transfer-encoding'] || '7bit';
if (!bodyType || !bodySubtype) {
// prevent strange content types like (NIL "/ms-word") that may break some clients
if (bodyType === 'text' || bodySubtype === 'plain') {
bodyType = 'text';
bodySubtype = 'plain';
} else {
bodyType = 'application';
bodySubtype = 'octet-stream';
}
}
return [
// body type
options.upperCaseKeys ? (bodyType && bodyType.toUpperCase()) || null : bodyType,

2
lib/api/users.js
View file

@ -301,7 +301,7 @@ module.exports = (db, server, userHandler) => {
* @apiParam {Number} [quota] Allowed quota of the user in bytes
* @apiParam {Number} [recipients] How many messages per 24 hour can be sent
* @apiParam {Number} [forwards] How many messages per 24 hour can be forwarded
* @apiParam {Boolean} [disabled] If true then disables user account (can not login, can not receive messages)
* @apiParam {Object} [limits] Service specific limits
* @apiParam {String} [sess] Session identifier for the logs
* @apiParam {String} [ip] IP address for the logs
*

112
lib/handlers/on-append.js
View file

@ -15,64 +15,74 @@ module.exports = (server, messageHandler) => (path, flags, date, raw, session, c
path
);
db.users.collection('users').findOne({
_id: session.user.id
}, (err, userData) => {
if (err) {
return callback(err);
}
if (!userData) {
return callback(new Error('User not found'));
}
if (userData.quota && userData.storageUsed > userData.quota) {
return callback(false, 'OVERQUOTA');
}
messageHandler.counters.ttlcounter('iup:' + session.user.id, 0, config.imap.maxUploadMB * 1024 * 1024, false, (err, res) => {
db.users.collection('users').findOne(
{
_id: session.user.id
},
(err, userData) => {
if (err) {
return callback(err);
}
if (!res.success) {
let err = new Error('Upload was rate limited. Try again in ' + res.ttl + ' seconds');
err.response = 'NO';
return callback(err);
if (!userData) {
return callback(new Error('User not found'));
}
messageHandler.counters.ttlcounter('iup:' + session.user.id, raw.length, config.imap.maxUploadMB * 1024 * 1024, false, () => {
messageHandler.encryptMessage(userData.encryptMessages ? userData.pubKey : false, raw, (err, encrypted) => {
if (!err && encrypted) {
raw = encrypted;
if (userData.quota && userData.storageUsed > userData.quota) {
return callback(false, 'OVERQUOTA');
}
db.redis.hget('limits:' + session.user.id, 'imap:upload', (err, value) => {
let limit = (config.imap.maxUploadMB || 10) * 1024 * 1024;
if (!err && value && !isNaN(value)) {
limit = Number(value) || limit;
}
messageHandler.counters.ttlcounter('iup:' + session.user.id, 0, limit, false, (err, res) => {
if (err) {
return callback(err);
}
messageHandler.add(
{
user: session.user.id,
path,
meta: {
source: 'IMAP',
from: '',
to: [session.user.address || session.user.username],
origin: session.remoteAddress,
transtype: 'APPEND',
time: new Date()
},
session,
date,
flags,
raw
},
(err, status, data) => {
if (err) {
if (err.imapResponse) {
return callback(null, err.imapResponse);
}
return callback(err);
if (!res.success) {
let err = new Error('Upload was rate limited. Try again in ' + res.ttl + ' seconds');
err.response = 'NO';
return callback(err);
}
messageHandler.counters.ttlcounter('iup:' + session.user.id, raw.length, limit, false, () => {
messageHandler.encryptMessage(userData.encryptMessages ? userData.pubKey : false, raw, (err, encrypted) => {
if (!err && encrypted) {
raw = encrypted;
}
callback(null, status, data);
}
);
messageHandler.add(
{
user: session.user.id,
path,
meta: {
source: 'IMAP',
from: '',
to: [session.user.address || session.user.username],
origin: session.remoteAddress,
transtype: 'APPEND',
time: new Date()
},
session,
date,
flags,
raw
},
(err, status, data) => {
if (err) {
if (err.imapResponse) {
return callback(null, err.imapResponse);
}
return callback(err);
}
callback(null, status, data);
}
);
});
});
});
});
});
});
}
);
};

24
lib/handlers/on-auth.js
View file

@ -29,15 +29,21 @@ module.exports = (server, userHandler) => (login, session, callback) => {
let checkConnectionLimits = next => {
if (typeof server.notifier.allocateConnection === 'function') {
return server.notifier.allocateConnection(
{
service: 'imap',
session,
user: result.user,
limit: config.imap.maxConnections || 15
},
next
);
return userHandler.redis.hget('limits:' + result.user, 'imap:connections', (err, value) => {
let limit = config.imap.maxConnections || 15;
if (!err && value && !isNaN(value)) {
limit = Number(value) || limit;
}
server.notifier.allocateConnection(
{
service: 'imap',
session,
user: result.user,
limit
},
next
);
});
}
return next(null, true);

429
lib/handlers/on-fetch.js
View file

@ -19,230 +19,249 @@ module.exports = (server, messageHandler) => (path, options, session, callback)
session.id,
path
);
db.database.collection('mailboxes').findOne({
user: session.user.id,
path
}, (err, mailboxData) => {
if (err) {
return callback(err);
}
if (!mailboxData) {
return callback(null, 'NONEXISTENT');
}
messageHandler.counters.ttlcounter('idw:' + session.user.id, 0, config.imap.maxDownloadMB * 1024 * 1024, false, (err, res) => {
db.database.collection('mailboxes').findOne(
{
user: session.user.id,
path
},
(err, mailboxData) => {
if (err) {
return callback(err);
}
if (!res.success) {
let err = new Error('Download was rate limited. Check again in ' + res.ttl + ' seconds');
err.response = 'NO';
return callback(err);
if (!mailboxData) {
return callback(null, 'NONEXISTENT');
}
let projection = {
uid: true,
modseq: true,
idate: true,
flags: true,
envelope: true,
bodystructure: true,
size: true
};
if (!options.metadataOnly) {
projection.mimeTree = true;
}
let query = {
mailbox: mailboxData._id
};
if (options.changedSince) {
query = {
mailbox: mailboxData._id,
modseq: {
$gt: options.changedSince
}
};
}
let queryAll = false;
if (options.messages.length !== session.selected.uidList.length) {
// do not use uid selector for 1:*
query.uid = tools.checkRangeQuery(options.messages);
} else {
// 1:*
queryAll = true;
// uid is part of the sharding key so we need it somehow represented in the query
query.uid = {
$gt: 0,
$lt: mailboxData.uidNext
};
}
let isUpdated = false;
let updateEntries = [];
let notifyEntries = [];
let done = (...args) => {
if (updateEntries.length) {
return db.database.collection('messages').bulkWrite(updateEntries, {
ordered: false,
w: 1
}, () => {
updateEntries = [];
server.notifier.addEntries(session.user.id, path, notifyEntries, () => {
notifyEntries = [];
server.notifier.fire(session.user.id, path);
return callback(...args);
});
});
db.redis.hget('limits:' + session.user.id, 'imap:download', (err, value) => {
let limit = (config.imap.maxDownloadMB || 10) * 1024 * 1024;
if (!err && value && !isNaN(value)) {
limit = Number(value) || limit;
}
if (isUpdated) {
server.notifier.fire(session.user.id, path);
}
return callback(...args);
};
let cursor = db.database
.collection('messages')
.find(query)
.project(projection)
.sort([['uid', 1]]);
let rowCount = 0;
let processNext = () => {
cursor.next((err, message) => {
messageHandler.counters.ttlcounter('idw:' + session.user.id, 0, limit, false, (err, res) => {
if (err) {
return done(err);
return callback(err);
}
if (!message) {
return cursor.close(() => {
done(null, true);
});
if (!res.success) {
let err = new Error('Download was rate limited. Check again in ' + res.ttl + ' seconds');
err.response = 'NO';
return callback(err);
}
if (queryAll && !session.selected.uidList.includes(message.uid)) {
// skip processing messages that we do not know about yet
return processNext();
let projection = {
uid: true,
modseq: true,
idate: true,
flags: true,
envelope: true,
bodystructure: true,
size: true
};
if (!options.metadataOnly) {
projection.mimeTree = true;
}
let markAsSeen = options.markAsSeen && !message.flags.includes('\\Seen');
if (markAsSeen) {
message.flags.unshift('\\Seen');
}
let query = {
mailbox: mailboxData._id
};
let stream = imapHandler.compileStream(
session.formatResponse('FETCH', message.uid, {
query: options.query,
values: session.getQueryResponse(options.query, message, {
logger: server.logger,
fetchOptions: {},
database: db.database,
attachmentStorage: messageHandler.attachmentStorage,
acceptUTF8Enabled: session.isUTF8Enabled()
})
})
);
stream.description = util.format('* FETCH #%s uid=%s size=%sB ', ++rowCount, message.uid, message.size);
stream.once('error', err => {
err.processed = true;
server.logger.error(
{
err,
tnx: 'fetch',
cid: session.id
},
'[%s] FETCHFAIL %s. %s',
session.id,
message._id,
err.message
);
session.socket.end('\n* BYE Internal Server Error\n');
return cursor.close(() => done());
});
let limiter = new LimitedFetch({
key: 'idw:' + session.user.id,
ttlcounter: messageHandler.counters.ttlcounter,
maxBytes: config.imap.maxDownloadMB * 1024 * 1024
});
stream.pipe(limiter);
// send formatted response to socket
session.writeStream.write(limiter, () => {
if (!markAsSeen) {
return processNext();
}
server.logger.debug(
{
tnx: 'flags',
cid: session.id
},
'[%s] UPDATE FLAGS for "%s"',
session.id,
message.uid
);
isUpdated = true;
updateEntries.push({
updateOne: {
filter: {
_id: message._id,
// include sharding key in query
mailbox: mailboxData._id,
uid: message.uid
},
update: {
$addToSet: {
flags: '\\Seen'
},
$set: {
unseen: false
}
}
if (options.changedSince) {
query = {
mailbox: mailboxData._id,
modseq: {
$gt: options.changedSince
}
});
};
}
notifyEntries.push({
command: 'FETCH',
ignore: session.id,
uid: message.uid,
flags: message.flags,
message: message._id,
unseenChange: true
});
let queryAll = false;
if (options.messages.length !== session.selected.uidList.length) {
// do not use uid selector for 1:*
query.uid = tools.checkRangeQuery(options.messages);
} else {
// 1:*
queryAll = true;
// uid is part of the sharding key so we need it somehow represented in the query
query.uid = {
$gt: 0,
$lt: mailboxData.uidNext
};
}
if (updateEntries.length >= consts.BULK_BATCH_SIZE) {
return db.database.collection('messages').bulkWrite(updateEntries, {
ordered: false,
w: 1
}, err => {
updateEntries = [];
if (err) {
return cursor.close(() => done(err));
let isUpdated = false;
let updateEntries = [];
let notifyEntries = [];
let done = (...args) => {
if (updateEntries.length) {
return db.database.collection('messages').bulkWrite(
updateEntries,
{
ordered: false,
w: 1
},
() => {
updateEntries = [];
server.notifier.addEntries(session.user.id, path, notifyEntries, () => {
notifyEntries = [];
server.notifier.fire(session.user.id, path);
return callback(...args);
});
}
);
}
if (isUpdated) {
server.notifier.fire(session.user.id, path);
}
return callback(...args);
};
let cursor = db.database
.collection('messages')
.find(query)
.project(projection)
.sort([['uid', 1]]);
let rowCount = 0;
let processNext = () => {
cursor.next((err, message) => {
if (err) {
return done(err);
}
if (!message) {
return cursor.close(() => {
done(null, true);
});
}
if (queryAll && !session.selected.uidList.includes(message.uid)) {
// skip processing messages that we do not know about yet
return processNext();
}
let markAsSeen = options.markAsSeen && !message.flags.includes('\\Seen');
if (markAsSeen) {
message.flags.unshift('\\Seen');
}
let stream = imapHandler.compileStream(
session.formatResponse('FETCH', message.uid, {
query: options.query,
values: session.getQueryResponse(options.query, message, {
logger: server.logger,
fetchOptions: {},
database: db.database,
attachmentStorage: messageHandler.attachmentStorage,
acceptUTF8Enabled: session.isUTF8Enabled()
})
})
);
stream.description = util.format('* FETCH #%s uid=%s size=%sB ', ++rowCount, message.uid, message.size);
stream.once('error', err => {
err.processed = true;
server.logger.error(
{
err,
tnx: 'fetch',
cid: session.id
},
'[%s] FETCHFAIL %s. %s',
session.id,
message._id,
err.message
);
session.socket.end('\n* BYE Internal Server Error\n');
return cursor.close(() => done());
});
let limiter = new LimitedFetch({
key: 'idw:' + session.user.id,
ttlcounter: messageHandler.counters.ttlcounter,
maxBytes: limit
});
stream.pipe(limiter);
// send formatted response to socket
session.writeStream.write(limiter, () => {
if (!markAsSeen) {
return processNext();
}
server.notifier.addEntries(session.user.id, path, notifyEntries, () => {
notifyEntries = [];
server.notifier.fire(session.user.id, path);
processNext();
});
});
} else {
processNext();
}
});
});
};
server.logger.debug(
{
tnx: 'flags',
cid: session.id
},
'[%s] UPDATE FLAGS for "%s"',
session.id,
message.uid
);
processNext();
});
});
isUpdated = true;
updateEntries.push({
updateOne: {
filter: {
_id: message._id,
// include sharding key in query
mailbox: mailboxData._id,
uid: message.uid
},
update: {
$addToSet: {
flags: '\\Seen'
},
$set: {
unseen: false
}
}
}
});
notifyEntries.push({
command: 'FETCH',
ignore: session.id,
uid: message.uid,
flags: message.flags,
message: message._id,
unseenChange: true
});
if (updateEntries.length >= consts.BULK_BATCH_SIZE) {
return db.database.collection('messages').bulkWrite(
updateEntries,
{
ordered: false,
w: 1
},
err => {
updateEntries = [];
if (err) {
return cursor.close(() => done(err));
}
server.notifier.addEntries(session.user.id, path, notifyEntries, () => {
notifyEntries = [];
server.notifier.fire(session.user.id, path);
processNext();
});
}
);
} else {
processNext();
}
});
});
};
processNext();
});
});
}
);
};

4
lib/user-handler.js
View file

@ -2005,6 +2005,10 @@ class UserHandler {
// This method deletes non expireing records from database
delete(user, meta, callback) {
meta = meta || {};
// clear limits in Redis
this.redis.del('limits:' + user, () => false);
this.database.collection('messages').updateMany(
{ user },
{

9
package.json
View file

@ -8,7 +8,10 @@
"test": "mongo --eval 'db.dropDatabase()' wildduck-test && redis-cli -n 13 flushdb && NODE_ENV=test grunt",
"apidoc": "apidoc -i lib/api/ -o docs/"
},
"keywords": ["imap", "mail server"],
"keywords": [
"imap",
"mail server"
],
"author": "Andris Reinman",
"license": "EUPL-1.1",
"devDependencies": {
@ -53,11 +56,11 @@
"mongodb-extended-json": "^1.10.0",
"nodemailer": "4.4.0",
"npmlog": "4.1.2",
"openpgp": "2.5.14",
"openpgp": "2.6.0",
"qrcode": "1.0.0",
"restify": "6.3.4",
"seq-index": "1.1.0",
"smtp-server": "3.4.0",
"smtp-server": "3.4.1",
"speakeasy": "2.0.0",
"tlds": "1.199.0",
"u2f": "0.1.3",

90
pop3.js
View file

@ -146,50 +146,56 @@ const serverOptions = {
},
onFetchMessage(message, session, callback) {
messageHandler.counters.ttlcounter('pdw:' + session.user.id, 0, config.pop3.maxDownloadMB * 1024 * 1024, false, (err, res) => {
if (err) {
return callback(err);
db.redis.hget('limits:' + session.user.id, 'pop3:download', (err, value) => {
let limit = (config.pop3.maxDownloadMB || 10) * 1024 * 1024;
if (!err && value && !isNaN(value)) {
limit = Number(value) || limit;
}
if (!res.success) {
let err = new Error('Download was rate limited. Check again in ' + res.ttl + ' seconds');
return callback(err);
}
db.database.collection('messages').findOne(
{
_id: new ObjectID(message.id),
// shard key
mailbox: message.mailbox,
uid: message.uid
},
{
mimeTree: true,
size: true
},
(err, message) => {
if (err) {
return callback(err);
}
if (!message) {
return callback(new Error('Message does not exist or is already deleted'));
}
let response = messageHandler.indexer.rebuild(message.mimeTree);
if (!response || response.type !== 'stream' || !response.value) {
return callback(new Error('Can not fetch message'));
}
let limiter = new LimitedFetch({
key: 'pdw:' + session.user.id,
ttlcounter: messageHandler.counters.ttlcounter,
maxBytes: config.pop3.maxDownloadMB * 1024 * 1024
});
response.value.pipe(limiter);
response.value.once('error', err => limiter.emit('error', err));
callback(null, limiter);
messageHandler.counters.ttlcounter('pdw:' + session.user.id, 0, limit, false, (err, res) => {
if (err) {
return callback(err);
}
);
if (!res.success) {
let err = new Error('Download was rate limited. Check again in ' + res.ttl + ' seconds');
return callback(err);
}
db.database.collection('messages').findOne(
{
_id: new ObjectID(message.id),
// shard key
mailbox: message.mailbox,
uid: message.uid
},
{
mimeTree: true,
size: true
},
(err, message) => {
if (err) {
return callback(err);
}
if (!message) {
return callback(new Error('Message does not exist or is already deleted'));
}
let response = messageHandler.indexer.rebuild(message.mimeTree);
if (!response || response.type !== 'stream' || !response.value) {
return callback(new Error('Can not fetch message'));
}
let limiter = new LimitedFetch({
key: 'pdw:' + session.user.id,
ttlcounter: messageHandler.counters.ttlcounter,
maxBytes: limit
});
response.value.pipe(limiter);
response.value.once('error', err => limiter.emit('error', err));
callback(null, limiter);
}
);
});
});
},