nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2025-11-08 07:20:59 +08:00
Code Issues Projects Releases Packages Wiki Activity

Store audit messages

Browse source
This commit is contained in:
Andris Reinman 2019-10-01 11:22:10 +03:00
parent 83bab195e7
commit 0ec8851660
10 changed files with 626 additions and 504 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/api_data.js
View file

File diff suppressed because one or more lines are too long

2
docs/api_data.json
View file

File diff suppressed because one or more lines are too long

2
docs/api_project.js
View file

@ -1 +1 @@
define({ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs", "title": "WildDuck API", "url": "https://api.wildduck.email", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2019-09-27T17:57:40.204Z", "url": "http://apidocjs.com", "version": "0.17.7" } });
define({ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs", "title": "WildDuck API", "url": "https://api.wildduck.email", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2019-10-01T08:21:54.311Z", "url": "http://apidocjs.com", "version": "0.17.7" } });

2
docs/api_project.json
View file

@ -1 +1 @@
{ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs", "title": "WildDuck API", "url": "https://api.wildduck.email", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2019-09-27T17:57:40.204Z", "url": "http://apidocjs.com", "version": "0.17.7" } }
{ "name": "wildduck", "version": "1.0.0", "description": "WildDuck API docs", "title": "WildDuck API", "url": "https://api.wildduck.email", "sampleUrl": false, "defaultVersion": "0.0.0", "apidoc": "0.3.0", "generator": { "name": "apidoc", "time": "2019-10-01T08:21:54.311Z", "url": "http://apidocjs.com", "version": "0.17.7" } }

9
lib/api/audit.js
View file

@ -20,6 +20,7 @@ module.exports = (db, server, auditHandler) => {
* @apiParam {String} user Users unique ID.
* @apiParam {String} [start] Start time as ISO date
* @apiParam {String} [end] End time as ISO date
* @apiParam {String} expires Expiration date. Audit data is deleted after this date
*
* @apiSuccess {Boolean} success Indicates successful response
* @apiSuccess {String} id ID for the created Audit
@ -64,6 +65,10 @@ module.exports = (db, server, auditHandler) => {
end: Joi.date()
.empty('')
.allow(false),
expires: Joi.date()
.empty('')
.greater('now')
.required(),
sess: Joi.string().max(255),
ip: Joi.string().ip({
version: ['ipv4', 'ipv6'],
@ -91,11 +96,13 @@ module.exports = (db, server, auditHandler) => {
let user = new ObjectID(result.value.user);
let start = result.value.start;
let end = result.value.end;
let expires = result.value.expires;
let audit = await auditHandler.create({
user,
start,
end
end,
expires
});
res.json({

22
lib/audit-handler.js
View file

@ -34,12 +34,25 @@ class AuditHandler {
user: typeof options.user === 'string' ? new ObjectID(options.user) : options.user,
start: options.start, // Date or null
end: options.end, // Date or null
'import.status': 'queued'
expires: options.expires, // Date
import: {
status: 'queued',
failed: 0,
copied: 0
}
};
let r = await this.database.collection('audits').insertOne(auditData);
let r;
try {
r = await this.database.collection('audits').insertOne(auditData);
} catch (err) {
err.code = 'InternalDatabaseError';
throw err;
}
if (!r.insertedId) {
let err = new Error();
let err = new Error('Failed to create audit entry');
err.code = 'InternalDatabaseError';
throw err;
}
@ -139,10 +152,11 @@ class AuditHandler {
for (let chunk of message) {
if (stream.write(chunk) === false) {
await new Promise(resolve => {
stream.once('drain', resolve());
stream.once('drain', resolve);
});
}
}
stream.end();
};
if (Array.isArray(message)) {

963
lib/filter-handler.js
View file

File diff suppressed because it is too large Load diff

49
lib/message-handler.js
View file

@ -6,6 +6,7 @@ const ObjectID = require('mongodb').ObjectID;
const Indexer = require('../imap-core/lib/indexer/indexer');
const ImapNotifier = require('./imap-notifier');
const AttachmentStorage = require('./attachment-storage');
const AuditHandler = require('./audit-handler');
const libmime = require('libmime');
const counters = require('./counters');
const consts = require('./consts');
@ -45,6 +46,14 @@ class MessageHandler {
this.users = options.users || options.database;
this.counters = counters(this.redis);
this.auditHandler = new AuditHandler({
database: this.database,
users: this.users,
gridfs: options.gridfs || this.database,
bucket: 'audit',
loggelf: message => this.loggelf(message)
});
}
getMailbox(options, callback) {
@ -458,12 +467,40 @@ class MessageHandler {
},
() => {
this.notifier.fire(mailboxData.user);
return cleanup(null, true, {
uidValidity,
uid,
id: messageData._id,
mailbox: mailboxData._id,
status: 'new'
let raw = options.rawchunks || options.raw;
let processAudits = async () => {
let audits = await this.database
.collection('audits')
.find({ user: mailboxData.user })
.toArray();
let now = new Date();
for (let auditData of audits) {
if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) {
// audit not active
continue;
}
await this.auditHandler.store(auditData._id, raw, {
date: messageData.idate,
msgid: messageData.msgid,
header: messageData.mimeTree && messageData.mimeTree.parsedHeader,
ha: messageData.ha,
mailbox: mailboxData._id,
mailboxPath: mailboxData.path,
info: messageData.meta
});
}
};
return processAudits().finally(() => {
return cleanup(null, true, {
uidValidity,
uid,
id: messageData._id,
mailbox: mailboxData._id,
status: 'new'
});
});
}
);

73
lib/tasks/audit.js
View file

@ -29,18 +29,28 @@ let run = async (taskData, options) => {
query.idate.$lte = end;
}
let mailboxes = new Map(
(await db.database
.collection('mailboxes')
.find({ user: taskData.user })
.toArray()).map(mailboxData => [mailboxData._id.toString(), mailboxData])
);
let processMessage = async messageData => {
let builder = messageHandler.indexer.rebuild(messageData.mimeTree);
if (!builder || builder.type !== 'stream' || !builder.value) {
return false;
}
let mailboxData = messageData.mailbox ? mailboxes.get(messageData.mailbox.toString()) : false;
let auditMessage = await auditHandler.store(taskData.audit, builder.value, {
date: messageData.idate,
msgid: messageData.msgid,
header: messageData.mimeTree && messageData.mimeTree.parsedHeader,
ha: messageData.ha,
info: messageData.meta
mailbox: messageData.mailbox,
mailboxPath: mailboxData ? mailboxData.path : false,
info: messageData.meta,
draft: messageData.draft
});
return auditMessage;
@ -55,7 +65,13 @@ let run = async (taskData, options) => {
projection: {
_id: true,
user: true,
mimeTree: true
mimeTree: true,
meta: true,
mailbox: true,
idate: true,
msgid: true,
ha: true,
draft: true
}
});
@ -75,6 +91,18 @@ let run = async (taskData, options) => {
auditMessage
);
copied++;
try {
await db.database.collection('audits').updateOne(
{ _id: taskData.audit },
{
$inc: {
'import.copied': 1
}
}
);
} catch (e) {
//ignore
}
} catch (err) {
log.error(
'Tasks',
@ -86,6 +114,18 @@ let run = async (taskData, options) => {
err.message
);
failed++;
try {
await db.database.collection('audits').updateOne(
{ _id: taskData.audit },
{
$inc: {
'import.failed': 1
}
}
);
} catch (e) {
//ignore
}
}
}
await cursor.close();
@ -105,30 +145,45 @@ let run = async (taskData, options) => {
}
};
await db.database.collection('audits').updateOne(
{ _id: taskData.audit },
{
$set: {
'import.status': 'importing'
}
}
);
try {
await processMessages('messages');
} catch (err) {
status = 'import failed';
status = 'failed';
}
try {
await processMessages('archive');
} catch (err) {
status = 'import failed';
status = 'failed';
}
await db.database.collection('audits').updateOne(
{ _id: taskData.audit },
{
$set: {
'import.status': status,
'import.copied': copied,
'import.failed': failed
'import.status': status
}
}
);
log.verbose('Tasks', 'task=audit id=%s user=%s message=%s', taskData._id, taskData.user, `Copied user messages for auditing`);
log.verbose(
'Tasks',
'task=audit id=%s user=%s message=%s copied=%s failed=%s',
taskData._id,
taskData.user,
`Copied user messages for auditing`,
copied,
failed
);
return true;
};

6
package.json
View file

@ -19,7 +19,7 @@
"apidoc": "0.17.7",
"browserbox": "0.9.1",
"chai": "4.2.0",
"eslint": "6.4.0",
"eslint": "6.5.1",
"eslint-config-nodemailer": "1.2.0",
"eslint-config-prettier": "6.3.0",
"grunt": "1.0.4",
@ -29,7 +29,7 @@
"grunt-shell-spawn": "0.4.0",
"grunt-wait": "0.3.0",
"mailparser": "2.7.1",
"mocha": "6.2.0",
"mocha": "6.2.1",
"request": "2.88.0",
"supertest": "4.0.2"
},
@ -47,7 +47,7 @@
"ioredfour": "1.0.2-ioredis-02",
"ioredis": "4.14.1",
"isemail": "3.2.0",
"joi": "14.3.1",
"joi": "^14.3.1",
"js-yaml": "3.13.1",
"key-fingerprint": "1.1.0",
"libbase64": "1.2.1",