From 4a194b6502b32404c2d82011d880083071406163 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Wed, 1 Jul 2020 10:37:28 +0300 Subject: [PATCH] updated autiding --- indexes.yaml | 14 +++++++ lib/audit-handler.js | 28 ++++++++++--- lib/filter-handler.js | 23 +++++++++++ lib/message-handler.js | 5 ++- lib/tasks/audit.js | 39 +++++++++++-------- package.json | 4 +- .../00_install_global_functions_variables.sh | 2 +- setup/04_install_import_keys.sh | 9 ++++- 8 files changed, 96 insertions(+), 28 deletions(-) diff --git a/indexes.yaml b/indexes.yaml index 656f3f3..0a39572 100644 --- a/indexes.yaml +++ b/indexes.yaml @@ -561,6 +561,20 @@ indexes: locked: 1 lockedUntil: 1 + - collection: audits + index: + name: user_expire_time + key: + user: 1 + expires: 1 + + - collection: audits + index: + name: expire_time + key: + expires: 1 + deleted: 1 + - collection: audit.files type: gridfs # index applies to gridfs database index: diff --git a/lib/audit-handler.js b/lib/audit-handler.js index 715879e..f435ce3 100644 --- a/lib/audit-handler.js +++ b/lib/audit-handler.js @@ -36,12 +36,18 @@ class AuditHandler { start: options.start, // Date or null end: options.end, // Date or null expires: options.expires, // Date + deleted: false, // Boolean + notes: options.nodes, // String + meta: options.meta || {}, // Object import: { status: 'queued', failed: 0, copied: 0 - } + }, + + audited: 0, + lastAuditedMessage: null }; let r; @@ -96,7 +102,7 @@ class AuditHandler { * @param {Mixed} message Either a Buffer, an Array of Buffers or a Stream * @param {Object} metadata Metadata for the stored message */ - async store(audit, message, metadata) { + async store(audit, message, metadata, skipCounters) { if (!message) { throw new Error('Missing message content'); } @@ -111,7 +117,7 @@ class AuditHandler { metadata.audit = metadata.audit || audit; metadata.date = metadata.date || new Date(); - return new Promise((resolve, reject) => { + let result = await new Promise((resolve, reject) => { let stream = this.gridstore.openUploadStreamWithId(id, null, { contentType: 'message/rfc822', metadata @@ -145,6 +151,12 @@ class AuditHandler { message.pipe(stream); }); + + if (result && !skipCounters) { + await this.database.collection('audits').updateOne({ _id: metadata.audit }, { $inc: { audited: 1 }, $set: { lastAuditedMessage: new Date() } }); + } + + return result; } /** @@ -207,15 +219,19 @@ class AuditHandler { } await cursor.close(); - await this.database.collection('audits').deleteOne({ _id: auditData._id }); - log.info('Audit', 'Deleted expired audit %s (%s messages)', auditData._id, messages); + await this.database.collection('audits').updateOne({ _id: auditData._id }, { $set: { deleted: true, deletedTime: new Date() } }); + log.info('Audit', 'Deleted audit %s (%s messages)', auditData._id, messages); + return { + audit: auditData._id, + messages + }; } async cleanExpired() { let expiredAudits = await this.database .collection('audits') .find({ - expires: { $lt: new Date() } + expires: { $lt: new Date(), deleted: false } }) .toArray(); diff --git a/lib/filter-handler.js b/lib/filter-handler.js index 7c3bfbf..8126cf2 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -482,6 +482,29 @@ class FilterHandler { filterResults.push({ delete: true }); + try { + let audits = await this.database + .collection('audits') + .find({ user: userData._id, expires: { $gt: new Date() } }) + .toArray(); + + let now = new Date(); + for (let auditData of audits) { + if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) { + // audit not active + continue; + } + await this.auditHandler.store(auditData._id, rawchunks, { + date: prepared.idate || new Date(), + msgid: prepared.msgid, + header: prepared.mimeTree && prepared.mimeTree.parsedHeader, + ha: prepared.ha, + info: Object.assign({ notStored: true }, meta || {}) + }); + } + } catch (err) { + log.error('Filter', '%s AUDITFAIL from=%s to=%s error=%s', prepared.id.toString(), '<>', sender, err.message); + } return { response: { userData, diff --git a/lib/message-handler.js b/lib/message-handler.js index e4ceeff..f6a4b4a 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -473,7 +473,10 @@ class MessageHandler { let raw = options.rawchunks || options.raw; let processAudits = async () => { - let audits = await this.database.collection('audits').find({ user: mailboxData.user }).toArray(); + let audits = await this.database + .collection('audits') + .find({ user: mailboxData.user, expires: { $gt: new Date() } }) + .toArray(); let now = new Date(); for (let auditData of audits) { diff --git a/lib/tasks/audit.js b/lib/tasks/audit.js index 7a18f78..a343781 100644 --- a/lib/tasks/audit.js +++ b/lib/tasks/audit.js @@ -30,12 +30,7 @@ let run = async (taskData, options) => { } let mailboxes = new Map( - ( - await db.database - .collection('mailboxes') - .find({ user: taskData.user }) - .toArray() - ).map(mailboxData => [mailboxData._id.toString(), mailboxData]) + (await db.database.collection('mailboxes').find({ user: taskData.user }).toArray()).map(mailboxData => [mailboxData._id.toString(), mailboxData]) ); let processMessage = async messageData => { @@ -44,16 +39,22 @@ let run = async (taskData, options) => { return false; } let mailboxData = messageData.mailbox ? mailboxes.get(messageData.mailbox.toString()) : false; - let auditMessage = await auditHandler.store(taskData.audit, builder.value, { - date: messageData.idate, - msgid: messageData.msgid, - header: messageData.mimeTree && messageData.mimeTree.parsedHeader, - ha: messageData.ha, - mailbox: messageData.mailbox, - mailboxPath: mailboxData ? mailboxData.path : false, - info: messageData.meta, - draft: messageData.draft - }); + let auditMessage = await auditHandler.store( + taskData.audit, + builder.value, + { + date: messageData.idate, + msgid: messageData.msgid, + header: messageData.mimeTree && messageData.mimeTree.parsedHeader, + ha: messageData.ha, + mailbox: messageData.mailbox, + mailboxPath: mailboxData ? mailboxData.path : false, + info: messageData.meta, + draft: messageData.draft, + imported: true + }, + true + ); return auditMessage; }; @@ -98,7 +99,11 @@ let run = async (taskData, options) => { { _id: taskData.audit }, { $inc: { - 'import.copied': 1 + 'import.copied': 1, + audited: 1 + }, + $set: { + lastAuditedMessage: new Date() } } ); diff --git a/package.json b/package.json index 088db4a..5c3029e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wildduck", - "version": "1.26.3", + "version": "1.26.4", "description": "IMAP/POP3 server built with Node.js and MongoDB", "main": "server.js", "scripts": { @@ -44,7 +44,7 @@ "he": "1.2.0", "html-to-text": "5.1.1", "humanname": "0.2.2", - "iconv-lite": "0.6.0", + "iconv-lite": "0.6.1", "ioredfour": "1.0.2-ioredis-03", "ioredis": "4.17.3", "isemail": "3.2.0", diff --git a/setup/00_install_global_functions_variables.sh b/setup/00_install_global_functions_variables.sh index 0a74fa1..cc1c31f 100755 --- a/setup/00_install_global_functions_variables.sh +++ b/setup/00_install_global_functions_variables.sh @@ -261,7 +261,7 @@ SERVICE_NAME=$1 # Ensure required files and permissions echo "d /var/log/${SERVICE_NAME} 0750 syslog adm" > /etc/tmpfiles.d/${SERVICE_NAME}-log.conf -# Redirect MongoDB log output from syslog to mongodb log file +# Redirect MongoDB log output from syslog to service specific log file echo "if ( \$programname startswith \"$SERVICE_NAME\" ) then { action(type=\"omfile\" file=\"/var/log/${SERVICE_NAME}/${SERVICE_NAME}.log\") stop diff --git a/setup/04_install_import_keys.sh b/setup/04_install_import_keys.sh index 05fe845..dd63a29 100755 --- a/setup/04_install_import_keys.sh +++ b/setup/04_install_import_keys.sh @@ -27,8 +27,15 @@ echo "deb https://deb.nodesource.com/$NODEREPO $CODENAME main" > /etc/apt/source echo "deb-src https://deb.nodesource.com/$NODEREPO $CODENAME main" >> /etc/apt/sources.list.d/nodesource.list # mongo keys + +MONGORELEASE=$CODENAME +if [ "$MONGORELEASE" = "focal" ]; then + # Ubuntu 20 is not yet supported (as of 2020-07-01), fallback to 18 + MONGORELEASE="bionic" +fi + wget -qO- https://www.mongodb.org/static/pgp/server-${MONGODB}.asc | apt-key add -echo "deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu $CODENAME/mongodb-org/$MONGODB multiverse" > /etc/apt/sources.list.d/mongodb-org.list +echo "deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu $MONGORELEASE/mongodb-org/$MONGODB multiverse" > /etc/apt/sources.list.d/mongodb-org.list # rspamd wget -O- https://rspamd.com/apt-stable/gpg.key | apt-key add -