updated autiding

indexes.yaml

@@ -561,6 +561,20 @@ indexes:
               locked: 1
               lockedUntil: 1
 
+    - collection: audits
+      index:
+          name: user_expire_time
+          key:
+              user: 1
+              expires: 1
+
+    - collection: audits
+      index:
+          name: expire_time
+          key:
+              expires: 1
+              deleted: 1
+
     - collection: audit.files
       type: gridfs # index applies to gridfs database
       index:

lib/audit-handler.js

@@ -36,12 +36,18 @@ class AuditHandler {
             start: options.start, // Date or null
             end: options.end, // Date or null
             expires: options.expires, // Date
+            deleted: false, // Boolean
+            notes: options.nodes, // String
+            meta: options.meta || {}, // Object
 
             import: {
                 status: 'queued',
                 failed: 0,
                 copied: 0
-            }
+            },
+
+            audited: 0,
+            lastAuditedMessage: null
         };
 
         let r;
@@ -96,7 +102,7 @@ class AuditHandler {
      * @param {Mixed} message Either a Buffer, an Array of Buffers or a Stream
      * @param {Object} metadata Metadata for the stored message
      */
-    async store(audit, message, metadata) {
+    async store(audit, message, metadata, skipCounters) {
         if (!message) {
             throw new Error('Missing message content');
         }
@@ -111,7 +117,7 @@ class AuditHandler {
         metadata.audit = metadata.audit || audit;
         metadata.date = metadata.date || new Date();
 
-        return new Promise((resolve, reject) => {
+        let result = await new Promise((resolve, reject) => {
             let stream = this.gridstore.openUploadStreamWithId(id, null, {
                 contentType: 'message/rfc822',
                 metadata
@@ -145,6 +151,12 @@ class AuditHandler {
 
             message.pipe(stream);
         });
+
+        if (result && !skipCounters) {
+            await this.database.collection('audits').updateOne({ _id: metadata.audit }, { $inc: { audited: 1 }, $set: { lastAuditedMessage: new Date() } });
+        }
+
+        return result;
     }
 
     /**
@@ -207,15 +219,19 @@ class AuditHandler {
         }
         await cursor.close();
 
-        await this.database.collection('audits').deleteOne({ _id: auditData._id });
+        await this.database.collection('audits').updateOne({ _id: auditData._id }, { $set: { deleted: true, deletedTime: new Date() } });
-        log.info('Audit', 'Deleted expired audit %s (%s messages)', auditData._id, messages);
+        log.info('Audit', 'Deleted audit %s (%s messages)', auditData._id, messages);
+        return {
+            audit: auditData._id,
+            messages
+        };
     }
 
     async cleanExpired() {
         let expiredAudits = await this.database
             .collection('audits')
             .find({
-                expires: { $lt: new Date() }
+                expires: { $lt: new Date(), deleted: false }
             })
             .toArray();
 

lib/filter-handler.js

@@ -482,6 +482,29 @@ class FilterHandler {
 
             filterResults.push({ delete: true });
 
+            try {
+                let audits = await this.database
+                    .collection('audits')
+                    .find({ user: userData._id, expires: { $gt: new Date() } })
+                    .toArray();
+
+                let now = new Date();
+                for (let auditData of audits) {
+                    if ((auditData.start && auditData.start > now) || (auditData.end && auditData.end < now)) {
+                        // audit not active
+                        continue;
+                    }
+                    await this.auditHandler.store(auditData._id, rawchunks, {
+                        date: prepared.idate || new Date(),
+                        msgid: prepared.msgid,
+                        header: prepared.mimeTree && prepared.mimeTree.parsedHeader,
+                        ha: prepared.ha,
+                        info: Object.assign({ notStored: true }, meta || {})
+                    });
+                }
+            } catch (err) {
+                log.error('Filter', '%s AUDITFAIL from=%s to=%s error=%s', prepared.id.toString(), '<>', sender, err.message);
+            }
             return {
                 response: {
                     userData,

lib/message-handler.js

@@ -473,7 +473,10 @@ class MessageHandler {
 
                                                     let raw = options.rawchunks || options.raw;
                                                     let processAudits = async () => {
-                                                        let audits = await this.database.collection('audits').find({ user: mailboxData.user }).toArray();
+                                                        let audits = await this.database
+                                                            .collection('audits')
+                                                            .find({ user: mailboxData.user, expires: { $gt: new Date() } })
+                                                            .toArray();
 
                                                         let now = new Date();
                                                         for (let auditData of audits) {

lib/tasks/audit.js

@@ -30,12 +30,7 @@ let run = async (taskData, options) => {
     }
 
     let mailboxes = new Map(
-        (
+        (await db.database.collection('mailboxes').find({ user: taskData.user }).toArray()).map(mailboxData => [mailboxData._id.toString(), mailboxData])
-            await db.database
-                .collection('mailboxes')
-                .find({ user: taskData.user })
-                .toArray()
-        ).map(mailboxData => [mailboxData._id.toString(), mailboxData])
     );
 
     let processMessage = async messageData => {
@@ -44,16 +39,22 @@ let run = async (taskData, options) => {
             return false;
         }
         let mailboxData = messageData.mailbox ? mailboxes.get(messageData.mailbox.toString()) : false;
-        let auditMessage = await auditHandler.store(taskData.audit, builder.value, {
+        let auditMessage = await auditHandler.store(
-            date: messageData.idate,
+            taskData.audit,
-            msgid: messageData.msgid,
+            builder.value,
-            header: messageData.mimeTree && messageData.mimeTree.parsedHeader,
+            {
-            ha: messageData.ha,
+                date: messageData.idate,
-            mailbox: messageData.mailbox,
+                msgid: messageData.msgid,
-            mailboxPath: mailboxData ? mailboxData.path : false,
+                header: messageData.mimeTree && messageData.mimeTree.parsedHeader,
-            info: messageData.meta,
+                ha: messageData.ha,
-            draft: messageData.draft
+                mailbox: messageData.mailbox,
-        });
+                mailboxPath: mailboxData ? mailboxData.path : false,
+                info: messageData.meta,
+                draft: messageData.draft,
+                imported: true
+            },
+            true
+        );
 
         return auditMessage;
     };
@@ -98,7 +99,11 @@ let run = async (taskData, options) => {
                             { _id: taskData.audit },
                             {
                                 $inc: {
-                                    'import.copied': 1
+                                    'import.copied': 1,
+                                    audited: 1
+                                },
+                                $set: {
+                                    lastAuditedMessage: new Date()
                                 }
                             }
                         );

package.json

@@ -1,6 +1,6 @@
 {
     "name": "wildduck",
-    "version": "1.26.3",
+    "version": "1.26.4",
     "description": "IMAP/POP3 server built with Node.js and MongoDB",
     "main": "server.js",
     "scripts": {
@@ -44,7 +44,7 @@
         "he": "1.2.0",
         "html-to-text": "5.1.1",
         "humanname": "0.2.2",
-        "iconv-lite": "0.6.0",
+        "iconv-lite": "0.6.1",
         "ioredfour": "1.0.2-ioredis-03",
         "ioredis": "4.17.3",
         "isemail": "3.2.0",

setup/00_install_global_functions_variables.sh

@@ -261,7 +261,7 @@ SERVICE_NAME=$1
 # Ensure required files and permissions
 echo "d /var/log/${SERVICE_NAME} 0750 syslog adm" > /etc/tmpfiles.d/${SERVICE_NAME}-log.conf
 
-# Redirect MongoDB log output from syslog to mongodb log file
+# Redirect MongoDB log output from syslog to service specific log file
 echo "if ( \$programname startswith \"$SERVICE_NAME\" ) then {
     action(type=\"omfile\" file=\"/var/log/${SERVICE_NAME}/${SERVICE_NAME}.log\")
     stop

setup/04_install_import_keys.sh

@@ -27,8 +27,15 @@ echo "deb https://deb.nodesource.com/$NODEREPO $CODENAME main" > /etc/apt/source
 echo "deb-src https://deb.nodesource.com/$NODEREPO $CODENAME main" >> /etc/apt/sources.list.d/nodesource.list
 
 # mongo keys
+
+MONGORELEASE=$CODENAME
+if [ "$MONGORELEASE" = "focal" ]; then
+  # Ubuntu 20 is not yet supported (as of 2020-07-01), fallback to 18
+  MONGORELEASE="bionic"
+fi
+
 wget -qO- https://www.mongodb.org/static/pgp/server-${MONGODB}.asc | apt-key add
-echo "deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu $CODENAME/mongodb-org/$MONGODB multiverse" > /etc/apt/sources.list.d/mongodb-org.list
+echo "deb [ arch=amd64 ] http://repo.mongodb.org/apt/ubuntu $MONGORELEASE/mongodb-org/$MONGODB multiverse" > /etc/apt/sources.list.d/mongodb-org.list
 
 # rspamd
 wget -O- https://rspamd.com/apt-stable/gpg.key | apt-key add -