Added sendTime option for draft submissions

docs/api/api_data.js

@@ -9266,6 +9266,13 @@ define({ "api": [
             "optional": false,
             "field": "deleteFiles",
             "description": "<p>If true then deletes attachment files listed in metaData.files array</p>"
+          },
+          {
+            "group": "Parameter",
+            "type": "String",
+            "optional": true,
+            "field": "sendTime",
+            "description": "<p>Datestring for delivery if message should be sent some later time</p>"
           }
         ]
       }

docs/api/api_data.json

@@ -9266,6 +9266,13 @@
             "optional": false,
             "field": "deleteFiles",
             "description": "<p>If true then deletes attachment files listed in metaData.files array</p>"
+          },
+          {
+            "group": "Parameter",
+            "type": "String",
+            "optional": true,
+            "field": "sendTime",
+            "description": "<p>Datestring for delivery if message should be sent some later time</p>"
           }
         ]
       }

docs/api/api_project.js

@@ -9,8 +9,8 @@ define({
   "apidoc": "0.3.0",
   "generator": {
     "name": "apidoc",
-    "time": "2020-07-23T08:56:11.392Z",
+    "time": "2020-09-10T07:07:14.809Z",
-    "url": "http://apidocjs.com",
+    "url": "https://apidocjs.com",
-    "version": "0.24.0"
+    "version": "0.25.0"
   }
 });

docs/api/api_project.json

@@ -9,8 +9,8 @@
   "apidoc": "0.3.0",
   "generator": {
     "name": "apidoc",
-    "time": "2020-07-23T08:56:11.392Z",
+    "time": "2020-09-10T07:07:14.809Z",
-    "url": "http://apidocjs.com",
+    "url": "https://apidocjs.com",
-    "version": "0.24.0"
+    "version": "0.25.0"
   }
 }

docs/api/main.js

@@ -616,7 +616,7 @@ function init($, _, locale, Handlebars, apiProject, apiData, Prism, sampleReques
     // as these actions modify the content
     // and would make it jump to the wrong position or not jump at all.
     if (window.location.hash) {
-        var id = window.location.hash;
+        var id = decodeURI(window.location.hash);
         if ($(id).length > 0)
             $('html,body').animate({ scrollTop: parseInt($(id).offset().top) }, 0);
     }

docs/api/utils/send_sample_request_utils.js

@@ -5,6 +5,8 @@ if (typeof define !== 'function') {
 
 define(['lodash'], function (_) {
 
+    var log = console;
+
     function handleNestedFields(object, key, params, paramType) {
         var attributes = key.split('.');
         var field = attributes[0];
@@ -50,14 +52,14 @@ define(['lodash'], function (_) {
                 } else if (val === 'false') {
                     _.set(object, path, false);
                 } else {
-                    console.warn('Failed to parse object value at path [' + path + ']. Value: (' + val + '). Type: (' + type + ')');
+                    log.warn('Failed to parse object value at path [' + path + ']. Value: (' + val + '). Type: (' + type + ')');
                 }
             } else if (type === 'Number') {
                 var parsedInt = parseInt(val, 10);
                 if (!_.isNaN(parsedInt)) {
                     _.set(object, path, parsedInt);
                 } else {
-                    console.warn('Failed to parse object value at path [' + path + ']. Value: (' + val + '). Type: (' + type + ')');
+                    log.warn('Failed to parse object value at path [' + path + ']. Value: (' + val + '). Type: (' + type + ')');
                 }
             }
         }
@@ -82,5 +84,14 @@ define(['lodash'], function (_) {
         return url.replace(/{(.+?)}/g, ':$1');
     }
 
-    return {handleNestedAndParsingFields,convertPathParams,tryParsingWithTypes};
+    function setLogger(logger) {
+      log = logger;
+    }
+
+    return {
+        handleNestedAndParsingFields,
+        convertPathParams,
+        tryParsingWithTypes,
+        setLogger
+    };
 });

docs/api/vendor/prism.css

@@ -1,5 +1,5 @@
-/* PrismJS 1.20.0
+/* PrismJS 1.21.0
-https://prismjs.com/download.html#themes=prism-tomorrow&languages=clike+javascript+bash+c+csharp+cpp+clojure+elixir+erlang+go+http+json+jsonp+json5+lua+perl+python+rust */
+https://prismjs.com/download.html#themes=prism-tomorrow&languages=clike+javascript+bash+c+csharp+cpp+clojure+elixir+erlang+go+http+json+json5+jsonp+lua+perl+python+rust */
 /**
  * prism.js tomorrow night eighties for JavaScript, CoffeeScript, CSS and HTML
  * Based on https://github.com/chriskempson/tomorrow-theme

imap-core/lib/commands/fetch.js

@@ -266,6 +266,7 @@ module.exports = {
             })
         );
 
+        let startTime = Date.now();
         let logdata = {
             short_message: '[FETCH]',
             _mail_action: 'fetch',
@@ -296,6 +297,8 @@ module.exports = {
             },
             this.session,
             (err, success, info) => {
+                logdata._query_time = Date.now() - startTime;
+
                 Object.keys(info || {}).forEach(key => {
                     let vkey = '_' + key.replace(/[A-Z]+/g, c => '_' + c.toLowerCase());
                     if (vkey === '_id') {

lib/api/messages.js

@@ -733,8 +733,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                 return next();
             }
 
-            let mailboxNeeded = false;
-
             // NB! Scattered query, searches over all user mailboxes and all shards
             let filter = {
                 user
@@ -750,6 +748,25 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
 
             if (mailbox) {
                 filter.mailbox = mailbox;
+            } else {
+                // filter out Trash and Junk
+                let mailboxes;
+                try {
+                    mailboxes = await db.database
+                        .collection('mailboxes')
+                        .find({ user, specialUse: { $in: ['\\Junk', '\\Trash'] } })
+                        .project({
+                            _id: true
+                        })
+                        .toArray();
+                } catch (err) {
+                    res.json({
+                        error: 'MongoDB Error: ' + err.message,
+                        code: 'InternalDatabaseError'
+                    });
+                    return next();
+                }
+                filter.mailbox = { $nin: mailboxes.map(m => m._id) };
             }
 
             if (thread) {
@@ -763,6 +780,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
 
             if (filterUnseen) {
                 filter.unseen = true;
+                filter.searchable = true;
             }
 
             if (filterSearchable) {
@@ -774,7 +792,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                     filter.idate = {};
                 }
                 filter.idate.$gte = datestart;
-                mailboxNeeded = true;
             }
 
             if (dateend) {
@@ -782,7 +799,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                     filter.idate = {};
                 }
                 filter.idate.$lte = dateend;
-                mailboxNeeded = true;
             }
 
             if (filterFrom) {
@@ -801,7 +817,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         }
                     }
                 });
-                mailboxNeeded = true;
             }
 
             if (orTerms.from) {
@@ -817,7 +832,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         }
                     }
                 });
-                mailboxNeeded = true;
             }
 
             if (filterTo) {
@@ -851,7 +865,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         }
                     ]
                 });
-                mailboxNeeded = true;
             }
 
             if (orTerms.to) {
@@ -880,8 +893,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         }
                     }
                 });
-
-                mailboxNeeded = true;
             }
 
             if (filterSubject) {
@@ -900,7 +911,6 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         }
                     }
                 });
-                mailboxNeeded = true;
             }
 
             if (orTerms.subject) {
@@ -916,39 +926,16 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         }
                     }
                 });
-                mailboxNeeded = true;
             }
 
             if (filterAttachments) {
                 filter.ha = true;
-                mailboxNeeded = true;
             }
 
             if (orQuery.length) {
                 filter.$or = orQuery;
             }
 
-            if (!mailbox && mailboxNeeded) {
-                // generate a list of mailbox ID values
-                let mailboxes;
-                try {
-                    mailboxes = await db.database
-                        .collection('mailboxes')
-                        .find({ user, specialUse: { $nin: ['\\Junk', '\\Trash'] } })
-                        .project({
-                            _id: true
-                        })
-                        .toArray();
-                } catch (err) {
-                    res.json({
-                        error: 'MongoDB Error: ' + err.message,
-                        code: 'InternalDatabaseError'
-                    });
-                    return next();
-                }
-                filter.mailbox = { $in: mailboxes.map(m => m._id) };
-            }
-
             let total = await getFilteredMessageCount(filter);
             log.verbose('API', 'Searching %s', JSON.stringify(filter));
 
@@ -2801,6 +2788,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
      * @apiParam {String} mailbox ID of the Mailbox
      * @apiParam {Number} message Message ID
      * @apiParam {Boolean} deleteFiles If true then deletes attachment files listed in metaData.files array
+     * @apiParam {String} [sendTime] Datestring for delivery if message should be sent some later time
      *
      * @apiSuccess {Boolean} success Indicates successful response
      * @apiSuccess {String} queueId Message ID in outbound queue
@@ -2844,6 +2832,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                 mailbox: Joi.string().hex().lowercase().length(24).required(),
                 message: Joi.number().required(),
                 deleteFiles: booleanSchema,
+                sendTime: Joi.date(),
                 sess: sessSchema,
                 ip: sessIPSchema
             });
@@ -2874,6 +2863,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
             let mailbox = new ObjectID(result.value.mailbox);
             let message = result.value.message;
             let deleteFiles = result.value.deleteFiles;
+            let sendTime = result.value.sendTime;
 
             let userData;
             try {
@@ -2935,6 +2925,42 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                 return next();
             }
 
+            let now = new Date();
+            if (!sendTime || sendTime < now) {
+                sendTime = now;
+            }
+
+            // update message headers, use updated Date value
+            if (messageData.mimeTree.header) {
+                let headerFound = false;
+                for (let i = 0; i < messageData.mimeTree.header.length; i++) {
+                    if (/^date\s*:/i.test(messageData.mimeTree.header[i])) {
+                        headerFound = true;
+                        messageData.mimeTree.header[i] = `Date: ${sendTime.toUTCString().replace(/GMT/, '+0000')}`;
+                    }
+                }
+                if (!headerFound) {
+                    messageData.mimeTree.header.push(`Date: ${sendTime.toUTCString().replace(/GMT/, '+0000')}`);
+                }
+
+                messageData.mimeTree.parsedHeader.date = sendTime;
+
+                // update Draft message entry. This is later moved to Sent Mail folder so the Date values
+                // must be correct ones
+                await db.database.collection('messages').updateOne(
+                    {
+                        _id: messageData._id
+                    },
+                    {
+                        $set: {
+                            'mimeTree.header': messageData.mimeTree.header,
+                            'mimeTree.parsedHeader.date': sendTime,
+                            hdate: sendTime
+                        }
+                    }
+                );
+            }
+
             let envelope = messageData.meta.envelope;
             if (!envelope) {
                 // fetch envelope data from message headers
@@ -2970,7 +2996,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                 return next();
             }
 
-            let queueId = await submitMessage(userData, envelope, rebuilder.value);
+            let queueId = await submitMessage(userData, envelope, sendTime, rebuilder.value);
             let response = {
                 success: true
             };
@@ -3828,7 +3854,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
         return address;
     }
 
-    function submitMessage(userData, envelope, stream) {
+    function submitMessage(userData, envelope, sendTime, stream) {
         return new Promise((resolve, reject) => {
             messageHandler.counters.ttlcounter('wdr:' + userData._id.toString(), envelope.to.length, userData.recipients, false, (err, result) => {
                 if (err) {
@@ -3864,9 +3890,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                         reason: 'submit',
                         from: envelope.from,
                         to: envelope.to,
-
+                        sendTime
-                        // make sure we send out a message with current timestamp
-                        updateDate: true
                     },
                     (err, ...args) => {
                         if (err || !args[0]) {

lib/handlers/on-fetch.js

@@ -18,6 +18,13 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
         session.id,
         mailbox
     );
+    const socket = (session.socket && session.socket._parent) || session.socket;
+
+    try {
+        tools.checkSocket(socket);
+    } catch (err) {
+        return callback(err);
+    }
 
     db.database.collection('mailboxes').findOne(
         {
@@ -164,6 +171,25 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
                                 );
                                 return done(err);
                             }
+
+                            try {
+                                // stop processing if IMAP socket is not open anymore
+                                tools.checkSocket(socket);
+                            } catch (err) {
+                                server.logger.error(
+                                    {
+                                        tnx: 'fetch',
+                                        cid: session.id,
+                                        err
+                                    },
+                                    '[%s] FETCHERR error=%s query=%s',
+                                    session.id,
+                                    err.message,
+                                    JSON.stringify(query)
+                                );
+                                return done(err);
+                            }
+
                             if (!messageData) {
                                 return cursor.close(() => {
                                     server.logger.debug(

lib/handlers/on-search.js

@@ -181,10 +181,7 @@ module.exports = server => (mailbox, options, session, callback) => {
                                                       : {
                                                             // not can not have a regex, so try exact match instead even if it fails
                                                             $not: {
-                                                                $eq: Buffer.from(term.value, 'binary')
+                                                                $eq: Buffer.from(term.value, 'binary').toString().toLowerCase().trim()
-                                                                    .toString()
-                                                                    .toLowerCase()
-                                                                    .trim()
                                                             }
                                                         }
                                               }

package.json

@@ -15,22 +15,22 @@
     "author": "Andris Reinman",
     "license": "EUPL-1.1+",
     "devDependencies": {
-        "ajv": "6.12.3",
+        "ajv": "6.12.4",
-        "apidoc": "0.24.0",
+        "apidoc": "0.25.0",
         "chai": "4.2.0",
         "docsify-cli": "4.4.1",
-        "eslint": "7.5.0",
+        "eslint": "7.8.1",
         "eslint-config-nodemailer": "1.2.0",
         "eslint-config-prettier": "6.11.0",
-        "grunt": "1.2.1",
+        "grunt": "1.3.0",
         "grunt-cli": "1.3.2",
         "grunt-eslint": "23.0.0",
         "grunt-mocha-test": "0.13.3",
         "grunt-shell-spawn": "0.4.0",
         "grunt-wait": "0.3.0",
-        "imapflow": "1.0.47",
+        "imapflow": "1.0.50",
-        "mailparser": "2.7.7",
+        "mailparser": "3.0.0",
-        "mocha": "8.0.1",
+        "mocha": "8.1.3",
         "request": "2.88.2",
         "supertest": "4.0.2"
     },
@@ -48,21 +48,21 @@
         "ioredfour": "1.0.2-ioredis-03",
         "ioredis": "4.17.3",
         "isemail": "3.2.0",
-        "joi": "17.1.1",
+        "joi": "17.2.1",
         "js-yaml": "3.14.0",
         "key-fingerprint": "1.1.0",
         "libbase64": "1.2.1",
-        "libmime": "4.2.1",
+        "libmime": "5.0.0",
         "libqp": "1.1.0",
         "mailsplit": "5.0.0",
         "mobileconfig": "2.3.1",
-        "mongo-cursor-pagination": "7.3.0",
+        "mongo-cursor-pagination": "7.3.1",
-        "mongodb": "3.5.9",
+        "mongodb": "3.6.1",
         "mongodb-extended-json": "1.11.0",
-        "node-forge": "0.9.1",
+        "node-forge": "0.10.0",
-        "nodemailer": "6.4.10",
+        "nodemailer": "6.4.11",
         "npmlog": "4.1.2",
-        "openpgp": "4.10.7",
+        "openpgp": "4.10.8",
         "pem": "1.14.4",
         "pwnedpasswords": "1.0.5",
         "qrcode": "1.4.4",
@@ -74,9 +74,9 @@
         "speakeasy": "2.0.0",
         "u2f": "0.1.3",
         "unixcrypt": "1.0.11",
-        "uuid": "8.2.0",
+        "uuid": "8.3.0",
         "wild-config": "1.5.1",
-        "yargs": "15.4.1"
+        "yargs": "16.0.0"
     },
     "repository": {
         "type": "git",