updated autoreplies

docs/api.md

@@ -1771,26 +1771,33 @@ Wild Duck supports setting up autoreply messages that are sent to senders by LMT
 
 #### PUT /users/{user}/autoreply
 
-This call prepares the user to support 2FA tokens. If 2FA is already enabled then this call fails.
+This call sets up or updates autoreply message for the user.
 
 **Parameters**
 
 - **user** (required) is the ID of the user
 - **status** is a boolean that indicates if autoreply messages should be sent (true) or not (false)
 - **subject** is the subject line of autoreply message
-- **message** is text body of the autoreply message
+- **text** is text body of the autoreply message
+- **html** is html body of the autoreply message
+- **start** is the start time of the autoreply
+- **end** is the end time of the autoreply
 
 **Response fields**
 
 - **success** should be `true`
 
+Autoreply update calls can be done partially, eg. only updating status or subject.
+
 **Example**
 
 ```
 curl -XPUT "http://localhost:8080/users/5971da1754cfdc7f0983b2ec/autoreply" -H 'content-type: application/json' -d '{
   "status": true,
   "subject": "Out of office",
-  "message": "I'm out of office this week"
+  "text": "I'm out of office this week",
+  "start": "2017-11-15T00:00:00.000Z",
+  "end": "2017-11-19T00:00:00.000Z",
 }'
 ```
 

indexes.yaml

@@ -40,19 +40,14 @@ indexes:
     unique: true
     key:
       unameview: 1
+
 - collection: users
   type: users # index applies to users database
   index:
     name: show_new
     key:
       created: -1
-- collection: users
+
-  type: users # index applies to users database
-  index:
-    name: users_namespace
-    key:
-      ns: 1
-      unameview: 1
 - collection: users
   type: users # index applies to users database
   index:
@@ -62,12 +57,14 @@ indexes:
     sparse: true
 
 # Indexes for the addresses collection
+# note: is it really needed? maybe sort and list using addrview?
 - collection: addresses
   type: users # index applies to users database
   index:
     name: address
     key:
       address: 1
+
 - collection: addresses
   type: users # index applies to users database
   index:
@@ -76,6 +73,7 @@ indexes:
     key:
       addrview: 1
       _id: 1
+
 - collection: addresses
   type: users # index applies to users database
   index:
@@ -93,7 +91,6 @@ indexes:
       user: 1
 
 # Indexes for the authentication log collection
-
 - collection: authlog
   type: users # index applies to users database
   index:
@@ -130,12 +127,14 @@ indexes:
 
 - collection: autoreplies
   index:
-    name: user
+    name: autoreply
     key:
       user: 1
+      start: 1
+      end: 1
 
 # Indexes for the mailboxes collection
-
+# note: should mailboxes collection be sharded? could be by user
 - collection: mailboxes
   index:
     name: user_path
@@ -143,12 +142,14 @@ indexes:
     key:
       user: 1
       path: 1
+
 - collection: mailboxes
   index:
     name: user_subscribed
     key:
       user: 1
       subscribed: 1
+
 - collection: mailboxes
   index:
     name: find_by_type
@@ -158,29 +159,6 @@ indexes:
 
 # Indexes for the messages collection
 
-- collection: messages
-  index:
-    name: user_messages_by_thread
-    key:
-      user: 1
-      thread: 1
-- collection: messages
-  index:
-    # use also as sharding key
-    name: mailbox_uid
-    key:
-      mailbox: 1
-      uid: 1
-      _id: 1
-- collection: messages
-  index:
-    # use also as sharding key
-    name: mailbox_uid_reverse
-    key:
-      mailbox: 1
-      uid: -1
-      _id: -1
-
 - collection: messages
   index:
     # several message related queries include the shard key values
@@ -192,10 +170,28 @@ indexes:
 
 - collection: messages
   index:
-    name: newer_first
+    name: user_messages_by_thread
+    key:
+      user: 1
+      thread: 1
+
+- collection: messages
+  index:
+    # use also as sharding key
+    name: mailbox_uid
+    key:
+      mailbox: 1
+      uid: 1
+      _id: 1
+
+- collection: messages
+  index:
+    name: mailbox_uid_reverse
     key:
       mailbox: 1
       uid: -1
+      _id: -1
+
 - collection: messages
   index:
     name: mailbox_modseq_uid
@@ -203,24 +199,21 @@ indexes:
       mailbox: 1
       modseq: 1
       uid: 1
+
 - collection: messages
   index:
     name: mailbox_flags
     key:
       mailbox: 1
       flags: 1
-- collection: messages
+
-  index:
-    name: by_modseq
-    key:
-      mailbox: 1
-      modseq: 1
 - collection: messages
   index:
     name: by_idate
     key:
       mailbox: 1
       idate: 1
+
 - collection: messages
   index:
     name: by_hdate
@@ -228,12 +221,14 @@ indexes:
       mailbox: 1
       hdate: 1
       msgid: 1
+
 - collection: messages
   index:
     name: by_size
     key:
       mailbox: 1
       size: 1
+
 - collection: messages
   index:
     name: by_headers
@@ -241,6 +236,7 @@ indexes:
       mailbox: 1
       headers.key: 1
       headers.value: 1
+
 - collection: messages
   index:
     # there can be only one $text index per collection
@@ -252,6 +248,7 @@ indexes:
       text: text
     partialFilterExpression:
       searchable: true
+
 - collection: messages
   index:
     # in most cases we only care about unseen, not seen messages
@@ -259,6 +256,7 @@ indexes:
     key:
       mailbox: 1
       unseen: 1
+
 - collection: messages
   index:
     # some mail agents list messages that do not have the \Deleted flag set
@@ -266,24 +264,28 @@ indexes:
     key:
       mailbox: 1
       undeleted: 1
+
 - collection: messages
   index:
     name: mailbox_flagged_flag
     key:
       mailbox: 1
       flagged: 1
+
 - collection: messages
   index:
     name: mailbox_draft_flag
     key:
       mailbox: 1
       draft: 1
+
 - collection: messages
   index:
     name: has_attachment
     key:
       mailbox: 1
       ha: 1
+
 - collection: messages
   index:
     # This filter finds all messages that are expired and must be deleted.
@@ -305,6 +307,7 @@ indexes:
     name: attachment_id_hashed
     key:
       _id: hashed
+
 - collection: attachments.files
   type: gridfs # index applies to gridfs database
   index:
@@ -312,6 +315,7 @@ indexes:
     key:
       metadata.c: 1
       metadata.m: 1
+
 - collection: attachments.chunks
   type: gridfs # index applies to gridfs database
   index:
@@ -319,6 +323,7 @@ indexes:
     name: chunks_shard
     key:
       files_id: hashed
+
 - collection: attachments.chunks
   type: gridfs # index applies to gridfs database
   index:
@@ -337,6 +342,7 @@ indexes:
     key:
       mailbox: 1
       modseq: 1
+
 - collection: journal
   index:
     # this index is used to send updates to a logged in webmail user
@@ -344,6 +350,7 @@ indexes:
     key:
       user: 1
       _id: 1
+
 - collection: journal
   index:
     # this index is used to find the latest journal entry
@@ -351,6 +358,7 @@ indexes:
     key:
       user: 1
       _id: -1
+
 - collection: journal
   # delete journal entries after 3 hours
   index:
@@ -366,12 +374,14 @@ indexes:
     name: thread_shard
     key:
       user: hashed
+
 - collection: threads
   index:
     name: thread
     key:
       user: 1
       ids: 1
+
 - collection: threads
   index:
     name: thread_autoexpire
@@ -392,7 +402,7 @@ indexes:
     name: messagelog_parent
     key:
       parentId: 1
-    sparse: true
+    sparse: true # only messages inserted by mail store have parentId set
 
 - collection: messagelog
   index:

lib/api/autoreply.js

@@ -8,10 +8,28 @@ module.exports = (db, server) => {
         res.charSet('utf-8');
 
         const schema = Joi.object().keys({
-            user: Joi.string().hex().lowercase().length(24).required(),
+            user: Joi.string()
-            status: Joi.boolean().truthy(['Y', 'true', 'yes', 1]).default(false),
+                .hex()
-            subject: Joi.string().empty('').trim().max(128),
+                .lowercase()
-            message: Joi.string().empty('').trim().max(10 * 1024)
+                .length(24)
+                .required(),
+            status: Joi.boolean()
+                .truthy(['Y', 'true', 'yes', 1])
+                .default(false),
+            subject: Joi.string()
+                .empty('')
+                .trim()
+                .max(128),
+            text: Joi.string()
+                .empty('')
+                .trim()
+                .max(128 * 1024),
+            html: Joi.string()
+                .empty('')
+                .trim()
+                .max(128 * 1024),
+            start: Joi.date(),
+            end: Joi.date().min(Joi.ref('start'))
         });
 
         const result = Joi.validate(req.params, schema, {
@@ -30,8 +48,20 @@ module.exports = (db, server) => {
             result.value.subject = '';
         }
 
-        if (!result.value.message && 'message' in req.params) {
+        if (!result.value.text && 'text' in req.params) {
-            result.value.message = '';
+            result.value.text = '';
+            if (!result.value.html) {
+                // make sure we also update html part
+                result.value.html = '';
+            }
+        }
+
+        if (!result.value.html && 'html' in req.params) {
+            result.value.html = '';
+            if (!result.value.text) {
+                // make sure we also update plaintext part
+                result.value.text = '';
+            }
         }
 
         let user = (result.value.user = new ObjectID(result.value.user));
@@ -71,7 +101,11 @@ module.exports = (db, server) => {
         res.charSet('utf-8');
 
         const schema = Joi.object().keys({
-            user: Joi.string().hex().lowercase().length(24).required()
+            user: Joi.string()
+                .hex()
+                .lowercase()
+                .length(24)
+                .required()
         });
 
         const result = Joi.validate(req.params, schema, {
@@ -101,7 +135,10 @@ module.exports = (db, server) => {
                 success: true,
                 status: !!entry.status,
                 subject: entry.subject || '',
-                message: entry.message || ''
+                text: entry.text || '',
+                html: entry.html || '',
+                start: entry.start,
+                end: entry.end
             });
 
             return next();
@@ -112,7 +149,11 @@ module.exports = (db, server) => {
         res.charSet('utf-8');
 
         const schema = Joi.object().keys({
-            user: Joi.string().hex().lowercase().length(24).required()
+            user: Joi.string()
+                .hex()
+                .lowercase()
+                .length(24)
+                .required()
         });
 
         const result = Joi.validate(req.params, schema, {

lib/autoreply.js

@@ -12,7 +12,16 @@ module.exports = (options, callback) => {
         return callback(null, false);
     }
 
-    db.database.collection('autoreplies').findOne({ user: options.userData._id }, (err, autoreply) => {
+    let curtime = new Date();
+    db.database.collection('autoreplies').findOne({
+        user: options.userData._id,
+        start: {
+            $gte: curtime
+        },
+        end: {
+            $lte: curtime
+        }
+    }, (err, autoreply) => {
         if (err) {
             return callback(err);
         }
@@ -53,9 +62,11 @@ module.exports = (options, callback) => {
             db.redis
                 .multi()
                 // delete all old entries
-                .zremrangebyscore('war:' + options.userData._id, '-inf', Date.now() - consts.MAX_AUTOREPLY_INTERVAL)
+                .zremrangebyscore('war:' + autoreply._id, '-inf', Date.now() - consts.MAX_AUTOREPLY_INTERVAL)
-                // add enw entry if not present
+                // add new entry if not present
-                .zadd('war:' + options.userData._id, 'NX', Date.now(), options.sender)
+                .zadd('war:' + autoreply._id, 'NX', Date.now(), options.sender)
+                // if no-one touches this key from now, then delete after max interval has passed
+                .expire('war:' + autoreply._id, consts.MAX_AUTOREPLY_INTERVAL)
                 .exec((err, result) => {
                     if (err) {
                         errors.notify(err, { userId: options.userData._id });
@@ -90,11 +101,13 @@ module.exports = (options, callback) => {
                                     value: 'Auto: Re: ' + headers.getFirst('Subject')
                                 },
                             headers: {
-                                'Auto-Submitted': 'auto-replied'
+                                'Auto-Submitted': 'auto-replied',
+                                'X-WD-Autoreply-For': (options.parentId || '').toString()
                             },
                             inReplyTo: headers.getFirst('Message-ID'),
                             references: (headers.getFirst('Message-ID') + ' ' + headers.getFirst('References')).trim(),
-                            text: autoreply.message
+                            text: autoreply.text,
+                            html: autoreply.html
                         };
 
                         let compiler = new MailComposer(data);

lib/consts.js

@@ -1,14 +1,17 @@
 'use strict';
 
 module.exports = {
-    // home many modifications to cache before writing
+    SCHEMA_VERSION: '1.0',
+
+    // how many modifications to cache before writing
     BULK_BATCH_SIZE: 150,
 
     // how often to clear expired messages
     GC_INTERVAL: 10 * 60 * 1000,
 
     // artificail delay between deleting next expired message in ms
-    GC_DELAY_DELETE: 100,
+    // set to 0 to disable
+    GC_DELAY_DELETE: 80,
 
     MAX_STORAGE: 1 * (1024 * 1024 * 1024),
     MAX_RECIPIENTS: 2000,
@@ -18,15 +21,13 @@ module.exports = {
 
     MAILBOX_COUNTER_TTL: 24 * 3600,
 
-    SCHEMA_VERSION: '1.0',
-
     // how much plaintext to store in a full text indexed field
-    MAX_PLAINTEXT_INDEXED: 2 * 1024,
+    MAX_PLAINTEXT_INDEXED: 1 * 1024,
 
     // how much plaintext to store before truncating
     MAX_PLAINTEXT_CONTENT: 100 * 1024,
 
-    // how much HTML content to store. not indexed
+    // how much HTML content to store before truncating. not indexed
     MAX_HTML_CONTENT: 300 * 1024,
 
     MAX_AUTOREPLY_INTERVAL: 4 * 24 * 3600 * 1000,