updated user delete task

lib/api/users.js

@@ -1344,9 +1344,9 @@ module.exports = (db, server, userHandler) => {
 
             let user = new ObjectID(result.value.user);
 
-            let status;
+            let task;
             try {
-                status = await userHandler.delete(user, {});
+                task = await userHandler.delete(user, {});
             } catch (err) {
                 res.json({
                     error: err.message,
@@ -1355,10 +1355,15 @@ module.exports = (db, server, userHandler) => {
                 return next();
             }
 
-            res.json({
+            res.json(
-                success: status,
+                Object.assign(
-                code: 'TaskScheduled'
+                    {
-            });
+                        success: !!task,
+                        code: 'TaskScheduled'
+                    },
+                    task || {}
+                )
+            );
             return next();
         })
     );

lib/events.js

@@ -24,7 +24,8 @@ module.exports = {
     ASP_DELETED: 'asp.deleted',
     USER_CREATED: 'user.created',
     USER_PASSWORD_CHANGED: 'user.password.changed',
-    USER_DELETED: 'user.deleted',
+    USER_DELETE_STARTED: 'user.delete.started',
+    USER_DELETE_COMPLETED: 'user.delete.completed',
     AUTOREPLY_USER_ENABLED: 'autoreply.user.enabled',
     AUTOREPLY_USER_DISABLED: 'autoreply.user.disabled',
     MFA_TOTP_ENABLED: 'mfa.totp.enabled',

lib/tasks/user-delete.js

@@ -4,114 +4,213 @@ const log = require('npmlog');
 const db = require('../db');
 const consts = require('../consts');
 
-const BATCH_SIZE = 200;
+const { publish, USER_DELETE_COMPLETED } = require('../events');
 
-let run = async taskData => {
+const BATCH_SIZE = 500;
-    let cursor = await db.users.collection('messages').find(
-        {
-            user: taskData.user,
-            userDeleted: { $ne: true }
-        },
-        {
-            projection: {
-                _id: true
-            }
-        }
-    );
 
+const deleteMessages = async taskData => {
     let rdate = new Date(Date.now() + consts.DELETED_USER_MESSAGE_RETENTION).getTime();
+    let lastId;
 
-    let messageData;
+    let markedAsDeleted = 0;
-    let updateEntries = [];
-
-    let executeBatchUpdate = async () => {
-        await db.database.collection('messages').bulkWrite(updateEntries, {
-            ordered: false,
-            w: 1
-        });
-        log.verbose('Tasks', 'task=user-delete id=%s user=%s message=%s', taskData._id, taskData.user, `Marked ${updateEntries.length} messages for deletion`);
-        updateEntries = [];
-    };
 
     try {
-        while ((messageData = await cursor.next())) {
+        let done = false;
-            updateEntries.push({
+        while (!done) {
-                updateOne: {
+            let query = {
-                    filter: {
+                user: taskData.user,
-                        _id: messageData._id
+                userDeleted: { $ne: true }
+            };
+
+            if (lastId) {
+                query._id = { $gt: lastId };
+            }
+
+            let messages = await db.users
+                .collection('messages')
+                .find(query, {
+                    sort: { _id: 1 },
+                    projection: {
+                        _id: true
                     },
-                    update: {
+                    limit: BATCH_SIZE
-                        $set: {
+                })
-                            exp: true,
+                .toArray();
-                            rdate,
+            if (!messages.length) {
-                            userDeleted: true
+                // all done
+                done = true;
+                break;
+            }
+            messages = messages.map(messageData => messageData._id);
+            lastId = messages[messages.length - 1];
+
+            let updateEntries = [];
+            messages.forEach(message => {
+                updateEntries.push({
+                    updateOne: {
+                        filter: {
+                            _id: message
+                        },
+                        update: {
+                            $set: {
+                                exp: true,
+                                rdate,
+                                userDeleted: true
+                            }
                         }
                     }
-                }
+                });
+            });
+            let bulkResult = await db.database.collection('messages').bulkWrite(updateEntries, {
+                ordered: false,
+                w: 1
             });
 
-            if (updateEntries.length >= BATCH_SIZE) {
+            markedAsDeleted += (bulkResult && bulkResult.modifiedCount) || 0;
-                try {
-                    await executeBatchUpdate();
-                } catch (err) {
-                    await cursor.close();
-                    throw err;
-                }
-            }
         }
-        await cursor.close();
     } catch (err) {
-        log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to fetch messages', err.message);
+        err.markedAsDeleted = markedAsDeleted;
-        err.code = 'InternalDatabaseError';
         throw err;
+    } finally {
+        log.verbose('Tasks', 'task=user-delete id=%s user=%s message=%s', taskData._id, taskData.user, `Marked ${markedAsDeleted} messages for deletion`);
     }
+    return markedAsDeleted;
+};
 
-    if (updateEntries.length) {
+const deleteRegistryAddresses = async taskData => {
-        await executeBatchUpdate();
+    let lastId;
-    }
+
+    let deleted = 0;
 
     try {
-        await db.database.collection('mailboxes').deleteMany({ user: taskData.user });
+        let done = false;
+        while (!done) {
+            let query = {
+                user: taskData.user
+            };
+
+            if (lastId) {
+                query._id = { $gt: lastId };
+            }
+
+            let addresses = await db.users
+                .collection('addressregister')
+                .find(query, {
+                    sort: { _id: 1 },
+                    projection: {
+                        _id: true
+                    },
+                    limit: BATCH_SIZE
+                })
+                .toArray();
+            if (!addresses.length) {
+                // all done
+                done = true;
+                break;
+            }
+            addresses = addresses.map(addresseData => addresseData._id);
+            lastId = addresses[addresses.length - 1];
+
+            let updateEntries = [];
+            addresses.forEach(message => {
+                updateEntries.push({
+                    deleteOne: {
+                        filter: {
+                            _id: message
+                        }
+                    }
+                });
+            });
+            let bulkResult = await db.database.collection('messages').bulkWrite(updateEntries, {
+                ordered: false,
+                w: 1
+            });
+
+            deleted += (bulkResult && bulkResult.deletedCount) || 0;
+        }
+    } catch (err) {
+        err.deleted = deleted;
+        throw err;
+    } finally {
+        log.verbose('Tasks', 'task=user-delete id=%s user=%s message=%s', taskData._id, taskData.user, `Deleted ${deleted} addresses from registry`);
+    }
+    return deleted;
+};
+
+const run = async taskData => {
+    let result = {};
+
+    try {
+        let delRes = await db.database.collection('mailboxes').deleteMany({ user: taskData.user });
+        result.mailboxes = { deleted: delRes.deletedCount };
     } catch (err) {
         log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete mailboxes', err.message);
         err.code = 'InternalDatabaseError';
+        result.mailboxes = { error: err.message };
         throw err;
     }
 
     try {
-        await db.users.collection('asps').deleteMany({ user: taskData.user });
+        let delRes = await db.users.collection('asps').deleteMany({ user: taskData.user });
+        result.asps = { deleted: delRes.deletedCount };
     } catch (err) {
         log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete asps', err.message);
         err.code = 'InternalDatabaseError';
+        result.asps = { error: err.message };
         throw err;
     }
 
     try {
-        await db.users.collection('filters').deleteMany({ user: taskData.user });
+        let delRes = await db.users.collection('filters').deleteMany({ user: taskData.user });
+        result.filters = { deleted: delRes.deletedCount };
     } catch (err) {
         log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete filters', err.message);
         err.code = 'InternalDatabaseError';
+        result.filters = { error: err.message };
         throw err;
     }
 
     try {
-        await db.users.collection('autoreplies').deleteMany({ user: taskData.user });
+        let delRes = await db.users.collection('autoreplies').deleteMany({ user: taskData.user });
+        result.autoreplies = { deleted: delRes.deletedCount };
     } catch (err) {
         log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete autoreplies', err.message);
         err.code = 'InternalDatabaseError';
+        result.autoreplies = { error: err.message };
         throw err;
     }
 
     try {
-        // Should this run in a batch instead? Might have quite a lot of addresses tracked
+        let deleted = await deleteRegistryAddresses(taskData);
-        await db.database.collection('addressregister').deleteMany({ user: taskData.user });
+        result.addressregister = { deleted };
     } catch (err) {
         log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete autoreplies', err.message);
         err.code = 'InternalDatabaseError';
+        result.addressregister = { error: err.message, deleted: err.deleted };
+        throw err;
+    }
+
+    try {
+        // mark messages for deletion
+        let markedAsDeleted = await deleteMessages(taskData);
+        result.messages = { deleted: markedAsDeleted };
+    } catch (err) {
+        log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to fetch messages', err.message);
+        err.code = 'InternalDatabaseError';
+        result.messages = { error: err.message, deleted: err.markedAsDeleted };
         throw err;
     }
 
     log.verbose('Tasks', 'task=user-delete id=%s user=%s message=%s', taskData._id, taskData.user, `Cleared user specific data`);
+
+    result.task = taskData._id.toString();
+
+    await publish(db.redis, {
+        ev: USER_DELETE_COMPLETED,
+        user: taskData.user,
+        result
+    });
+
     return true;
 };
 

lib/user-handler.js

@@ -24,7 +24,7 @@ const {
     ASP_CREATED,
     ASP_DELETED,
     USER_CREATED,
-    USER_DELETED,
+    USER_DELETE_STARTED,
     MFA_TOTP_ENABLED,
     MFA_TOTP_DISABLED,
     MFA_CUSTOM_ENABLED,
@@ -3332,8 +3332,11 @@ class UserHandler {
             // ignore
         }
 
+        let result = {};
+
         try {
-            await this.users.collection('addresses').deleteMany({ user });
+            let delRes = await this.users.collection('addresses').deleteMany({ user });
+            result.addresses = { deleted: delRes.deletedCount };
         } catch (err) {
             log.error('USERDEL', 'Failed to delete addresses for id=%s error=%s', user, err.message);
             err.code = 'InternalDatabaseError';
@@ -3341,7 +3344,8 @@ class UserHandler {
         }
 
         try {
-            await this.users.collection('users').deleteOne({ _id: user });
+            let delRes = await this.users.collection('users').deleteOne({ _id: user });
+            result.users = { deleted: delRes.deletedCount };
         } catch (err) {
             log.error('USERDEL', 'Failed to delete user id=%s error=%s', user, err.message);
             err.code = 'InternalDatabaseError';
@@ -3350,7 +3354,7 @@ class UserHandler {
 
         // set up a task to delete user messages
         let now = new Date();
-        await this.database.collection('tasks').insertOne({
+        let insRes = await this.database.collection('tasks').insertOne({
             task: 'user-delete',
             locked: false,
             lockedUntil: now,
@@ -3359,6 +3363,8 @@ class UserHandler {
             user
         });
 
+        result.task = insRes.insertedId && insRes.insertedId.toString();
+
         try {
             await this.logAuthEvent(user, {
                 action: 'delete user',
@@ -3371,11 +3377,12 @@ class UserHandler {
         }
 
         await publish(this.redis, {
-            ev: USER_DELETED,
+            ev: USER_DELETE_STARTED,
-            user
+            user,
+            result
         });
 
-        return true;
+        return result;
     }
 
     async pushDefaultMessages(userData, tags) {

package.json

@@ -1,6 +1,6 @@
 {
     "name": "wildduck",
-    "version": "1.32.1",
+    "version": "1.32.2",
     "description": "IMAP/POP3 server built with Node.js and MongoDB",
     "main": "server.js",
     "scripts": {