From fcd344bfd1a24097b7e4dc30dac8a6f8b2d254ab Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Fri, 13 Sep 2019 15:05:36 +0300 Subject: [PATCH] Updated user-delete task to not update all messages at once --- lib/api/users.js | 4 +- lib/tasks/user-delete.js | 177 ++++++++++++++++++++++----------------- 2 files changed, 103 insertions(+), 78 deletions(-) diff --git a/lib/api/users.js b/lib/api/users.js index c45c8188..d7504d92 100644 --- a/lib/api/users.js +++ b/lib/api/users.js @@ -2089,8 +2089,10 @@ module.exports = (db, server, userHandler) => { }); return next(); } + res.json({ - success: status + success: status, + code: 'TaskScheduled' }); return next(); }) diff --git a/lib/tasks/user-delete.js b/lib/tasks/user-delete.js index 5a4ddd4a..cd8c2e69 100644 --- a/lib/tasks/user-delete.js +++ b/lib/tasks/user-delete.js @@ -4,87 +4,110 @@ const log = require('npmlog'); const db = require('../db'); const consts = require('../consts'); -module.exports = (taskData, options, callback) => { - // keep messages around for a while, delete other stuff +const BATCH_SIZE = 200; - let processMessages = done => { - db.database.collection('messages').updateMany( - { user: taskData.user }, - { - $set: { - exp: true, - rdate: new Date(Date.now() + consts.DELETED_USER_MESSAGE_RETENTION), - userDeleted: true - } - }, - err => { - if (err) { - log.error( - 'Tasks', - 'task=user-delete id=%s user=%s message=%s error=%s', - taskData._id, - taskData.user, - 'Failed to update messages', - err.message - ); - err.code = 'InternalDatabaseError'; - return callback(err); - } - done(); +let run = async taskData => { + let cursor = await db.users.collection('messages').find( + { + user: taskData.user, + userDeleted: { $ne: true } + }, + { + projection: { + _id: true } - ); + } + ); + + let rdate = new Date(Date.now() + consts.DELETED_USER_MESSAGE_RETENTION); + + let messageData; + let updateEntries = []; + + let executeBatchUpdate = async () => { + await db.database.collection('messages').bulkWrite(updateEntries, { + ordered: false, + w: 1 + }); + log.verbose('Tasks', 'task=user-delete id=%s user=%s message=%s', taskData._id, taskData.user, `Marked ${updateEntries.length} messages for deletion`); + updateEntries = []; }; - processMessages(() => { - db.database.collection('mailboxes').deleteMany({ user: taskData.user }, err => { - if (err) { - log.error( - 'Tasks', - 'task=user-delete id=%s user=%s message=%s error=%s', - taskData._id, - taskData.user, - 'Failed to delete mailboxes', - err.message - ); - err.code = 'InternalDatabaseError'; - } - - db.users.collection('asps').deleteMany({ user: taskData.user }, err => { - if (err) { - log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete asps', err.message); - err.code = 'InternalDatabaseError'; - } - - db.users.collection('filters').deleteMany({ user: taskData.user }, err => { - if (err) { - log.error( - 'Tasks', - 'task=user-delete id=%s user=%s message=%s error=%s', - taskData._id, - taskData.user, - 'Failed to delete filters', - err.message - ); - err.code = 'InternalDatabaseError'; - } - - db.users.collection('autoreplies').deleteMany({ user: taskData.user }, err => { - if (err) { - log.error( - 'Tasks', - 'task=user-delete id=%s user=%s message=%s error=%s', - taskData._id, - taskData.user, - 'Failed to delete autoreplies', - err.message - ); - err.code = 'InternalDatabaseError'; + try { + while ((messageData = await cursor.next())) { + updateEntries.push({ + updateOne: { + filter: { + _id: messageData._id + }, + update: { + $set: { + exp: true, + rdate, + userDeleted: true } - - return callback(null, true); - }); - }); + } + } }); - }); - }); + + if (updateEntries.length >= BATCH_SIZE) { + try { + await executeBatchUpdate(); + } catch (err) { + await cursor.close(); + throw err; + } + } + } + await cursor.close(); + } catch (err) { + log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to fetch messages', err.message); + err.code = 'InternalDatabaseError'; + throw err; + } + + if (updateEntries.length) { + await executeBatchUpdate(); + } + + try { + await db.database.collection('mailboxes').deleteMany({ user: taskData.user }); + } catch (err) { + log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete mailboxes', err.message); + err.code = 'InternalDatabaseError'; + throw err; + } + + try { + await db.users.collection('asps').deleteMany({ user: taskData.user }); + } catch (err) { + log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete asps', err.message); + err.code = 'InternalDatabaseError'; + throw err; + } + + try { + await db.users.collection('filters').deleteMany({ user: taskData.user }); + } catch (err) { + log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete filters', err.message); + err.code = 'InternalDatabaseError'; + throw err; + } + + try { + await db.users.collection('autoreplies').deleteMany({ user: taskData.user }); + } catch (err) { + log.error('Tasks', 'task=user-delete id=%s user=%s message=%s error=%s', taskData._id, taskData.user, 'Failed to delete autoreplies', err.message); + err.code = 'InternalDatabaseError'; + throw err; + } + + log.verbose('Tasks', 'task=user-delete id=%s user=%s message=%s', taskData._id, taskData.user, `Cleared user specific data`); + return true; +}; + +module.exports = (taskData, options, callback) => { + run(taskData) + .then(response => callback(null, response)) + .catch(callback); };