nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2025-09-13 16:45:27 +08:00
Code Issues Projects Releases Packages Wiki Activity

use paging when FETCHing mailbox contents

Browse source
This commit is contained in:
Andris Reinman 2020-12-13 10:07:23 +02:00
parent 345ea114a7
commit 736ef7de44
3 changed files with 242 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/consts.js
View file

@ -103,5 +103,8 @@ module.exports = {
TOTP_WINDOW_SIZE: 6,
// how often to send processing updates for long running commands
LONG_COMMAND_NOTIFY_TTL: 1 * 60 * 1000
LONG_COMMAND_NOTIFY_TTL: 1 * 60 * 1000,
// when paging through a large list, how many entries to request per page
CURSOR_MAX_PAGE_SIZE: 250
};

22
lib/handlers/on-fetch.js
View file

@ -137,12 +137,22 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
return callback(...args);
};
let lastUid = false;
// instead of fetching all messages at once from a large mailbox
// we page it into smaller queries
let processPage = () => {
if (lastUid) {
query.uid = { $gt: lastUid };
}
let sort = { uid: 1 };
let cursor = db.database
.collection('messages')
.find(query)
.project(projection)
.sort(sort)
.limit(consts.CURSOR_MAX_PAGE_SIZE)
.setReadPreference('secondaryPreferred')
.maxTimeMS(consts.DB_MAX_TIME_MESSAGES);
@ -155,6 +165,7 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
let startTime = Date.now();
let rowCount = 0;
let totalBytes = 0;
let processedCount = 0;
let processNext = () => {
cursor.next((err, messageData) => {
if (err) {
@ -192,6 +203,11 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
if (!messageData) {
return cursor.close(() => {
if (processedCount === consts.CURSOR_MAX_PAGE_SIZE) {
// might have more entries, check next page
return setTimeout(processPage, 10);
}
server.logger.debug(
{
tnx: 'fetch',
@ -212,6 +228,9 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
});
}
processedCount++;
lastUid = messageData.uid;
if (queryAll && !session.selected.uidList.includes(messageData.uid)) {
// skip processing messages that we do not know about yet
return processNext();
@ -380,6 +399,9 @@ module.exports = (server, messageHandler, userCache) => (mailbox, options, sessi
};
processNext();
};
processPage();
});
});
}

1
lib/tools.js
View file

@ -28,6 +28,7 @@ function checkRangeQuery(uids, ne) {
for (let i = 1, len = uids.length; i < len; i++) {
if (uids[i] !== uids[i - 1] + 1) {
// TODO: group into AND conditions, otherwise expands too much!
return {
[!ne ? '$in' : '$nin']: uids
};