replaced browserbox with imapflow, utf7 with iconv-lite

.eslintrc

@@ -7,6 +7,6 @@
     },
     "extends": ["nodemailer", "prettier"],
     "parserOptions": {
-        "ecmaVersion": 2017
+        "ecmaVersion": 2018
     }
 }

docs/README.md

@@ -13,7 +13,7 @@ WildDuck tries to follow Gmail in product design. If there's a decision to be ma
 
 -   _MongoDB_ to store all data
 -   _Redis_ for pubsub and counters
--   _Node.js_ at least version 8.0.0
+-   _Node.js_ at least version 10.0.0
 
 **Optional requirements**
 
@@ -34,7 +34,6 @@ Attachment de-duplication and compression gives up to 56% of storage size reduct
 
 ![](https://raw.githubusercontent.com/nodemailer/wildduck/master/assets/storage.png)
 
-
 ## Goals of the Project
 
 1.  Build a scalable and distributed IMAP/POP3 server that uses clustered database instead of single machine file system as mail store
@@ -60,4 +59,3 @@ Attachment de-duplication and compression gives up to 56% of storage size reduct
 ## License
 
 WildDuck Mail Agent is licensed under the [European Union Public License 1.1](http://ec.europa.eu/idabc/eupl.html) or later.
-

examples/append.js

@@ -1,59 +1,47 @@
 /* eslint no-console:0 */
 
 'use strict';
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
-const rawpath = process.argv[2];
 
+const rawpath = process.argv[2];
 const config = require('wild-config');
-const BrowserBox = require('browserbox');
+const { ImapFlow } = require('imapflow');
 
 const raw = require('fs').readFileSync(rawpath);
 console.log('Processing %s of %s bytes', rawpath, raw.length);
 
-const client = new BrowserBox('localhost', config.imap.port, {
+const client = new ImapFlow({
-    useSecureTransport: config.imap.secure,
+    host: '127.0.0.1',
+    port: config.imap.port,
+    secure: config.imap.secure,
     auth: {
         user: 'myuser',
         pass: 'verysecret'
     },
-    id: {
-        name: 'My Client',
-        version: '0.1'
-    },
     tls: {
         rejectUnauthorized: false
+    },
+    clientInfo: {
+        name: 'My Client',
+        version: '0.1'
     }
 });
 
-client.onerror = function(err) {
+client.on('error', err => {
     console.log(err);
     process.exit(1);
-};
+});
 
-client.onauth = function() {
+client
-    client.upload('INBOX', raw, false, err => {
+    .connect()
-        if (err) {
+    .then(() => client.append('INBOX', raw))
-            console.log(err);
+    .then(() => client.mailboxOpen('INBOX'))
-            return process.exit(1);
+    .then(mailbox => client.fetchOne(mailbox.exists, { bodyStructure: true, source: true }))
-        }
+    .then(data => {
-
+        console.log(data);
-        client.selectMailbox('INBOX', (err, mailbox) => {
+        console.log('<<<%s>>>', data.source.toString());
-            if (err) {
+        return process.exit(0);
-                console.log(err);
+    })
-                return process.exit(1);
+    .catch(err => {
-            }
+        console.log(err);
-            console.log(mailbox);
+        process.exit(1);
-
-            client.listMessages(mailbox.exists, ['BODY.PEEK[]', 'BODYSTRUCTURE'], (err, data) => {
-                if (err) {
-                    console.log(err);
-                    return process.exit(1);
-                }
-                console.log('<<<%s>>>', data[0]['body[]']);
-                return process.exit(0);
-            });
-        });
     });
-};
-
-client.connect();

imap-core/lib/commands/create.js

@@ -1,7 +1,6 @@
 'use strict';
 
-const imapTools = require('../imap-tools');
+const { normalizeMailbox, utf7decode } = require('../imap-tools');
-const utf7 = require('utf7').imap;
 
 // tag CREATE "mailbox"
 
@@ -20,7 +19,7 @@ module.exports = {
 
         if (!this.acceptUTF8Enabled) {
             // decode before normalizing to uncover stuff like ending / etc.
-            path = utf7.decode(path);
+            path = utf7decode(path);
         }
 
         // Check if CREATE method is set
@@ -58,7 +57,7 @@ module.exports = {
             });
         }
 
-        path = imapTools.normalizeMailbox(path);
+        path = normalizeMailbox(path);
 
         let logdata = {
             short_message: '[CREATE]',

imap-core/lib/commands/getquotaroot.js

@@ -1,8 +1,7 @@
 'use strict';
 
 const imapHandler = require('../handler/imap-handler');
-const imapTools = require('../imap-tools');
+const { normalizeMailbox, utf7encode } = require('../imap-tools');
-const utf7 = require('utf7').imap;
 
 // tag GETQUOTAROOT "mailbox"
 
@@ -18,7 +17,7 @@ module.exports = {
 
     handler(command, callback) {
         let path = Buffer.from((command.attributes[0] && command.attributes[0].value) || '', 'binary').toString();
-        path = imapTools.normalizeMailbox(path, !this.acceptUTF8Enabled);
+        path = normalizeMailbox(path, !this.acceptUTF8Enabled);
 
         if (typeof this._server.onGetQuota !== 'function') {
             return callback(null, {
@@ -64,7 +63,7 @@ module.exports = {
             }
 
             if (!this.acceptUTF8Enabled) {
-                path = utf7.encode(path);
+                path = utf7encode(path);
             } else {
                 path = Buffer.from(path);
             }

imap-core/lib/commands/list.js

@@ -1,8 +1,7 @@
 'use strict';
 
 const imapHandler = require('../handler/imap-handler');
-const imapTools = require('../imap-tools');
+const { normalizeMailbox, utf7encode, filterFolders, generateFolderListing } = require('../imap-tools');
-const utf7 = require('utf7').imap;
 
 // tag LIST (SPECIAL-USE) "" "%" RETURN (SPECIAL-USE)
 
@@ -107,7 +106,7 @@ module.exports = {
             });
         }
 
-        let query = imapTools.normalizeMailbox(reference + path, !this.acceptUTF8Enabled);
+        let query = normalizeMailbox(reference + path, !this.acceptUTF8Enabled);
 
         let logdata = {
             short_message: '[LIST]',
@@ -130,7 +129,7 @@ module.exports = {
                 });
             }
 
-            imapTools.filterFolders(imapTools.generateFolderListing(list), query).forEach(folder => {
+            filterFolders(generateFolderListing(list), query).forEach(folder => {
                 if (!folder) {
                     return;
                 }
@@ -162,7 +161,7 @@ module.exports = {
 
                 let path = folder.path;
                 if (!this.acceptUTF8Enabled) {
-                    path = utf7.encode(path);
+                    path = utf7encode(path);
                 } else {
                     path = Buffer.from(path);
                 }

imap-core/lib/commands/lsub.js

@@ -1,8 +1,7 @@
 'use strict';
 
 const imapHandler = require('../handler/imap-handler');
-const imapTools = require('../imap-tools');
+const { normalizeMailbox, utf7encode, filterFolders, generateFolderListing } = require('../imap-tools');
-const utf7 = require('utf7').imap;
 
 // tag LSUB "" "%"
 
@@ -32,7 +31,7 @@ module.exports = {
             });
         }
 
-        let query = imapTools.normalizeMailbox(reference + path, !this.acceptUTF8Enabled);
+        let query = normalizeMailbox(reference + path, !this.acceptUTF8Enabled);
 
         let logdata = {
             short_message: '[LSUB]',
@@ -55,14 +54,14 @@ module.exports = {
                 });
             }
 
-            imapTools.filterFolders(imapTools.generateFolderListing(list, true), query).forEach(folder => {
+            filterFolders(generateFolderListing(list, true), query).forEach(folder => {
                 if (!folder) {
                     return;
                 }
 
                 let path = folder.path;
                 if (!this.acceptUTF8Enabled) {
-                    path = utf7.encode(path);
+                    path = utf7encode(path);
                 } else {
                     path = Buffer.from(path);
                 }
@@ -98,6 +97,6 @@ module.exports = {
 
         // Do folder listing
         // Concat reference and mailbox. No special reference handling whatsoever
-        this._server.onLsub(imapTools.normalizeMailbox(reference + path), this.session, lsubResponse);
+        this._server.onLsub(normalizeMailbox(reference + path), this.session, lsubResponse);
     }
 };

imap-core/lib/imap-tools.js

@@ -1,13 +1,19 @@
 'use strict';
 
 const Indexer = require('./indexer/indexer');
-const utf7 = require('utf7').imap;
 const libmime = require('libmime');
 const punycode = require('punycode');
+const iconv = require('iconv-lite');
 
 module.exports.systemFlagsFormatted = ['\\Answered', '\\Flagged', '\\Draft', '\\Deleted', '\\Seen'];
 module.exports.systemFlags = ['\\answered', '\\flagged', '\\draft', '\\deleted', '\\seen'];
 
+const utf7encode = str => iconv.encode(str, 'utf-7-imap').toString();
+const utf7decode = str => iconv.decode(Buffer.from(str), 'utf-7-imap').toString();
+
+module.exports.utf7encode = utf7encode;
+module.exports.utf7decode = utf7decode;
+
 module.exports.fetchSchema = {
     body: [
         true,
@@ -195,11 +201,11 @@ module.exports.searchMapping = {
  * @param {range} range Sequence range, eg "1,2,3:7"
  * @returns {Boolean} True if the string looks like a sequence range
  */
-module.exports.validateSequnce = function(range) {
+module.exports.validateSequnce = function (range) {
     return !!(range.length && /^(\d+|\*)(:\d+|:\*)?(,(\d+|\*)(:\d+|:\*)?)*$/.test(range));
 };
 
-module.exports.normalizeMailbox = function(mailbox, utf7Encoded) {
+module.exports.normalizeMailbox = function (mailbox, utf7Encoded) {
     if (!mailbox) {
         return '';
     }
@@ -214,7 +220,7 @@ module.exports.normalizeMailbox = function(mailbox, utf7Encoded) {
     }
 
     if (utf7Encoded) {
-        parts = parts.map(value => utf7.decode(value));
+        parts = parts.map(value => utf7decode(value));
     }
 
     mailbox = parts.join('/');
@@ -222,7 +228,7 @@ module.exports.normalizeMailbox = function(mailbox, utf7Encoded) {
     return mailbox;
 };
 
-module.exports.generateFolderListing = function(folders, skipHierarchy) {
+module.exports.generateFolderListing = function (folders, skipHierarchy) {
     let items = new Map();
     let parents = [];
 
@@ -328,7 +334,7 @@ module.exports.generateFolderListing = function(folders, skipHierarchy) {
     return result;
 };
 
-module.exports.filterFolders = function(folders, query) {
+module.exports.filterFolders = function (folders, query) {
     query = query
         // remove excess * and %
         .replace(/\*\*+/g, '*')
@@ -345,7 +351,7 @@ module.exports.filterFolders = function(folders, query) {
     return folders.filter(folder => !!regex.test(folder.path));
 };
 
-module.exports.getMessageRange = function(uidList, range, isUid) {
+module.exports.getMessageRange = function (uidList, range, isUid) {
     range = (range || '').toString();
 
     let result = [];
@@ -390,7 +396,7 @@ module.exports.getMessageRange = function(uidList, range, isUid) {
     return result;
 };
 
-module.exports.packMessageRange = function(uidList) {
+module.exports.packMessageRange = function (uidList) {
     if (!Array.isArray(uidList)) {
         uidList = [].concat(uidList || []);
     }
@@ -427,7 +433,7 @@ module.exports.packMessageRange = function(uidList) {
  * @param {Date} date Date object to parse
  * @returns {String} Internaldate formatted date
  */
-module.exports.formatInternalDate = function(date) {
+module.exports.formatInternalDate = function (date) {
     let day = date.getUTCDate(),
         month = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][date.getUTCMonth()],
         year = date.getUTCFullYear(),
@@ -485,7 +491,7 @@ module.exports.formatInternalDate = function(date) {
  * @param {Object} options Options for the indexer
  * @returns {Array} Resolved responses
  */
-module.exports.getQueryResponse = function(query, message, options) {
+module.exports.getQueryResponse = function (query, message, options) {
     options = options || {};
 
     // for optimization purposes try to use cached mimeTree etc. if available

imap-core/test/client.js

@@ -5,52 +5,42 @@
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
 
 const config = require('wild-config');
-const BrowserBox = require('browserbox');
+const { ImapFlow } = require('imapflow');
 
-const client = new BrowserBox('localhost', config.imap.port, {
+const client = new ImapFlow({
-    useSecureTransport: config.imap.secure,
+    host: '127.0.0.1',
+    port: config.imap.port,
+    secure: config.imap.secure,
     auth: {
         user: 'testuser',
         pass: 'secretpass'
     },
-    id: {
-        name: 'My Client',
-        version: '0.1'
-    },
     tls: {
         rejectUnauthorized: false
+    },
+    clientInfo: {
+        name: 'My Client',
+        version: '0.1'
     }
 });
 
-client.onerror = function(err) {
+client.on('error', err => {
     console.log(err);
     process.exit(1);
-};
+});
 
-client.onauth = function() {
+const raw = Buffer.from('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz\r\n');
-    client.upload('INBOX', 'from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz\r\n', false, err => {
-        if (err) {
-            console.log(err);
-            return process.exit(1);
-        }
 
-        client.selectMailbox('INBOX', (err, mailbox) => {
+client
-            if (err) {
+    .connect()
-                console.log(err);
+    .then(() => client.append('INBOX', raw))
-                return process.exit(1);
+    .then(() => client.mailboxOpen('INBOX'))
-            }
+    .then(mailbox => client.fetchOne(mailbox.exists, { bodyStructure: true, source: true }))
-            console.log(mailbox);
+    .then(data => {
-
+        console.log('<<<%s>>>', data.source.toString());
-            client.listMessages(mailbox.exists, ['BODY.PEEK[]', 'BODYSTRUCTURE'], (err, data) => {
+        return process.exit(0);
-                if (err) {
+    })
-                    console.log(err);
+    .catch(err => {
-                    return process.exit(1);
+        console.log(err);
-                }
+        process.exit(1);
-                console.log('<<<%s>>>', data[0]['body[]']);
-                return process.exit(0);
-            });
-        });
     });
-};
-
-client.connect();

package.json

@@ -17,10 +17,9 @@
     "devDependencies": {
         "ajv": "6.12.2",
         "apidoc": "0.22.1",
-        "browserbox": "0.9.1",
         "chai": "4.2.0",
         "docsify-cli": "4.4.0",
-        "eslint": "6.8.0",
+        "eslint": "7.0.0",
         "eslint-config-nodemailer": "1.2.0",
         "eslint-config-prettier": "6.11.0",
         "grunt": "1.1.0",
@@ -29,6 +28,7 @@
         "grunt-mocha-test": "0.13.3",
         "grunt-shell-spawn": "0.4.0",
         "grunt-wait": "0.3.0",
+        "imapflow": "1.0.46",
         "mailparser": "2.7.7",
         "mocha": "7.1.2",
         "request": "2.88.2",
@@ -54,7 +54,7 @@
         "libbase64": "1.2.1",
         "libmime": "4.2.1",
         "libqp": "1.1.0",
-        "mailsplit": "4.6.4",
+        "mailsplit": "5.0.0",
         "mobileconfig": "2.3.1",
         "mongo-cursor-pagination": "7.3.0",
         "mongodb": "3.5.7",
@@ -68,11 +68,11 @@
         "qrcode": "1.4.4",
         "restify": "8.5.1",
         "restify-logger": "2.0.1",
+        "saslprep": "1.0.3",
         "seq-index": "1.1.0",
         "smtp-server": "3.6.0",
         "speakeasy": "2.0.0",
         "u2f": "0.1.3",
-        "utf7": "1.0.2",
         "uuid": "8.0.0",
         "wild-config": "1.5.1",
         "yargs": "15.3.1"
@@ -80,5 +80,8 @@
     "repository": {
         "type": "git",
         "url": "git://github.com/wildduck-email/wildduck.git"
+    },
+    "engines": {
+        "node": ">=10.0.0"
     }
 }

test/filtering-test.js

@@ -10,9 +10,9 @@ const crypto = require('crypto');
 const chai = require('chai');
 const request = require('request');
 const fs = require('fs');
-const BrowserBox = require('browserbox');
 const simpleParser = require('mailparser').simpleParser;
 const nodemailer = require('nodemailer');
+const { ImapFlow } = require('imapflow');
 
 const transporter = nodemailer.createTransport({
     lmtp: true,
@@ -340,59 +340,61 @@ describe('Send multiple messages', function () {
             crypto.createHash('md5').update(swanJpg).digest('hex')
         ];
 
-        const client = new BrowserBox('localhost', 9993, {
+        const client = new ImapFlow({
-            useSecureTransport: true,
+            host: '127.0.0.1',
+            port: 9993,
+            secure: true,
             auth: {
                 user: 'user4',
                 pass: 'secretpass'
             },
-            id: {
-                name: 'My Client',
-                version: '0.1'
-            },
             tls: {
                 rejectUnauthorized: false
+            },
+            clientInfo: {
+                name: 'My Client',
+                version: '0.1'
             }
         });
 
-        client.onerror = err => {
+        client.on('error', err => {
             expect(err).to.not.exist;
-        };
+            done();
+        });
+        client.on('close', () => done());
 
-        client.onclose = done;
+        client
-
+            .connect()
-        client.onauth = () => {
+            .then(async () => {
-            client.listMailboxes((err, result) => {
+                const result = await client.list();
-                expect(err).to.not.exist;
+                const folders = result.map(mbox => ({ name: mbox.name, specialUse: mbox.specialUse || false })).sort((a, b) => a.name.localeCompare(b.name));
-                let folders = result.children.map(mbox => ({ name: mbox.name, specialUse: mbox.specialUse || false }));
                 expect(folders).to.deep.equal([
-                    { name: 'INBOX', specialUse: false },
                     { name: 'Drafts', specialUse: '\\Drafts' },
+                    { name: 'INBOX', specialUse: '\\Inbox' },
                     { name: 'Junk', specialUse: '\\Junk' },
                     { name: 'Sent Mail', specialUse: '\\Sent' },
                     { name: 'Trash', specialUse: '\\Trash' }
                 ]);
-                client.selectMailbox('INBOX', { condstore: true }, (err, result) => {
-                    expect(err).to.not.exist;
-                    expect(result.exists).gte(1);
 
-                    client.listMessages(result.exists, ['uid', 'flags', 'body.peek[]'], (err, messages) => {
+                const mailbox = await client.mailboxOpen('INBOX');
-                        expect(err).to.not.exist;
+                expect(mailbox.exists).gte(1);
-                        expect(messages.length).equal(1);
 
-                        let messageInfo = messages[0];
+                let messages = [];
-                        simpleParser(messageInfo['body[]'], (err, parsed) => {
+                for await (let msg of client.fetch(mailbox.exists, { uid: true, source: true })) {
-                            expect(err).to.not.exist;
+                    messages.push(msg);
-                            checksums.forEach((checksum, i) => {
+                }
-                                expect(checksum).to.equal(parsed.attachments[i].checksum);
+                expect(messages.length).equal(1);
-                            });
+
-                            client.close();
+                let messageInfo = messages[0];
-                        });
+                let parsed = await simpleParser(messageInfo.source);
-                    });
+                checksums.forEach((checksum, i) => {
+                    expect(checksum).to.equal(parsed.attachments[i].checksum);
                 });
+                client.close();
+            })
+            .catch(err => {
+                expect(err).to.not.exist;
+                client.close();
             });
-        };
-
-        client.connect();
     });
 });