diff --git a/.eslintrc b/.eslintrc index 140a1b3c..cf1fc4a8 100644 --- a/.eslintrc +++ b/.eslintrc @@ -7,6 +7,6 @@ }, "extends": ["nodemailer", "prettier"], "parserOptions": { - "ecmaVersion": 2017 + "ecmaVersion": 2018 } } diff --git a/docs/README.md b/docs/README.md index 1408215e..867e9fe8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,7 @@ WildDuck tries to follow Gmail in product design. If there's a decision to be ma - _MongoDB_ to store all data - _Redis_ for pubsub and counters -- _Node.js_ at least version 8.0.0 +- _Node.js_ at least version 10.0.0 **Optional requirements** @@ -34,7 +34,6 @@ Attachment de-duplication and compression gives up to 56% of storage size reduct ![](https://raw.githubusercontent.com/nodemailer/wildduck/master/assets/storage.png) - ## Goals of the Project 1. Build a scalable and distributed IMAP/POP3 server that uses clustered database instead of single machine file system as mail store @@ -60,4 +59,3 @@ Attachment de-duplication and compression gives up to 56% of storage size reduct ## License WildDuck Mail Agent is licensed under the [European Union Public License 1.1](http://ec.europa.eu/idabc/eupl.html) or later. - diff --git a/examples/append.js b/examples/append.js index a4b9d45f..a6184143 100644 --- a/examples/append.js +++ b/examples/append.js @@ -1,59 +1,47 @@ /* eslint no-console:0 */ 'use strict'; -process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; -const rawpath = process.argv[2]; +const rawpath = process.argv[2]; const config = require('wild-config'); -const BrowserBox = require('browserbox'); +const { ImapFlow } = require('imapflow'); const raw = require('fs').readFileSync(rawpath); console.log('Processing %s of %s bytes', rawpath, raw.length); -const client = new BrowserBox('localhost', config.imap.port, { - useSecureTransport: config.imap.secure, +const client = new ImapFlow({ + host: '127.0.0.1', + port: config.imap.port, + secure: config.imap.secure, auth: { user: 'myuser', pass: 'verysecret' }, - id: { - name: 'My Client', - version: '0.1' - }, tls: { rejectUnauthorized: false + }, + clientInfo: { + name: 'My Client', + version: '0.1' } }); -client.onerror = function(err) { +client.on('error', err => { console.log(err); process.exit(1); -}; +}); -client.onauth = function() { - client.upload('INBOX', raw, false, err => { - if (err) { - console.log(err); - return process.exit(1); - } - - client.selectMailbox('INBOX', (err, mailbox) => { - if (err) { - console.log(err); - return process.exit(1); - } - console.log(mailbox); - - client.listMessages(mailbox.exists, ['BODY.PEEK[]', 'BODYSTRUCTURE'], (err, data) => { - if (err) { - console.log(err); - return process.exit(1); - } - console.log('<<<%s>>>', data[0]['body[]']); - return process.exit(0); - }); - }); +client + .connect() + .then(() => client.append('INBOX', raw)) + .then(() => client.mailboxOpen('INBOX')) + .then(mailbox => client.fetchOne(mailbox.exists, { bodyStructure: true, source: true })) + .then(data => { + console.log(data); + console.log('<<<%s>>>', data.source.toString()); + return process.exit(0); + }) + .catch(err => { + console.log(err); + process.exit(1); }); -}; - -client.connect(); diff --git a/imap-core/lib/commands/create.js b/imap-core/lib/commands/create.js index 7270a7fa..057c9164 100644 --- a/imap-core/lib/commands/create.js +++ b/imap-core/lib/commands/create.js @@ -1,7 +1,6 @@ 'use strict'; -const imapTools = require('../imap-tools'); -const utf7 = require('utf7').imap; +const { normalizeMailbox, utf7decode } = require('../imap-tools'); // tag CREATE "mailbox" @@ -20,7 +19,7 @@ module.exports = { if (!this.acceptUTF8Enabled) { // decode before normalizing to uncover stuff like ending / etc. - path = utf7.decode(path); + path = utf7decode(path); } // Check if CREATE method is set @@ -58,7 +57,7 @@ module.exports = { }); } - path = imapTools.normalizeMailbox(path); + path = normalizeMailbox(path); let logdata = { short_message: '[CREATE]', diff --git a/imap-core/lib/commands/getquotaroot.js b/imap-core/lib/commands/getquotaroot.js index f92e06e6..a4cb396f 100644 --- a/imap-core/lib/commands/getquotaroot.js +++ b/imap-core/lib/commands/getquotaroot.js @@ -1,8 +1,7 @@ 'use strict'; const imapHandler = require('../handler/imap-handler'); -const imapTools = require('../imap-tools'); -const utf7 = require('utf7').imap; +const { normalizeMailbox, utf7encode } = require('../imap-tools'); // tag GETQUOTAROOT "mailbox" @@ -18,7 +17,7 @@ module.exports = { handler(command, callback) { let path = Buffer.from((command.attributes[0] && command.attributes[0].value) || '', 'binary').toString(); - path = imapTools.normalizeMailbox(path, !this.acceptUTF8Enabled); + path = normalizeMailbox(path, !this.acceptUTF8Enabled); if (typeof this._server.onGetQuota !== 'function') { return callback(null, { @@ -64,7 +63,7 @@ module.exports = { } if (!this.acceptUTF8Enabled) { - path = utf7.encode(path); + path = utf7encode(path); } else { path = Buffer.from(path); } diff --git a/imap-core/lib/commands/list.js b/imap-core/lib/commands/list.js index f8eba607..00652b9d 100644 --- a/imap-core/lib/commands/list.js +++ b/imap-core/lib/commands/list.js @@ -1,8 +1,7 @@ 'use strict'; const imapHandler = require('../handler/imap-handler'); -const imapTools = require('../imap-tools'); -const utf7 = require('utf7').imap; +const { normalizeMailbox, utf7encode, filterFolders, generateFolderListing } = require('../imap-tools'); // tag LIST (SPECIAL-USE) "" "%" RETURN (SPECIAL-USE) @@ -107,7 +106,7 @@ module.exports = { }); } - let query = imapTools.normalizeMailbox(reference + path, !this.acceptUTF8Enabled); + let query = normalizeMailbox(reference + path, !this.acceptUTF8Enabled); let logdata = { short_message: '[LIST]', @@ -130,7 +129,7 @@ module.exports = { }); } - imapTools.filterFolders(imapTools.generateFolderListing(list), query).forEach(folder => { + filterFolders(generateFolderListing(list), query).forEach(folder => { if (!folder) { return; } @@ -162,7 +161,7 @@ module.exports = { let path = folder.path; if (!this.acceptUTF8Enabled) { - path = utf7.encode(path); + path = utf7encode(path); } else { path = Buffer.from(path); } diff --git a/imap-core/lib/commands/lsub.js b/imap-core/lib/commands/lsub.js index 823ffd2e..10db4c90 100644 --- a/imap-core/lib/commands/lsub.js +++ b/imap-core/lib/commands/lsub.js @@ -1,8 +1,7 @@ 'use strict'; const imapHandler = require('../handler/imap-handler'); -const imapTools = require('../imap-tools'); -const utf7 = require('utf7').imap; +const { normalizeMailbox, utf7encode, filterFolders, generateFolderListing } = require('../imap-tools'); // tag LSUB "" "%" @@ -32,7 +31,7 @@ module.exports = { }); } - let query = imapTools.normalizeMailbox(reference + path, !this.acceptUTF8Enabled); + let query = normalizeMailbox(reference + path, !this.acceptUTF8Enabled); let logdata = { short_message: '[LSUB]', @@ -55,14 +54,14 @@ module.exports = { }); } - imapTools.filterFolders(imapTools.generateFolderListing(list, true), query).forEach(folder => { + filterFolders(generateFolderListing(list, true), query).forEach(folder => { if (!folder) { return; } let path = folder.path; if (!this.acceptUTF8Enabled) { - path = utf7.encode(path); + path = utf7encode(path); } else { path = Buffer.from(path); } @@ -98,6 +97,6 @@ module.exports = { // Do folder listing // Concat reference and mailbox. No special reference handling whatsoever - this._server.onLsub(imapTools.normalizeMailbox(reference + path), this.session, lsubResponse); + this._server.onLsub(normalizeMailbox(reference + path), this.session, lsubResponse); } }; diff --git a/imap-core/lib/imap-tools.js b/imap-core/lib/imap-tools.js index 58e2124b..a1e48b2c 100644 --- a/imap-core/lib/imap-tools.js +++ b/imap-core/lib/imap-tools.js @@ -1,13 +1,19 @@ 'use strict'; const Indexer = require('./indexer/indexer'); -const utf7 = require('utf7').imap; const libmime = require('libmime'); const punycode = require('punycode'); +const iconv = require('iconv-lite'); module.exports.systemFlagsFormatted = ['\\Answered', '\\Flagged', '\\Draft', '\\Deleted', '\\Seen']; module.exports.systemFlags = ['\\answered', '\\flagged', '\\draft', '\\deleted', '\\seen']; +const utf7encode = str => iconv.encode(str, 'utf-7-imap').toString(); +const utf7decode = str => iconv.decode(Buffer.from(str), 'utf-7-imap').toString(); + +module.exports.utf7encode = utf7encode; +module.exports.utf7decode = utf7decode; + module.exports.fetchSchema = { body: [ true, @@ -195,11 +201,11 @@ module.exports.searchMapping = { * @param {range} range Sequence range, eg "1,2,3:7" * @returns {Boolean} True if the string looks like a sequence range */ -module.exports.validateSequnce = function(range) { +module.exports.validateSequnce = function (range) { return !!(range.length && /^(\d+|\*)(:\d+|:\*)?(,(\d+|\*)(:\d+|:\*)?)*$/.test(range)); }; -module.exports.normalizeMailbox = function(mailbox, utf7Encoded) { +module.exports.normalizeMailbox = function (mailbox, utf7Encoded) { if (!mailbox) { return ''; } @@ -214,7 +220,7 @@ module.exports.normalizeMailbox = function(mailbox, utf7Encoded) { } if (utf7Encoded) { - parts = parts.map(value => utf7.decode(value)); + parts = parts.map(value => utf7decode(value)); } mailbox = parts.join('/'); @@ -222,7 +228,7 @@ module.exports.normalizeMailbox = function(mailbox, utf7Encoded) { return mailbox; }; -module.exports.generateFolderListing = function(folders, skipHierarchy) { +module.exports.generateFolderListing = function (folders, skipHierarchy) { let items = new Map(); let parents = []; @@ -328,7 +334,7 @@ module.exports.generateFolderListing = function(folders, skipHierarchy) { return result; }; -module.exports.filterFolders = function(folders, query) { +module.exports.filterFolders = function (folders, query) { query = query // remove excess * and % .replace(/\*\*+/g, '*') @@ -345,7 +351,7 @@ module.exports.filterFolders = function(folders, query) { return folders.filter(folder => !!regex.test(folder.path)); }; -module.exports.getMessageRange = function(uidList, range, isUid) { +module.exports.getMessageRange = function (uidList, range, isUid) { range = (range || '').toString(); let result = []; @@ -390,7 +396,7 @@ module.exports.getMessageRange = function(uidList, range, isUid) { return result; }; -module.exports.packMessageRange = function(uidList) { +module.exports.packMessageRange = function (uidList) { if (!Array.isArray(uidList)) { uidList = [].concat(uidList || []); } @@ -427,7 +433,7 @@ module.exports.packMessageRange = function(uidList) { * @param {Date} date Date object to parse * @returns {String} Internaldate formatted date */ -module.exports.formatInternalDate = function(date) { +module.exports.formatInternalDate = function (date) { let day = date.getUTCDate(), month = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'][date.getUTCMonth()], year = date.getUTCFullYear(), @@ -485,7 +491,7 @@ module.exports.formatInternalDate = function(date) { * @param {Object} options Options for the indexer * @returns {Array} Resolved responses */ -module.exports.getQueryResponse = function(query, message, options) { +module.exports.getQueryResponse = function (query, message, options) { options = options || {}; // for optimization purposes try to use cached mimeTree etc. if available diff --git a/imap-core/test/client.js b/imap-core/test/client.js index 04eaeb0d..2c466b85 100644 --- a/imap-core/test/client.js +++ b/imap-core/test/client.js @@ -5,52 +5,42 @@ process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; const config = require('wild-config'); -const BrowserBox = require('browserbox'); +const { ImapFlow } = require('imapflow'); -const client = new BrowserBox('localhost', config.imap.port, { - useSecureTransport: config.imap.secure, +const client = new ImapFlow({ + host: '127.0.0.1', + port: config.imap.port, + secure: config.imap.secure, auth: { user: 'testuser', pass: 'secretpass' }, - id: { - name: 'My Client', - version: '0.1' - }, tls: { rejectUnauthorized: false + }, + clientInfo: { + name: 'My Client', + version: '0.1' } }); -client.onerror = function(err) { +client.on('error', err => { console.log(err); process.exit(1); -}; +}); -client.onauth = function() { - client.upload('INBOX', 'from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz\r\n', false, err => { - if (err) { - console.log(err); - return process.exit(1); - } +const raw = Buffer.from('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz\r\n'); - client.selectMailbox('INBOX', (err, mailbox) => { - if (err) { - console.log(err); - return process.exit(1); - } - console.log(mailbox); - - client.listMessages(mailbox.exists, ['BODY.PEEK[]', 'BODYSTRUCTURE'], (err, data) => { - if (err) { - console.log(err); - return process.exit(1); - } - console.log('<<<%s>>>', data[0]['body[]']); - return process.exit(0); - }); - }); +client + .connect() + .then(() => client.append('INBOX', raw)) + .then(() => client.mailboxOpen('INBOX')) + .then(mailbox => client.fetchOne(mailbox.exists, { bodyStructure: true, source: true })) + .then(data => { + console.log('<<<%s>>>', data.source.toString()); + return process.exit(0); + }) + .catch(err => { + console.log(err); + process.exit(1); }); -}; - -client.connect(); diff --git a/package.json b/package.json index 550f87ec..ff251d46 100644 --- a/package.json +++ b/package.json @@ -17,10 +17,9 @@ "devDependencies": { "ajv": "6.12.2", "apidoc": "0.22.1", - "browserbox": "0.9.1", "chai": "4.2.0", "docsify-cli": "4.4.0", - "eslint": "6.8.0", + "eslint": "7.0.0", "eslint-config-nodemailer": "1.2.0", "eslint-config-prettier": "6.11.0", "grunt": "1.1.0", @@ -29,6 +28,7 @@ "grunt-mocha-test": "0.13.3", "grunt-shell-spawn": "0.4.0", "grunt-wait": "0.3.0", + "imapflow": "1.0.46", "mailparser": "2.7.7", "mocha": "7.1.2", "request": "2.88.2", @@ -54,7 +54,7 @@ "libbase64": "1.2.1", "libmime": "4.2.1", "libqp": "1.1.0", - "mailsplit": "4.6.4", + "mailsplit": "5.0.0", "mobileconfig": "2.3.1", "mongo-cursor-pagination": "7.3.0", "mongodb": "3.5.7", @@ -68,11 +68,11 @@ "qrcode": "1.4.4", "restify": "8.5.1", "restify-logger": "2.0.1", + "saslprep": "1.0.3", "seq-index": "1.1.0", "smtp-server": "3.6.0", "speakeasy": "2.0.0", "u2f": "0.1.3", - "utf7": "1.0.2", "uuid": "8.0.0", "wild-config": "1.5.1", "yargs": "15.3.1" @@ -80,5 +80,8 @@ "repository": { "type": "git", "url": "git://github.com/wildduck-email/wildduck.git" + }, + "engines": { + "node": ">=10.0.0" } } diff --git a/test/filtering-test.js b/test/filtering-test.js index 8bc5e9a6..9bf697e7 100644 --- a/test/filtering-test.js +++ b/test/filtering-test.js @@ -10,9 +10,9 @@ const crypto = require('crypto'); const chai = require('chai'); const request = require('request'); const fs = require('fs'); -const BrowserBox = require('browserbox'); const simpleParser = require('mailparser').simpleParser; const nodemailer = require('nodemailer'); +const { ImapFlow } = require('imapflow'); const transporter = nodemailer.createTransport({ lmtp: true, @@ -340,59 +340,61 @@ describe('Send multiple messages', function () { crypto.createHash('md5').update(swanJpg).digest('hex') ]; - const client = new BrowserBox('localhost', 9993, { - useSecureTransport: true, + const client = new ImapFlow({ + host: '127.0.0.1', + port: 9993, + secure: true, auth: { user: 'user4', pass: 'secretpass' }, - id: { - name: 'My Client', - version: '0.1' - }, tls: { rejectUnauthorized: false + }, + clientInfo: { + name: 'My Client', + version: '0.1' } }); - client.onerror = err => { + client.on('error', err => { expect(err).to.not.exist; - }; + done(); + }); + client.on('close', () => done()); - client.onclose = done; - - client.onauth = () => { - client.listMailboxes((err, result) => { - expect(err).to.not.exist; - let folders = result.children.map(mbox => ({ name: mbox.name, specialUse: mbox.specialUse || false })); + client + .connect() + .then(async () => { + const result = await client.list(); + const folders = result.map(mbox => ({ name: mbox.name, specialUse: mbox.specialUse || false })).sort((a, b) => a.name.localeCompare(b.name)); expect(folders).to.deep.equal([ - { name: 'INBOX', specialUse: false }, { name: 'Drafts', specialUse: '\\Drafts' }, + { name: 'INBOX', specialUse: '\\Inbox' }, { name: 'Junk', specialUse: '\\Junk' }, { name: 'Sent Mail', specialUse: '\\Sent' }, { name: 'Trash', specialUse: '\\Trash' } ]); - client.selectMailbox('INBOX', { condstore: true }, (err, result) => { - expect(err).to.not.exist; - expect(result.exists).gte(1); - client.listMessages(result.exists, ['uid', 'flags', 'body.peek[]'], (err, messages) => { - expect(err).to.not.exist; - expect(messages.length).equal(1); + const mailbox = await client.mailboxOpen('INBOX'); + expect(mailbox.exists).gte(1); - let messageInfo = messages[0]; - simpleParser(messageInfo['body[]'], (err, parsed) => { - expect(err).to.not.exist; - checksums.forEach((checksum, i) => { - expect(checksum).to.equal(parsed.attachments[i].checksum); - }); - client.close(); - }); - }); + let messages = []; + for await (let msg of client.fetch(mailbox.exists, { uid: true, source: true })) { + messages.push(msg); + } + expect(messages.length).equal(1); + + let messageInfo = messages[0]; + let parsed = await simpleParser(messageInfo.source); + checksums.forEach((checksum, i) => { + expect(checksum).to.equal(parsed.attachments[i].checksum); }); + client.close(); + }) + .catch(err => { + expect(err).to.not.exist; + client.close(); }); - }; - - client.connect(); }); });