diff --git a/docs/api.md b/docs/api.md index f5294f10..01ad98c7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -337,7 +337,7 @@ Forces closing all active IMAP session of an user ``` curl -XPUT "http://localhost:8080/users/59467f27535f8f0f067ba8e6/logout" -H 'content-type: application/json' -d '{ - "reaosn": "Account was deleted" + "reason": "Account was deleted" }' ``` diff --git a/imap.js b/imap.js index 41ebf880..968512e3 100644 --- a/imap.js +++ b/imap.js @@ -11,7 +11,7 @@ const UserHandler = require('./lib/user-handler'); const MailboxHandler = require('./lib/mailbox-handler'); const db = require('./lib/db'); const consts = require('./lib/consts'); -const RedFour = require('redfour'); +const RedFour = require('ioredfour'); const packageData = require('./package.json'); const yaml = require('js-yaml'); const fs = require('fs'); @@ -224,7 +224,7 @@ module.exports = done => { } gcLock = new RedFour({ - redis: db.redisConfig, + redis: db.redis, namespace: 'wildduck' }); diff --git a/lib/counters.js b/lib/counters.js index cc10fcc3..54ea8577 100644 --- a/lib/counters.js +++ b/lib/counters.js @@ -1,97 +1,40 @@ 'use strict'; -const Scripty = require('node-redis-scripty'); - -const ttlCounterScript = ` -local key = KEYS[1]; -local increment = tonumber(ARGV[1]) or 0; -local limit = tonumber(ARGV[2]) or 0; -local windowSize = tonumber(ARGV[3]) or 0; -local current = tonumber(redis.call("GET", key)) or 0; - -if current >= limit then - local ttl = tonumber(redis.call("TTL", key)) or 0; - return {0, current, ttl}; -end; - -local updated; -local ttl; - -if increment > 0 then - -- increment - updated = tonumber(redis.call("INCRBY", key, increment)); - if current == 0 then - redis.call("EXPIRE", key, windowSize); - end; - ttl = tonumber(redis.call("TTL", key)) or 0; -else - -- return current - updated = current; - ttl = tonumber(redis.call("TTL", key)) or windowSize; -end; - -return {1, updated, ttl}; -`; - -const cachedCounterScript = ` -local key = KEYS[1]; -local increment = tonumber(ARGV[1]) or 0; -local ttl = tonumber(ARGV[2]) or 0; - -if redis.call("EXISTS", key) == 1 then - redis.call("INCRBY", key, increment); - local sum = tonumber(redis.call("GET", key)) or 0; - -- extend the life of this counter by ttl seconds - redis.call("EXPIRE", key, ttl); - return sum; -else - return nil; -end -`; +const fs = require('fs'); +const ttlCounterScript = fs.readFileSync(__dirname + '/lua/ttlcounter.lua', 'utf-8'); +const cachedCounterScript = fs.readFileSync(__dirname + '/lua/cachedcounter.lua', 'utf-8'); module.exports = redis => { - let scripty = new Scripty(redis); + redis.defineCommand('ttlcounter', { + numberOfKeys: 1, + lua: ttlCounterScript + }); + + redis.defineCommand('cachedcounter', { + numberOfKeys: 1, + lua: cachedCounterScript + }); return { ttlcounter(key, count, max, windowSize, callback) { - scripty.loadScript('ttlcounter', ttlCounterScript, (err, script) => { + redis.ttlcounter(key, count, max, windowSize || 86400, (err, res) => { if (err) { return callback(err); } - script.run(1, key, count, max, windowSize || 86400, (err, res) => { - if (err) { - return callback(err); - } - return callback(null, { - success: !!((res && res[0]) || 0), - value: (res && res[1]) || 0, - ttl: (res && res[2]) || 0 - }); + return callback(null, { + success: !!((res && res[0]) || 0), + value: (res && res[1]) || 0, + ttl: (res && res[2]) || 0 }); }); }, cachedcounter(key, count, ttl, callback) { - scripty.loadScript('cachedCounter', cachedCounterScript, (err, script) => { + redis.cachedcounter(key, count, ttl, (err, res) => { if (err) { return callback(err); } - - script.run( - 1, - key, - count, - ttl, - ( - err, - res => { - if (err) { - return callback(err); - } - callback(null, res); - } - ) - ); + callback(null, res); }); } }; diff --git a/lib/db.js b/lib/db.js index af11b300..50504c39 100644 --- a/lib/db.js +++ b/lib/db.js @@ -3,7 +3,7 @@ const config = require('wild-config'); const tools = require('./tools'); const mongodb = require('mongodb'); -const redis = require('redis'); +const Redis = require('ioredis'); const MongoClient = mongodb.MongoClient; module.exports.database = false; @@ -53,7 +53,7 @@ module.exports.connect = callback => { module.exports.senderDb = db; module.exports.redisConfig = tools.redisConfig(config.dbs.redis); - module.exports.redis = redis.createClient(module.exports.redisConfig); + module.exports.redis = new Redis(module.exports.redisConfig); return callback(null, module.exports.database); }); diff --git a/lib/imap-notifier.js b/lib/imap-notifier.js index 5f43aa49..06f001fa 100644 --- a/lib/imap-notifier.js +++ b/lib/imap-notifier.js @@ -5,7 +5,7 @@ const tools = require('./tools'); const consts = require('./consts'); const crypto = require('crypto'); const EventEmitter = require('events').EventEmitter; -const redis = require('redis'); +const Redis = require('ioredis'); const log = require('npmlog'); const counters = require('./counters'); @@ -14,7 +14,7 @@ class ImapNotifier extends EventEmitter { super(); this.database = options.database; - this.publisher = options.redis || redis.createClient(tools.redisConfig(config.dbs.redis)); + this.publisher = options.redis || new Redis(tools.redisConfig(config.dbs.redis)); this.cachedcounter = counters(this.publisher).cachedcounter; this.logger = options.logger || { @@ -29,7 +29,7 @@ class ImapNotifier extends EventEmitter { } // Subscriber needs its own client connection. This is relevant only in the context of IMAP - this.subsriber = redis.createClient(tools.redisConfig(config.dbs.redis)); + this.subsriber = new Redis(tools.redisConfig(config.dbs.redis)); this._listeners = new EventEmitter(); this._listeners.setMaxListeners(0); @@ -98,7 +98,10 @@ class ImapNotifier extends EventEmitter { */ _eventName(user, path) { if (path.length >= 32) { - path = crypto.createHash('md5').update(path).digest('hex'); + path = crypto + .createHash('md5') + .update(path) + .digest('hex'); } return user + ':' + path; } diff --git a/lib/irc/server.js b/lib/irc/server.js index 2d96b6e1..535afffb 100644 --- a/lib/irc/server.js +++ b/lib/irc/server.js @@ -9,7 +9,7 @@ const tlsOptions = require('../../imap-core/lib/tls-options'); const shared = require('nodemailer/lib/shared'); const IRCConnection = require('./connection'); const tools = require('../tools'); -const redis = require('redis'); +const Redis = require('ioredis'); const CLOSE_TIMEOUT = 1 * 1000; // how much to wait until pending connections are terminated @@ -67,8 +67,8 @@ class IRCServer extends EventEmitter { this.server = (this.options.secure ? tls : net).createServer(this.options, socket => this._onConnect(socket)); - this.publisher = redis.createClient(tools.redisConfig(config.dbs.redis)); - this.subscriber = redis.createClient(tools.redisConfig(config.dbs.redis)); + this.publisher = new Redis(tools.redisConfig(config.dbs.redis)); + this.subscriber = new Redis(tools.redisConfig(config.dbs.redis)); this.subscribers = new Map(); this._listeners = new EventEmitter(); diff --git a/lib/lua/cachedcounter.lua b/lib/lua/cachedcounter.lua new file mode 100644 index 00000000..82e4d48d --- /dev/null +++ b/lib/lua/cachedcounter.lua @@ -0,0 +1,13 @@ +local key = KEYS[1]; +local increment = tonumber(ARGV[1]) or 0; +local ttl = tonumber(ARGV[2]) or 0; + +if redis.call("EXISTS", key) == 1 then + redis.call("INCRBY", key, increment); + local sum = tonumber(redis.call("GET", key)) or 0; + -- extend the life of this counter by ttl seconds + redis.call("EXPIRE", key, ttl); + return sum; +else + return nil; +end diff --git a/lib/lua/ttlcounter.lua b/lib/lua/ttlcounter.lua new file mode 100644 index 00000000..d55deb57 --- /dev/null +++ b/lib/lua/ttlcounter.lua @@ -0,0 +1,28 @@ +local key = KEYS[1]; +local increment = tonumber(ARGV[1]) or 0; +local limit = tonumber(ARGV[2]) or 0; +local windowSize = tonumber(ARGV[3]) or 0; +local current = tonumber(redis.call("GET", key)) or 0; + +if current >= limit then + local ttl = tonumber(redis.call("TTL", key)) or 0; + return {0, current, ttl}; +end; + +local updated; +local ttl; + +if increment > 0 then + -- increment + updated = tonumber(redis.call("INCRBY", key, increment)); + if current == 0 then + redis.call("EXPIRE", key, windowSize); + end; + ttl = tonumber(redis.call("TTL", key)) or 0; +else + -- return current + updated = current; + ttl = tonumber(redis.call("TTL", key)) or windowSize; +end; + +return {1, updated, ttl}; diff --git a/lib/tools.js b/lib/tools.js index 49fe14fb..4b21f9a8 100644 --- a/lib/tools.js +++ b/lib/tools.js @@ -53,8 +53,15 @@ function normalizeAddress(address, withNames) { if (!address || !address.address) { return ''; } - let user = address.address.substr(0, address.address.lastIndexOf('@')).normalize('NFC').toLowerCase().trim(); - let domain = address.address.substr(address.address.lastIndexOf('@') + 1).toLowerCase().trim(); + let user = address.address + .substr(0, address.address.lastIndexOf('@')) + .normalize('NFC') + .toLowerCase() + .trim(); + let domain = address.address + .substr(address.address.lastIndexOf('@') + 1) + .toLowerCase() + .trim(); let encodedDomain = domain; try { encodedDomain = punycode.toUnicode(domain); @@ -76,40 +83,7 @@ function normalizeAddress(address, withNames) { // returns a redis config object with a retry strategy function redisConfig(defaultConfig) { - let response = {}; - - if (typeof defaultConfig === 'string') { - defaultConfig = { - url: defaultConfig - }; - } - - Object.keys(defaultConfig || {}).forEach(key => { - response[key] = defaultConfig[key]; - }); - if (!response.hasOwnProperty('retry_strategy')) { - response.retry_strategy = options => { - if (options.error && options.error.code === 'ECONNREFUSED') { - // End reconnecting on a specific error and flush all commands with a individual error - return new Error('The server refused the connection'); - } - - if (options.total_retry_time > 1000 * 60 * 60) { - // End reconnecting after a specific timeout and flush all commands with a individual error - return new Error('Retry time exhausted'); - } - - if (options.attempt > 10) { - // End reconnecting with built in error - return undefined; // eslint-disable-line no-undefined - } - - // reconnect after - return Math.min(options.attempt * 100, 3000); - }; - } - - return response; + return defaultConfig; } function decodeAddresses(addresses) { @@ -156,9 +130,13 @@ function getMailboxCounter(db, mailbox, type, done) { } // cache calculated sum in redis - db.redis.multi().set(prefix + ':' + mailbox.toString(), sum).expire(prefix + ':' + mailbox.toString(), consts.MAILBOX_COUNTER_TTL).exec(() => { - done(null, sum); - }); + db.redis + .multi() + .set(prefix + ':' + mailbox.toString(), sum) + .expire(prefix + ':' + mailbox.toString(), consts.MAILBOX_COUNTER_TTL) + .exec(() => { + done(null, sum); + }); }); }); } diff --git a/package.json b/package.json index aef14938..e8057c9a 100644 --- a/package.json +++ b/package.json @@ -16,22 +16,24 @@ "grunt": "^1.0.1", "grunt-cli": "^1.2.0", "grunt-eslint": "^20.1.0", - "grunt-mocha-test": "^0.13.2", + "grunt-mocha-test": "^0.13.3", "grunt-shell-spawn": "^0.3.10", "grunt-wait": "^0.1.0", "icedfrisby": "^1.3.1", - "mocha": "^3.5.3" + "mocha": "^4.0.0" }, "dependencies": { "addressparser": "1.0.1", "bcryptjs": "2.4.3", - "bugsnag": "1.12.2", + "bugsnag": "2.0.0", "generate-password": "1.3.0", "he": "1.1.1", "html-to-text": "3.3.0", "humanname": "0.2.2", "humanparser": "1.5.0", "iconv-lite": "0.4.19", + "ioredfour": "1.0.2-ioredis", + "ioredis": "3.1.4", "joi": "11.1.1", "js-yaml": "3.10.0", "libbase64": "0.2.0", @@ -42,13 +44,10 @@ "mobileconfig": "2.1.0", "mongo-cursor-pagination": "5.0.0", "mongodb": "2.2.31", - "node-redis-scripty": "0.0.5", "nodemailer": "4.1.1", "npmlog": "4.1.2", "openpgp": "2.5.11", "qrcode": "0.9.0", - "redfour": "1.0.2", - "redis": "2.8.0", "restify": "6.0.1", "seq-index": "1.1.0", "smtp-server": "3.2.0",