replace node_redis with ioredis

This commit is contained in:
Andris Reinman 2017-10-03 11:18:23 +03:00
parent a7ec6f9158
commit 47f8307d9b
10 changed files with 97 additions and 133 deletions

View file

@ -337,7 +337,7 @@ Forces closing all active IMAP session of an user
```
curl -XPUT "http://localhost:8080/users/59467f27535f8f0f067ba8e6/logout" -H 'content-type: application/json' -d '{
"reaosn": "Account was deleted"
"reason": "Account was deleted"
}'
```

View file

@ -11,7 +11,7 @@ const UserHandler = require('./lib/user-handler');
const MailboxHandler = require('./lib/mailbox-handler');
const db = require('./lib/db');
const consts = require('./lib/consts');
const RedFour = require('redfour');
const RedFour = require('ioredfour');
const packageData = require('./package.json');
const yaml = require('js-yaml');
const fs = require('fs');
@ -224,7 +224,7 @@ module.exports = done => {
}
gcLock = new RedFour({
redis: db.redisConfig,
redis: db.redis,
namespace: 'wildduck'
});

View file

@ -1,97 +1,40 @@
'use strict';
const Scripty = require('node-redis-scripty');
const ttlCounterScript = `
local key = KEYS[1];
local increment = tonumber(ARGV[1]) or 0;
local limit = tonumber(ARGV[2]) or 0;
local windowSize = tonumber(ARGV[3]) or 0;
local current = tonumber(redis.call("GET", key)) or 0;
if current >= limit then
local ttl = tonumber(redis.call("TTL", key)) or 0;
return {0, current, ttl};
end;
local updated;
local ttl;
if increment > 0 then
-- increment
updated = tonumber(redis.call("INCRBY", key, increment));
if current == 0 then
redis.call("EXPIRE", key, windowSize);
end;
ttl = tonumber(redis.call("TTL", key)) or 0;
else
-- return current
updated = current;
ttl = tonumber(redis.call("TTL", key)) or windowSize;
end;
return {1, updated, ttl};
`;
const cachedCounterScript = `
local key = KEYS[1];
local increment = tonumber(ARGV[1]) or 0;
local ttl = tonumber(ARGV[2]) or 0;
if redis.call("EXISTS", key) == 1 then
redis.call("INCRBY", key, increment);
local sum = tonumber(redis.call("GET", key)) or 0;
-- extend the life of this counter by ttl seconds
redis.call("EXPIRE", key, ttl);
return sum;
else
return nil;
end
`;
const fs = require('fs');
const ttlCounterScript = fs.readFileSync(__dirname + '/lua/ttlcounter.lua', 'utf-8');
const cachedCounterScript = fs.readFileSync(__dirname + '/lua/cachedcounter.lua', 'utf-8');
module.exports = redis => {
let scripty = new Scripty(redis);
redis.defineCommand('ttlcounter', {
numberOfKeys: 1,
lua: ttlCounterScript
});
redis.defineCommand('cachedcounter', {
numberOfKeys: 1,
lua: cachedCounterScript
});
return {
ttlcounter(key, count, max, windowSize, callback) {
scripty.loadScript('ttlcounter', ttlCounterScript, (err, script) => {
redis.ttlcounter(key, count, max, windowSize || 86400, (err, res) => {
if (err) {
return callback(err);
}
script.run(1, key, count, max, windowSize || 86400, (err, res) => {
if (err) {
return callback(err);
}
return callback(null, {
success: !!((res && res[0]) || 0),
value: (res && res[1]) || 0,
ttl: (res && res[2]) || 0
});
return callback(null, {
success: !!((res && res[0]) || 0),
value: (res && res[1]) || 0,
ttl: (res && res[2]) || 0
});
});
},
cachedcounter(key, count, ttl, callback) {
scripty.loadScript('cachedCounter', cachedCounterScript, (err, script) => {
redis.cachedcounter(key, count, ttl, (err, res) => {
if (err) {
return callback(err);
}
script.run(
1,
key,
count,
ttl,
(
err,
res => {
if (err) {
return callback(err);
}
callback(null, res);
}
)
);
callback(null, res);
});
}
};

View file

@ -3,7 +3,7 @@
const config = require('wild-config');
const tools = require('./tools');
const mongodb = require('mongodb');
const redis = require('redis');
const Redis = require('ioredis');
const MongoClient = mongodb.MongoClient;
module.exports.database = false;
@ -53,7 +53,7 @@ module.exports.connect = callback => {
module.exports.senderDb = db;
module.exports.redisConfig = tools.redisConfig(config.dbs.redis);
module.exports.redis = redis.createClient(module.exports.redisConfig);
module.exports.redis = new Redis(module.exports.redisConfig);
return callback(null, module.exports.database);
});

View file

@ -5,7 +5,7 @@ const tools = require('./tools');
const consts = require('./consts');
const crypto = require('crypto');
const EventEmitter = require('events').EventEmitter;
const redis = require('redis');
const Redis = require('ioredis');
const log = require('npmlog');
const counters = require('./counters');
@ -14,7 +14,7 @@ class ImapNotifier extends EventEmitter {
super();
this.database = options.database;
this.publisher = options.redis || redis.createClient(tools.redisConfig(config.dbs.redis));
this.publisher = options.redis || new Redis(tools.redisConfig(config.dbs.redis));
this.cachedcounter = counters(this.publisher).cachedcounter;
this.logger = options.logger || {
@ -29,7 +29,7 @@ class ImapNotifier extends EventEmitter {
}
// Subscriber needs its own client connection. This is relevant only in the context of IMAP
this.subsriber = redis.createClient(tools.redisConfig(config.dbs.redis));
this.subsriber = new Redis(tools.redisConfig(config.dbs.redis));
this._listeners = new EventEmitter();
this._listeners.setMaxListeners(0);
@ -98,7 +98,10 @@ class ImapNotifier extends EventEmitter {
*/
_eventName(user, path) {
if (path.length >= 32) {
path = crypto.createHash('md5').update(path).digest('hex');
path = crypto
.createHash('md5')
.update(path)
.digest('hex');
}
return user + ':' + path;
}

View file

@ -9,7 +9,7 @@ const tlsOptions = require('../../imap-core/lib/tls-options');
const shared = require('nodemailer/lib/shared');
const IRCConnection = require('./connection');
const tools = require('../tools');
const redis = require('redis');
const Redis = require('ioredis');
const CLOSE_TIMEOUT = 1 * 1000; // how much to wait until pending connections are terminated
@ -67,8 +67,8 @@ class IRCServer extends EventEmitter {
this.server = (this.options.secure ? tls : net).createServer(this.options, socket => this._onConnect(socket));
this.publisher = redis.createClient(tools.redisConfig(config.dbs.redis));
this.subscriber = redis.createClient(tools.redisConfig(config.dbs.redis));
this.publisher = new Redis(tools.redisConfig(config.dbs.redis));
this.subscriber = new Redis(tools.redisConfig(config.dbs.redis));
this.subscribers = new Map();
this._listeners = new EventEmitter();

13
lib/lua/cachedcounter.lua Normal file
View file

@ -0,0 +1,13 @@
local key = KEYS[1];
local increment = tonumber(ARGV[1]) or 0;
local ttl = tonumber(ARGV[2]) or 0;
if redis.call("EXISTS", key) == 1 then
redis.call("INCRBY", key, increment);
local sum = tonumber(redis.call("GET", key)) or 0;
-- extend the life of this counter by ttl seconds
redis.call("EXPIRE", key, ttl);
return sum;
else
return nil;
end

28
lib/lua/ttlcounter.lua Normal file
View file

@ -0,0 +1,28 @@
local key = KEYS[1];
local increment = tonumber(ARGV[1]) or 0;
local limit = tonumber(ARGV[2]) or 0;
local windowSize = tonumber(ARGV[3]) or 0;
local current = tonumber(redis.call("GET", key)) or 0;
if current >= limit then
local ttl = tonumber(redis.call("TTL", key)) or 0;
return {0, current, ttl};
end;
local updated;
local ttl;
if increment > 0 then
-- increment
updated = tonumber(redis.call("INCRBY", key, increment));
if current == 0 then
redis.call("EXPIRE", key, windowSize);
end;
ttl = tonumber(redis.call("TTL", key)) or 0;
else
-- return current
updated = current;
ttl = tonumber(redis.call("TTL", key)) or windowSize;
end;
return {1, updated, ttl};

View file

@ -53,8 +53,15 @@ function normalizeAddress(address, withNames) {
if (!address || !address.address) {
return '';
}
let user = address.address.substr(0, address.address.lastIndexOf('@')).normalize('NFC').toLowerCase().trim();
let domain = address.address.substr(address.address.lastIndexOf('@') + 1).toLowerCase().trim();
let user = address.address
.substr(0, address.address.lastIndexOf('@'))
.normalize('NFC')
.toLowerCase()
.trim();
let domain = address.address
.substr(address.address.lastIndexOf('@') + 1)
.toLowerCase()
.trim();
let encodedDomain = domain;
try {
encodedDomain = punycode.toUnicode(domain);
@ -76,40 +83,7 @@ function normalizeAddress(address, withNames) {
// returns a redis config object with a retry strategy
function redisConfig(defaultConfig) {
let response = {};
if (typeof defaultConfig === 'string') {
defaultConfig = {
url: defaultConfig
};
}
Object.keys(defaultConfig || {}).forEach(key => {
response[key] = defaultConfig[key];
});
if (!response.hasOwnProperty('retry_strategy')) {
response.retry_strategy = options => {
if (options.error && options.error.code === 'ECONNREFUSED') {
// End reconnecting on a specific error and flush all commands with a individual error
return new Error('The server refused the connection');
}
if (options.total_retry_time > 1000 * 60 * 60) {
// End reconnecting after a specific timeout and flush all commands with a individual error
return new Error('Retry time exhausted');
}
if (options.attempt > 10) {
// End reconnecting with built in error
return undefined; // eslint-disable-line no-undefined
}
// reconnect after
return Math.min(options.attempt * 100, 3000);
};
}
return response;
return defaultConfig;
}
function decodeAddresses(addresses) {
@ -156,9 +130,13 @@ function getMailboxCounter(db, mailbox, type, done) {
}
// cache calculated sum in redis
db.redis.multi().set(prefix + ':' + mailbox.toString(), sum).expire(prefix + ':' + mailbox.toString(), consts.MAILBOX_COUNTER_TTL).exec(() => {
done(null, sum);
});
db.redis
.multi()
.set(prefix + ':' + mailbox.toString(), sum)
.expire(prefix + ':' + mailbox.toString(), consts.MAILBOX_COUNTER_TTL)
.exec(() => {
done(null, sum);
});
});
});
}

View file

@ -16,22 +16,24 @@
"grunt": "^1.0.1",
"grunt-cli": "^1.2.0",
"grunt-eslint": "^20.1.0",
"grunt-mocha-test": "^0.13.2",
"grunt-mocha-test": "^0.13.3",
"grunt-shell-spawn": "^0.3.10",
"grunt-wait": "^0.1.0",
"icedfrisby": "^1.3.1",
"mocha": "^3.5.3"
"mocha": "^4.0.0"
},
"dependencies": {
"addressparser": "1.0.1",
"bcryptjs": "2.4.3",
"bugsnag": "1.12.2",
"bugsnag": "2.0.0",
"generate-password": "1.3.0",
"he": "1.1.1",
"html-to-text": "3.3.0",
"humanname": "0.2.2",
"humanparser": "1.5.0",
"iconv-lite": "0.4.19",
"ioredfour": "1.0.2-ioredis",
"ioredis": "3.1.4",
"joi": "11.1.1",
"js-yaml": "3.10.0",
"libbase64": "0.2.0",
@ -42,13 +44,10 @@
"mobileconfig": "2.1.0",
"mongo-cursor-pagination": "5.0.0",
"mongodb": "2.2.31",
"node-redis-scripty": "0.0.5",
"nodemailer": "4.1.1",
"npmlog": "4.1.2",
"openpgp": "2.5.11",
"qrcode": "0.9.0",
"redfour": "1.0.2",
"redis": "2.8.0",
"restify": "6.0.1",
"seq-index": "1.1.0",
"smtp-server": "3.2.0",