From f8b8dfb87f381380f78055064b54624a724b4adb Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 16:40:48 -0700 Subject: [PATCH 1/7] Revert to running pm2 in no-daemon mode in dev - Add a stream for pretty logging in dev mode --- package.json | 4 ++-- packages/nylas-core/logger.js | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index f95dacb81..625d7d2a7 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "devDependencies": { "babel-eslint": "6.x", + "bunyan-prettystream": "^0.1.3", "eslint": "2.x", "eslint-config-airbnb": "8.x", "eslint-plugin-import": "1.x", @@ -27,8 +28,7 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "pm2 kill && pm2 start ./pm2-dev.yml --watch && pm2 logs --raw | bunyan -o short", - "logs": "pm2 logs --raw | bunyan -o short", + "start": "pm2 start ./pm2-dev.yml --no-daemon", "stop": "pm2 kill", "postinstall": "lerna bootstrap" }, diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js index 40c73ad31..1bdcb9002 100644 --- a/packages/nylas-core/logger.js +++ b/packages/nylas-core/logger.js @@ -1,17 +1,25 @@ const bunyan = require('bunyan') const createCWStream = require('bunyan-cloudwatch') +const PrettyStream = require('bunyan-prettystream'); const NODE_ENV = process.env.NODE_ENV || 'unknown' function getLogStreams(name, env) { + if (env === 'development') { + const prettyStdOut = new PrettyStream(); + prettyStdOut.pipe(process.stdout); + const stdoutStream = { + type: 'raw', + level: 'debug', + stream: prettyStdOut, + } + return [stdoutStream] + } + const stdoutStream = { stream: process.stdout, level: 'info', } - if (env === 'development') { - return [stdoutStream] - } - const cloudwatchStream = { stream: createCWStream({ logGroupName: `k2-${env}`, From 492ac21bb6874bc2a61485a2bd7b605519ee8463 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 16:56:18 -0700 Subject: [PATCH 2/7] Primitive account deletion via DELETE /account --- packages/nylas-api/routes/accounts.js | 25 +++++++++++++++++++++ packages/nylas-core/database-connector.js | 18 ++++++++++----- packages/nylas-core/hook-account-crud.js | 10 ++++++--- packages/nylas-core/imap-connection.js | 6 +++-- packages/nylas-core/message-types.js | 1 + packages/nylas-sync/sync-process-manager.js | 1 + packages/nylas-sync/sync-worker.js | 19 +++++++++++----- 7 files changed, 64 insertions(+), 16 deletions(-) diff --git a/packages/nylas-api/routes/accounts.js b/packages/nylas-api/routes/accounts.js index 6f0ca9d61..856bd5a66 100644 --- a/packages/nylas-api/routes/accounts.js +++ b/packages/nylas-api/routes/accounts.js @@ -1,4 +1,5 @@ const Serialization = require('../serialization'); +const {DatabaseConnector} = require('nylas-core'); module.exports = (server) => { server.route({ @@ -21,4 +22,28 @@ module.exports = (server) => { reply(Serialization.jsonStringify(account)); }, }); + + server.route({ + method: 'DELETE', + path: '/account', + config: { + description: 'Deletes the current account and all data from the Nylas Cloud.', + notes: 'Notes go here', + tags: ['accounts'], + validate: { + params: { + }, + }, + }, + handler: (request, reply) => { + const account = request.auth.credentials; + account.destroy().then((saved) => + DatabaseConnector.destroyAccountDatabase(saved.id).then(() => + reply(Serialization.jsonStringify({status: 'success'})) + ) + ).catch((err) => { + reply(err).code(500); + }) + }, + }); }; diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index b37fe81cf..0f87e1fa0 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -79,11 +79,7 @@ class DatabaseConnector { const dbname = `a-${accountId}`; if (process.env.DB_HOSTNAME) { - const sequelize = new Sequelize(null, process.env.DB_USERNAME, process.env.DB_PASSWORD, { - host: process.env.DB_HOSTNAME, - dialect: "mysql", - logging: false, - }) + const sequelize = this._sequelizePoolForDatabase(null); return sequelize.authenticate().then(() => sequelize.query(`CREATE DATABASE \`${dbname}\``) ); @@ -91,6 +87,18 @@ class DatabaseConnector { return Promise.resolve() } + destroyAccountDatabase(accountId) { + const dbname = `a-${accountId}`; + if (process.env.DB_HOSTNAME) { + const sequelize = this._sequelizePoolForDatabase(null); + return sequelize.authenticate().then(() => + sequelize.query(`CREATE DATABASE \`${dbname}\``) + ); + } + fs.removeFileSync(path.join(STORAGE_DIR, `${dbname}.sqlite`)); + return Promise.resolve() + } + _sequelizeForShared() { const sequelize = this._sequelizePoolForDatabase(`shared`); const modelsPath = path.join(__dirname, 'models/shared'); diff --git a/packages/nylas-core/hook-account-crud.js b/packages/nylas-core/hook-account-crud.js index 016569b07..5efc658f6 100644 --- a/packages/nylas-core/hook-account-crud.js +++ b/packages/nylas-core/hook-account-crud.js @@ -17,7 +17,11 @@ module.exports = (db, sequelize) => { }); } }) - // TODO delete account from redis - // sequelize.addHook("afterDelete", ({dataValues, $modelOptions}) => { - // }) + sequelize.addHook("afterDestroy", ({dataValues, $modelOptions}) => { + if ($modelOptions.name.singular === 'account') { + PubsubConnector.notifyAccount(dataValues.id, { + type: MessageTypes.ACCOUNT_DELETED, + }); + } + }) } diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index c692f11bf..bbbef526f 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -270,9 +270,11 @@ class IMAPConnection extends EventEmitter { } end() { + if (this._imap) { + this._imap.end(); + this._imap = null; + } this._queue = []; - this._imap.end(); - this._imap = null; this._connectPromise = null; } diff --git a/packages/nylas-core/message-types.js b/packages/nylas-core/message-types.js index 2b4889a8c..43bbb3ba6 100644 --- a/packages/nylas-core/message-types.js +++ b/packages/nylas-core/message-types.js @@ -1,5 +1,6 @@ module.exports = { ACCOUNT_CREATED: "ACCOUNT_CREATED", ACCOUNT_UPDATED: "ACCOUNT_UPDATED", + ACCOUNT_DELETED: "ACCOUNT_DELETED", SYNCBACK_REQUESTED: "SYNCBACK_REQUESTED", } diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index 560bc1fad..2ca05e354 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -178,6 +178,7 @@ class SyncProcessManager { return; } this._logger.info({account_id: accountId}, `ProcessManager: Starting worker for Account`) + this._workers[account.id] = new SyncWorker(account, db, () => { this.removeWorkerForAccountId(accountId) }); diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 7178e59cf..943b9f986 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -27,7 +27,6 @@ class SyncWorker { this._logger = global.Logger.forAccount(account) this._syncTimer = null; - this._expirationTimer = null; this._destroyed = false; this.syncNow({reason: 'Initial'}); @@ -37,6 +36,8 @@ class SyncWorker { } cleanup() { + clearTimeout(this._syncTimer); + this._syncTimer = null; this._destroyed = true; this._listener.dispose(); this.closeConnection() @@ -51,13 +52,19 @@ class SyncWorker { _onMessage(msg) { const {type} = JSON.parse(msg); switch (type) { - case MessageTypes.ACCOUNT_UPDATED: - this._onAccountUpdated(); break; - case MessageTypes.SYNCBACK_REQUESTED: - this.syncNow({reason: 'Syncback Action Queued'}); break; case MessageTypes.ACCOUNT_CREATED: // No other processing currently required for account creation break; + case MessageTypes.ACCOUNT_UPDATED: + this._onAccountUpdated(); + break; + case MessageTypes.ACCOUNT_DELETED: + this.cleanup(); + this._onExpired(); + break; + case MessageTypes.SYNCBACK_REQUESTED: + this.syncNow({reason: 'Syncback Action Queued'}); + break; default: this._logger.error({message: msg}, 'SyncWorker: Invalid message') } @@ -208,7 +215,7 @@ class SyncWorker { const now = Date.now(); const syncGraphTimeLength = 60 * 30; // 30 minutes, should be the same as SyncGraph.config.timeLength - let lastSyncCompletions = [...this._account.lastSyncCompletions] + let lastSyncCompletions = [].concat(this._account.lastSyncCompletions) lastSyncCompletions = [now, ...lastSyncCompletions] while (now - lastSyncCompletions[lastSyncCompletions.length - 1] > 1000 * syncGraphTimeLength) { lastSyncCompletions.pop(); From 826db2e6f4516cdf169d894e1df378b3ea69566a Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 16:57:27 -0700 Subject: [PATCH 3/7] Remove redis from pm2 --- pm2-dev.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pm2-dev.yml b/pm2-dev.yml index b8efea594..21c1ea226 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -1,8 +1,4 @@ apps: - - script : redis-server - name : redis - - - script : packages/nylas-api/app.js watch : ["packages"] name : api From 039648bae84a9da8d40e0b2e79e150b074161cae Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 17:04:05 -0700 Subject: [PATCH 4/7] Add bunyan-prettystream to regular dependencies --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 625d7d2a7..4073a8ef4 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "bluebird": "3.x.x", "bunyan": "1.8.0", "bunyan-cloudwatch": "2.0.0", + "bunyan-prettystream": "^0.1.3", "lerna": "2.0.0-beta.23", "mysql": "^2.11.1", "newrelic": "^1.28.1", @@ -18,7 +19,6 @@ }, "devDependencies": { "babel-eslint": "6.x", - "bunyan-prettystream": "^0.1.3", "eslint": "2.x", "eslint-config-airbnb": "8.x", "eslint-plugin-import": "1.x", From 413a77c3232157673b7b0d27bda17adfa66a9c67 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 17:42:49 -0700 Subject: [PATCH 5/7] Log unhandled errors in api and sync --- packages/nylas-api/app.js | 4 ++++ packages/nylas-sync/app.js | 4 ++++ packages/nylas-sync/sync-process-manager.js | 14 +++++++------- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index dc9b08e08..8199a9687 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -14,6 +14,10 @@ const {DatabaseConnector, SchedulerUtils, Logger} = require(`nylas-core`); global.Promise = require('bluebird'); global.Logger = Logger.createLogger('nylas-k2-api') +const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error') +process.on('uncaughtException', onUnhandledError) +process.on('unhandledRejection', onUnhandledError) + const server = new Hapi.Server({ connections: { router: { diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index ac81a02f1..16bf4509f 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -5,6 +5,10 @@ const SyncProcessManager = require('./sync-process-manager'); global.Logger = Logger.createLogger('nylas-k2-sync') +const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error') +process.on('uncaughtException', onUnhandledError) +process.on('unhandledRejection', onUnhandledError) + const manager = new SyncProcessManager(); DatabaseConnector.forShared().then((db) => { diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index c32d2aa8b..3d205c384 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -61,13 +61,13 @@ class SyncProcessManager { updateHeartbeat() { const key = HEARTBEAT_FOR(IDENTITY); const client = PubsubConnector.broadcastClient(); - client.setAsync(key, Date.now()).then(() => - client.expireAsync(key, HEARTBEAT_EXPIRES) - ).then(() => - this._logger.info({ - accounts_syncing_count: Object.keys(this._workers).length, - }, "ProcessManager: 💘") - ) + client.setAsync(key, Date.now()) + .then(() => client.expireAsync(key, HEARTBEAT_EXPIRES)) + .then(() => { + this._logger.info({ + accounts_syncing_count: Object.keys(this._workers).length, + }, "ProcessManager: 💘") + }) } onSigInt() { From afeb0e7b27ad9aa1f5d0e9730b6eb2420ce4f188 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 17:57:43 -0700 Subject: [PATCH 6/7] Create one CloudWatch stream per host - This prevents `InvalidSequenceTokenException` when running in production with multiple machines in an elastic beanstalk environment - See https://github.com/mirkokiefer/bunyan-cloudwatch/issues/1#issuecomment-193116799 --- packages/nylas-core/logger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js index 1bdcb9002..b77ba2c29 100644 --- a/packages/nylas-core/logger.js +++ b/packages/nylas-core/logger.js @@ -1,3 +1,4 @@ +const os = require('os'); const bunyan = require('bunyan') const createCWStream = require('bunyan-cloudwatch') const PrettyStream = require('bunyan-prettystream'); @@ -23,7 +24,7 @@ function getLogStreams(name, env) { const cloudwatchStream = { stream: createCWStream({ logGroupName: `k2-${env}`, - logStreamName: `${name}-${env}`, + logStreamName: `${name}-${env}-${os.hostname()}`, cloudWatchLogsOptions: { region: 'us-east-1', }, From 73fc371825f2cbfbe7bb289306da308a9aa85de5 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 18:24:18 -0700 Subject: [PATCH 7/7] =?UTF-8?q?Make=20the=20graphs=20on=20the=20dashboard?= =?UTF-8?q?=20retina=20=F0=9F=92=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nylas-dashboard/public/js/sync-graph.jsx | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/packages/nylas-dashboard/public/js/sync-graph.jsx b/packages/nylas-dashboard/public/js/sync-graph.jsx index 7c1a1fc15..abde9a400 100644 --- a/packages/nylas-dashboard/public/js/sync-graph.jsx +++ b/packages/nylas-dashboard/public/js/sync-graph.jsx @@ -4,17 +4,36 @@ const ReactDOM = window.ReactDOM; class SyncGraph extends React.Component { componentDidMount() { - this.drawGraph(); - } - - componentDidUpdate() { this.drawGraph(true); } - drawGraph(isUpdate) { + componentDidUpdate() { + this.drawGraph(false); + } + + drawGraph(isInitial) { const now = Date.now(); const config = SyncGraph.config; - const context = ReactDOM.findDOMNode(this).getContext('2d'); + const node = ReactDOM.findDOMNode(this); + const context = node.getContext('2d'); + + if (isInitial) { + const totalHeight = config.height + config.labelFontSize + config.labelTopMargin; + node.width = config.width * 2; + node.height = totalHeight * 2; + node.style.width = `${config.width}px`; + node.style.height = `${totalHeight}px`; + context.scale(2, 2); + + // Axis labels + context.fillStyle = config.labelColor; + context.font = `${config.labelFontSize}px sans-serif`; + const fontY = config.height + config.labelFontSize + config.labelTopMargin; + const nowText = "now"; + const nowWidth = context.measureText(nowText).width; + context.fillText(nowText, config.width - nowWidth - 1, fontY); + context.fillText("-30m", 1, fontY); + } // Background // (This hides any previous data points, so we don't have to clear the canvas) @@ -25,6 +44,7 @@ class SyncGraph extends React.Component { const pxPerSec = config.width / config.timeLength; context.strokeStyle = config.dataColor; context.beginPath(); + for (const syncTimeMs of this.props.syncTimestamps) { const secsAgo = (now - syncTimeMs) / 1000; const pxFromRight = secsAgo * pxPerSec; @@ -43,17 +63,6 @@ class SyncGraph extends React.Component { context.lineTo(px, config.height); } context.stroke(); - - // Axis labels - if (!isUpdate) { // only draw these on the initial render - context.fillStyle = config.labelColor; - context.font = `${config.labelFontSize}px sans-serif`; - const fontY = config.height + config.labelFontSize + config.labelTopMargin; - const nowText = "now"; - const nowWidth = context.measureText(nowText).width; - context.fillText(nowText, config.width - nowWidth - 1, fontY); - context.fillText("-30m", 1, fontY); - } } render() { @@ -82,7 +91,7 @@ SyncGraph.config = { labelTopMargin: 2, labelColor: 'black', backgroundColor: 'black', - dataColor: 'blue', + dataColor: '#43a1ff', } SyncGraph.propTypes = {