Merge branch 'master' of github.com:nylas/K2 into contacts

.eslintrc

@@ -17,7 +17,6 @@
     "object-curly-spacing": "off",
     "max-len": "off",
     "new-cap": ["error", {"capIsNew": false}],
-    "no-console": "off",
     "no-constant-condition": "off",
     "no-loop-func": "off",
     "no-shadow": "error",

.gitignore

@@ -4,6 +4,7 @@ dump.rdb
 *npm-debug.log
 storage/
 lerna-debug.log
+newrelic_agent.log
 
 # Elastic Beanstalk Files
 .elasticbeanstalk/*

README.md

@@ -28,7 +28,9 @@
 # Developing Locally:
 
 ```
-npm start
+npm run start
+npm run logs
+npm run stop
 ```
 
 We use [pm2](http://pm2.keymetrics.io/) to launch a variety of processes

package.json

@@ -5,6 +5,8 @@
   "main": "",
   "dependencies": {
     "bluebird": "3.x.x",
+    "bunyan": "1.8.0",
+    "bunyan-cloudwatch": "2.0.0",
     "lerna": "2.0.0-beta.23",
     "mysql": "^2.11.1",
     "newrelic": "^1.28.1",
@@ -16,6 +18,7 @@
   },
   "devDependencies": {
     "babel-eslint": "6.x",
+    "bunyan-prettystream": "^0.1.3",
     "eslint": "2.x",
     "eslint-config-airbnb": "8.x",
     "eslint-plugin-import": "1.x",
@@ -25,8 +28,9 @@
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz"
   },
   "scripts": {
-    "start": "./node_modules/pm2/bin/pm2 start ./pm2-dev.yml  --no-daemon",
-    "postinstall": "node_modules/.bin/lerna bootstrap"
+    "start": "pm2 start ./pm2-dev.yml --no-daemon",
+    "stop": "pm2 kill",
+    "postinstall": "lerna bootstrap"
   },
   "repository": {
     "type": "git",

packages/nylas-api/app.js

@@ -1,3 +1,5 @@
+// require('newrelic');
+
 const Hapi = require('hapi');
 const HapiSwagger = require('hapi-swagger');
 const HapiBoom = require('hapi-boom-decorators')
@@ -7,8 +9,10 @@ const Vision = require('vision');
 const Package = require('./package');
 const fs = require('fs');
 const path = require('path');
+const {DatabaseConnector, SchedulerUtils, Logger} = require(`nylas-core`);
 
 global.Promise = require('bluebird');
+global.Logger = Logger.createLogger('nylas-k2-api')
 
 const server = new Hapi.Server({
   connections: {
@@ -33,8 +37,6 @@ const plugins = [Inert, Vision, HapiBasicAuth, HapiBoom, {
 let sharedDb = null;
 
 const validate = (request, username, password, callback) => {
-  const {DatabaseConnector, SchedulerUtils} = require(`nylas-core`);
-
   let getSharedDb = null;
   if (sharedDb) {
     getSharedDb = Promise.resolve(sharedDb)
@@ -88,6 +90,6 @@ server.register(plugins, (err) => {
 
   server.start((startErr) => {
     if (startErr) { throw startErr; }
-    console.log('API running at:', server.info.uri);
+    global.Logger.info({url: server.info.uri}, 'API running');
   });
 });

packages/nylas-api/decorators/connections.js

@@ -7,4 +7,10 @@ module.exports = (server) => {
     const account = this.auth.credentials;
     return DatabaseConnector.forAccount(account.id);
   });
+  server.decorate('request', 'logger', (request) => {
+    if (request.auth.credentials) {
+      return global.Logger.forAccount(request.auth.credentials)
+    }
+    return global.Logger
+  }, {apply: true});
 }

packages/nylas-api/newrelic.js

@@ -0,0 +1,24 @@
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+  /**
+   * Array of application names.
+   */
+  app_name: ['Nylas K2 API'],
+  /**
+   * Your New Relic license key.
+   */
+  license_key: 'e232d6ccc786bd87aa72b86782439710162e3739',
+  logging: {
+    /**
+     * Level at which to log. 'trace' is most useful to New Relic when diagnosing
+     * issues with the agent, 'info' and higher will impose the least overhead on
+     * production applications.
+     */
+    level: 'info',
+  },
+}

packages/nylas-api/routes/auth.js

@@ -43,24 +43,33 @@ const buildAccountWith = ({name, email, provider, settings, credentials}) => {
   return DatabaseConnector.forShared().then((db) => {
     const {AccountToken, Account} = db;
 
-    const account = Account.build({
-      name: name,
-      provider: provider,
-      emailAddress: email,
-      connectionSettings: settings,
-      syncPolicy: SyncPolicy.defaultPolicy(),
-      lastSyncCompletions: [],
-    })
-    account.setCredentials(credentials);
+    return Account.find({
+      where: {
+        emailAddress: email,
+        connectionSettings: JSON.stringify(settings),
+      },
+    }).then((existing) => {
+      const account = existing || Account.build({
+        name: name,
+        provider: provider,
+        emailAddress: email,
+        connectionSettings: settings,
+        syncPolicy: SyncPolicy.defaultPolicy(),
+        lastSyncCompletions: [],
+      })
 
-    return account.save().then((saved) =>
-      AccountToken.create({accountId: saved.id}).then((token) =>
-        DatabaseConnector.prepareAccountDatabase(saved.id).thenReturn({
-          account: saved,
-          token: token,
-        })
-      )
-    );
+      // always update with the latest credentials
+      account.setCredentials(credentials);
+
+      return account.save().then((saved) =>
+        AccountToken.create({accountId: saved.id}).then((token) =>
+          DatabaseConnector.prepareAccountDatabase(saved.id).thenReturn({
+            account: saved,
+            token: token,
+          })
+        )
+      );
+    });
   });
 }
 
@@ -97,7 +106,11 @@ module.exports = (server) => {
       const {settings, email, provider, name} = request.payload;
 
       if (provider === 'imap') {
-        connectionChecks.push(IMAPConnection.connect(dbStub, settings))
+        connectionChecks.push(IMAPConnection.connect({
+          logger: request.logger,
+          settings: settings,
+          db: dbStub,
+        }));
       }
 
       Promise.all(connectionChecks).then(() => {
@@ -186,9 +199,12 @@ module.exports = (server) => {
             client_id: GMAIL_CLIENT_ID,
             client_secret: GMAIL_CLIENT_SECRET,
           }
-
           Promise.all([
-            IMAPConnection.connect({}, Object.assign({}, settings, credentials)),
+            IMAPConnection.connect({
+              logger: request.logger,
+              settings: Object.assign({}, settings, credentials),
+              db: {},
+            }),
           ])
           .then(() =>
             buildAccountWith({

packages/nylas-api/routes/files.js

@@ -73,9 +73,7 @@ module.exports = (server) => {
     },
     handler: (request, reply) => {
       request.getAccountDatabase().then(({File}) => {
-        const {headers: {accept}} = request
         const {params: {id}} = request
-        const account = request.auth.credentials
 
         File.findOne({where: {id}}).then((file) => {
           if (!file) {
@@ -83,9 +81,9 @@ module.exports = (server) => {
           }
           return reply(Serialization.jsonStringify(file));
         })
-        .catch((error) => {
-          console.log('Error fetching file: ', error)
-          reply(error)
+        .catch((err) => {
+          request.logger.error(err, 'Error fetching file')
+          reply(err)
         })
       })
     },
@@ -107,7 +105,6 @@ module.exports = (server) => {
     handler: (request, reply) => {
       request.getAccountDatabase()
       .then((db) => {
-        const {headers: {accept}} = request
         const {params: {id}} = request
         const account = request.auth.credentials
 
@@ -116,12 +113,12 @@ module.exports = (server) => {
           if (!file) {
             return reply.notFound(`File ${id} not found`)
           }
-          return file.fetch({account, db})
+          return file.fetch({account, db, logger: request.logger})
           .then((stream) => reply(stream))
         })
-        .catch((error) => {
-          console.log('Error fetching file: ', error)
-          reply(error)
+        .catch((err) => {
+          request.logger.error(err, 'Error downloading file')
+          reply(err)
         })
       })
     },

packages/nylas-api/routes/messages.js

@@ -123,15 +123,16 @@ module.exports = (server) => {
             return reply.notFound(`Message ${id} not found`)
           }
           if (accept === 'message/rfc822') {
-            return message.fetchRaw({account, db}).then((rawMessage) =>
+            return message.fetchRaw({account, db, logger: request.logger})
+            .then((rawMessage) =>
               reply(rawMessage)
             )
           }
           return reply(Serialization.jsonStringify(message));
         })
-        .catch((error) => {
-          console.log('Error fetching message: ', error)
-          reply(error)
+        .catch((err) => {
+          request.logger.error(err, 'Error fetching message')
+          reply(err)
         })
       })
     },

packages/nylas-api/routes/ping.js

@@ -6,8 +6,8 @@ module.exports = (server) => {
       auth: false,
     },
     handler: (request, reply) => {
-      console.log("---> Ping!")
-      reply("pong")
+      request.logger.info('----> Pong!')
+      reply("Pong")
     },
   });
 };

packages/nylas-api/serialization.js

@@ -23,7 +23,7 @@ function jsonSchema(modelName) {
     //   connection_settings: Joi.object(),
     //   sync_policy: Joi.object(),
     //   sync_error: Joi.object().allow(null),
-    //   first_sync_completed_at: Joi.number().allow(null),
+    //   first_sync_completion: Joi.number().allow(null),
     //   last_sync_completions: Joi.array(),
     // })
   }

packages/nylas-core/imap-connection.js

@@ -156,8 +156,6 @@ class IMAPBox {
     }
     return this._imap.closeBoxAsync(expunge)
   }
-
-
 }
 
 
@@ -176,8 +174,17 @@ class IMAPConnection extends EventEmitter {
     return new IMAPConnection(...args).connect()
   }
 
-  constructor(db, settings) {
+  constructor({db, settings, logger} = {}) {
     super();
+
+    if (!(settings instanceof Object)) {
+      throw new Error("IMAPConnection: Must be instantiated with `settings`")
+    }
+    if (!logger) {
+      throw new Error("IMAPConnection: Must be instantiated with `logger`")
+    }
+
+    this._logger = logger;
     this._db = db;
     this._queue = [];
     this._currentOperation = null;
@@ -231,13 +238,13 @@ class IMAPConnection extends EventEmitter {
       this._imap = Promise.promisifyAll(new Imap(settings));
 
       this._imap.once('end', () => {
-        console.log('Underlying IMAP Connection ended');
+        this._logger.info('Underlying IMAP Connection ended');
         this._connectPromise = null;
         this._imap = null;
       });
 
       this._imap.on('alert', (msg) => {
-        console.log(`IMAP SERVER SAYS: ${msg}`)
+        this._logger.info({imap_server_msg: msg}, `IMAP server message`)
       })
 
       // Emitted when new mail arrives in the currently open mailbox.
@@ -346,14 +353,20 @@ class IMAPConnection extends EventEmitter {
 
     result.then(() => {
       this._currentOperation = null;
-      console.log(`Finished task: ${operation.description()}`)
+      this._logger.info({
+        operation_type: operation.constructor.name,
+        operation_description: operation.description(),
+      }, `Finished sync operation`)
       resolve();
       this.processNextOperation();
     })
     .catch((err) => {
       this._currentOperation = null;
-      console.log(`Task errored: ${operation.description()}`)
-      console.error(err)
+      this._logger.error({
+        err,
+        operation_type: operation.constructor.name,
+        operation_description: operation.description(),
+      }, `Sync operation errored`)
       reject(err);
     })
   }

packages/nylas-core/index.js

@@ -11,4 +11,5 @@ module.exports = {
   SyncPolicy: require('./sync-policy'),
   SchedulerUtils: require('./scheduler-utils'),
   MessageTypes: require('./message-types'),
+  Logger: require('./logger'),
 }

packages/nylas-core/logger.js

@@ -0,0 +1,61 @@
+const bunyan = require('bunyan')
+const createCWStream = require('bunyan-cloudwatch')
+const PrettyStream = require('bunyan-prettystream');
+const NODE_ENV = process.env.NODE_ENV || 'unknown'
+
+
+function getLogStreams(name, env) {
+  if (env === 'development') {
+    const prettyStdOut = new PrettyStream();
+    prettyStdOut.pipe(process.stdout);
+    const stdoutStream = {
+      type: 'raw',
+      level: 'debug',
+      stream: prettyStdOut,
+    }
+    return [stdoutStream]
+  }
+
+  const stdoutStream = {
+    stream: process.stdout,
+    level: 'info',
+  }
+  const cloudwatchStream = {
+    stream: createCWStream({
+      logGroupName: `k2-${env}`,
+      logStreamName: `${name}-${env}`,
+      cloudWatchLogsOptions: {
+        region: 'us-east-1',
+      },
+    }),
+    type: 'raw',
+    reemitErrorEvents: true,
+  }
+  return [stdoutStream, cloudwatchStream]
+}
+
+function createLogger(name, env = NODE_ENV) {
+  const childLogs = new Map()
+  const logger = bunyan.createLogger({
+    name,
+    serializers: bunyan.stdSerializers,
+    streams: getLogStreams(name, env),
+  })
+
+  return Object.assign(logger, {
+    forAccount(account = {}) {
+      if (!childLogs.has(account.id)) {
+        const childLog = logger.child({
+          account_id: account.id,
+          account_email: account.emailAddress,
+        })
+        childLogs.set(account.id, childLog)
+      }
+      return childLogs.get(account.id)
+    },
+  })
+}
+
+module.exports = {
+  createLogger,
+}

packages/nylas-core/models/account/file.js

@@ -15,11 +15,11 @@ module.exports = (sequelize, Sequelize) => {
       },
     },
     instanceMethods: {
-      fetch: function fetch({account, db}) {
+      fetch: function fetch({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
         return Promise.props({
           message: this.getMessage(),
-          connection: IMAPConnection.connect(db, settings),
+          connection: IMAPConnection.connect({db, settings, logger}),
         })
         .then(({message, connection}) => {
           return message.getFolder()

packages/nylas-core/models/account/message.js

@@ -69,11 +69,11 @@ module.exports = (sequelize, Sequelize) => {
         )
       },
 
-      fetchRaw: function fetchRaw({account, db}) {
+      fetchRaw: function fetchRaw({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
         return Promise.props({
           folder: this.getFolder(),
-          connection: IMAPConnection.connect(db, settings),
+          connection: IMAPConnection.connect({db, settings, logger}),
         })
         .then(({folder, connection}) => {
           return connection.openBox(folder.name)

packages/nylas-core/models/shared/account.js

@@ -12,7 +12,7 @@ module.exports = (sequelize, Sequelize) => {
     connectionCredentials: Sequelize.TEXT,
     syncPolicy: JSONType('syncPolicy'),
     syncError: JSONType('syncError', {defaultValue: null}),
-    firstSyncCompletedAt: Sequelize.INTEGER,
+    firstSyncCompletion: Sequelize.INTEGER(14),
     lastSyncCompletions: JSONARRAYType('lastSyncCompletions'),
   }, {
     classMethods: {
@@ -31,7 +31,7 @@ module.exports = (sequelize, Sequelize) => {
           connection_settings: this.connectionSettings,
           sync_policy: this.syncPolicy,
           sync_error: this.syncError,
-          first_sync_completed_at: this.firstSyncCompletedAt,
+          first_sync_completion: this.firstSyncCompletion,
           last_sync_completions: this.lastSyncCompletions,
           created_at: this.createdAt,
         }

packages/nylas-core/package.json

@@ -4,6 +4,7 @@
   "description": "Core shared packages",
   "main": "index.js",
   "dependencies": {
+    "bunyan": "^1.8.1",
     "imap": "0.8.x",
     "xoauth2": "1.x.x"
   },

packages/nylas-core/pubsub-connector.js

@@ -1,9 +1,11 @@
 const Rx = require('rx')
 const redis = require("redis");
+const log = global.Logger || console
 
 Promise.promisifyAll(redis.RedisClient.prototype);
 Promise.promisifyAll(redis.Multi.prototype);
 
+
 class PubsubConnector {
   constructor() {
     this._broadcastClient = null;
@@ -13,7 +15,7 @@ class PubsubConnector {
 
   buildClient() {
     const client = redis.createClient(process.env.REDIS_URL || null);
-    client.on("error", console.error);
+    client.on("error", log.error);
     return client;
   }
 

packages/nylas-core/scheduler-utils.js

@@ -22,7 +22,9 @@ const forEachAccountList = (forEachCallback) => {
 }
 
 const assignPolicy = (accountId, policy) => {
-  console.log(`Changing policy for ${accountId} to ${JSON.stringify(policy)}`)
+  const log = global.Logger || console
+  log.info({policy, account_id: accountId}, `Changing single policy`)
+
   const DatabaseConnector = require('./database-connector');
   return DatabaseConnector.forShared().then(({Account}) => {
     Account.find({where: {id: accountId}}).then((account) => {
@@ -33,7 +35,9 @@ const assignPolicy = (accountId, policy) => {
 }
 
 const assignPolicyToAcounts = (accountIds, policy) => {
-  console.log(`Changing policy for ${accountIds} to ${JSON.stringify(policy)}`)
+  const log = global.Logger || console
+  log.info({policy, account_ids: accountIds}, `Changing multiple policies`)
+
   const DatabaseConnector = require('./database-connector');
   return DatabaseConnector.forShared().then(({Account}) => {
     Account.findAll({where: {id: {$or: accountIds}}}).then((accounts) => {

packages/nylas-dashboard/app.js

@@ -1,11 +1,12 @@
 const Hapi = require('hapi');
 const HapiWebSocket = require('hapi-plugin-websocket');
 const Inert = require('inert');
-const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`);
+const {DatabaseConnector, PubsubConnector, SchedulerUtils, Logger} = require(`nylas-core`);
 const fs = require('fs');
 const path = require('path');
 
 global.Promise = require('bluebird');
+global.Logger = Logger.createLogger('nylas-k2-dashboard')
 
 const server = new Hapi.Server();
 server.connection({ port: process.env.PORT });
@@ -98,6 +99,6 @@ server.register([HapiWebSocket, Inert], () => {
 
   server.start((startErr) => {
     if (startErr) { throw startErr; }
-    console.log('Dashboard running at:', server.info.uri);
+    global.Logger.info({uri: server.info.uri}, 'Dashboard running');
   });
 });

packages/nylas-dashboard/public/css/app.css

@@ -59,10 +59,14 @@ pre {
 
 .action-link {
   display: inline-block;
-  margin: 5px;
   color: rgba(16, 83, 161, 0.88);
   text-decoration: underline;
   cursor: pointer;
+  margin: 5px 0;
+}
+
+.action-link.cancel {
+  margin-left: 5px;
 }
 
 .sync-policy textarea {
@@ -76,6 +80,7 @@ pre {
   width: 50%;
   margin: auto;
   padding: 20px;
+  overflow: auto;
 }
 
 .modal-bg {
@@ -88,6 +93,15 @@ pre {
   padding-top: 10%;
 }
 
+.modal-close {
+  position: relative;
+  float: right;
+  top: -10px;
+  cursor: pointer;
+  font-size: 14px;
+  font-weight: bold;
+}
+
 .sync-graph {
   margin-top: 3px;
 }
@@ -97,3 +111,65 @@ pre {
   margin-top: 5px;
   margin-bottom: 1px;
 }
+
+#syncback-request-details {
+  font-size: 15px;
+  color: black;
+}
+
+#syncback-request-details .counts {
+  margin: 10px;
+}
+
+#syncback-request-details span {
+  margin: 10px;
+}
+
+#syncback-request-details table {
+  width: 100%;
+}
+
+#syncback-request-details tr:nth-child(even) {
+  background-color: #F1F1F1;
+}
+
+#syncback-request-details tr:not(:first-child):hover {
+  background-color: #C9C9C9;
+}
+
+#syncback-request-details td, #syncback-request-details th {
+  text-align: center;
+  padding: 10px 5px;
+}
+
+.dropdown-arrow {
+  margin: 0 5px;
+  height: 7px;
+  vertical-align: middle;
+}
+
+.dropdown-options {
+  border: solid black 1px;
+  position: absolute;
+  background-color: white;
+  text-align: left;
+}
+
+.dropdown-option {
+  padding: 0px 2px;
+}
+
+.dropdown-option:hover {
+  background-color: rgb(114, 163, 255);
+}
+
+.dropdown-selected {
+  display: inline;
+}
+
+.dropdown-wrapper {
+  display: inline;
+  cursor: pointer;
+  position: absolute;
+  font-weight: normal;
+}

packages/nylas-dashboard/public/index.html

@@ -7,6 +7,8 @@
   <script src="/js/set-all-sync-policies.jsx" type="text/babel"></script>
   <script src="/js/account-filter.jsx" type="text/babel"></script>
   <script src="/js/sync-graph.jsx" type="text/babel"></script>
+  <script src="/js/dropdown.jsx" type="text/babel"></script>
+  <script src="/js/syncback-request-details.jsx" type="text/babel"></script>
   <script src="/js/app.jsx" type="text/babel"></script>
   <link rel='stylesheet' type="text/css" href="./css/app.css" />
   <link rel='shortcut icon' href='favicon.png' / >

packages/nylas-dashboard/public/js/app.jsx

@@ -1,4 +1,6 @@
 /* eslint react/react-in-jsx-scope: 0*/
+/* eslint no-console: 0*/
+
 const React = window.React;
 const ReactDOM = window.ReactDOM;
 const {
@@ -6,6 +8,7 @@ const {
   SetAllSyncPolicies,
   AccountFilter,
   SyncGraph,
+  SyncbackRequestDetails,
 } = window;
 
 class Account extends React.Component {
@@ -43,14 +46,15 @@ class Account extends React.Component {
     const timeSinceLastSync = (Date.now() - newestSync) / 1000;
 
     let firstSyncDuration = "Incomplete";
-    if (account.first_sync_completed_at) {
-      firstSyncDuration = (new Date(account.first_sync_completed_at) - new Date(account.created_at)) / 1000;
+    if (account.first_sync_completion) {
+      firstSyncDuration = (new Date(account.first_sync_completion) - new Date(account.created_at)) / 1000;
     }
 
     return (
       <div className={`account${errorClass}`}>
         <h3>{account.email_address} {active ? '🌕' : '🌑'}</h3>
         <strong>{assignment}</strong>
+        <SyncbackRequestDetails accountId={account.id} />
         <SyncPolicy
           accountId={account.id}
           stringifiedSyncPolicy={JSON.stringify(account.sync_policy, null, 2)}

packages/nylas-dashboard/public/js/dropdown.jsx

@@ -0,0 +1,71 @@
+const React = window.React;
+
+class Dropdown extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      closed: true,
+      selected: props.defaultOption,
+      onSelect: props.onSelect,
+    }
+  }
+
+  open() {
+    this.setState({closed: false});
+  }
+
+  selectAndClose(selection) {
+    this.setState({
+      closed: true,
+      selected: selection,
+    })
+    this.state.onSelect(selection);
+  }
+
+  close() {
+    this.setState({closed: true});
+  }
+
+  render() {
+    // Currently selected option (includes dropdown arrow)
+    const selectedOnClick = this.state.closed ? this.open : this.close;
+    const selected = (
+      <div className="dropdown-selected" onClick={() => selectedOnClick.call(this)}>
+        {this.state.selected}
+        <img className="dropdown-arrow" src="../images/dropdown.png" alt="dropdown arrow" />
+      </div>
+    );
+
+    // All options, not shown if dropdown is closed
+    let options = [];
+    let optionsWrapper = <span />;
+    if (!this.state.closed) {
+      for (const opt of this.props.options) {
+        options.push(
+          <div className="dropdown-option" onMouseDown={() => this.selectAndClose.call(this, opt)}> {opt} </div>
+        );
+      }
+      optionsWrapper = (
+        <div className="dropdown-options">
+          {options}
+        </div>
+      )
+    }
+
+    return (
+      <div className="dropdown-wrapper" tabIndex="0" onBlur={() => this.close.call(this)}>
+        {selected}
+        {optionsWrapper}
+      </div>
+    );
+  }
+
+}
+
+Dropdown.propTypes = {
+  options: React.PropTypes.arrayOf(React.PropTypes.string),
+  defaultOption: React.PropTypes.string,
+  onSelect: React.PropTypes.func,
+}
+
+window.Dropdown = Dropdown;

packages/nylas-dashboard/public/js/set-all-sync-policies.jsx

@@ -38,15 +38,20 @@ class SetAllSyncPolicies extends React.Component {
   render() {
     if (this.state.editMode) {
       return (
-        <div className="modal-bg">
-          <div className="sync-policy modal">
-            <div className="section">Sync Policy</div>
-            <textarea id="sync-policy-all">
-            </textarea>
-            <button onClick={() => this.applyToAllAccounts.call(this, this.props.accountIds)}>
-              Apply To All Displayed Accounts
-            </button>
-            <span className="action-link" onClick={() => this.cancel.call(this)}> Cancel </span>
+        <div>
+          <span className="action-link" id="set-all-sync" onClick={() => this.edit.call(this)}>
+            Set sync policies for currently displayed accounts
+          </span>
+          <div className="modal-bg">
+            <div className="sync-policy modal">
+              <div className="section">Sync Policy</div>
+              <textarea id="sync-policy-all">
+              </textarea>
+              <button onClick={() => this.applyToAllAccounts.call(this, this.props.accountIds)}>
+                Apply To All Displayed Accounts
+              </button>
+              <span className="action-link cancel" onClick={() => this.cancel.call(this)}> Cancel </span>
+            </div>
           </div>
         </div>
       )

packages/nylas-dashboard/public/js/sync-policy.jsx

@@ -43,7 +43,7 @@ class SyncPolicy extends React.Component {
             {this.props.stringifiedSyncPolicy}
           </textarea>
           <button onClick={() => this.save.call(this)}> Save </button>
-          <span className="action-link" onClick={() => this.cancel.call(this)}> Cancel </span>
+          <span className="action-link cancel" onClick={() => this.cancel.call(this)}> Cancel </span>
         </div>
 
       )

packages/nylas-dashboard/public/js/syncback-request-details.jsx

@@ -0,0 +1,171 @@
+const React = window.React;
+const Dropdown = window.Dropdown;
+
+class SyncbackRequestDetails extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      open: false,
+      accountId: props.accountId,
+      syncbackRequests: null,
+      counts: null,
+      statusFilter: 'all',
+    };
+  }
+
+  getDetails() {
+    const req = new XMLHttpRequest();
+    const url = `${window.location.protocol}/syncback-requests/${this.state.accountId}`;
+    req.open("GET", url, true);
+    req.onreadystatechange = () => {
+      if (req.readyState === XMLHttpRequest.DONE) {
+        if (req.status === 200) {
+          this.setState({syncbackRequests: req.responseText});
+        } else {
+          console.error(req.responseText);
+        }
+      }
+    }
+    req.send();
+  }
+
+  getCounts() {
+    const since = Date.now() - 1000 * 60 * 60; // one hour ago
+    const req = new XMLHttpRequest();
+    const url = `${window.location.protocol}/syncback-requests/${this.state.accountId}/counts?since=${since}`;
+    req.open("GET", url, true);
+    req.onreadystatechange = () => {
+      if (req.readyState === XMLHttpRequest.DONE) {
+        if (req.status === 200) {
+          this.setState({counts: JSON.parse(req.responseText)});
+        } else {
+          console.error(req.responseText);
+        }
+      }
+    }
+    req.send();
+  }
+
+  setStatusFilter(statusFilter) {
+    this.setState({statusFilter: statusFilter});
+  }
+
+  open() {
+    this.getDetails();
+    this.getCounts();
+    this.setState({open: true});
+  }
+
+  close() {
+    this.setState({open: false});
+  }
+
+  render() {
+    if (this.state.open) {
+      let counts = <span> Of requests created in the last hour: ... </span>
+      if (this.state.counts) {
+        const total = this.state.counts.new + this.state.counts.failed
+          + this.state.counts.succeeded;
+        if (total === 0) {
+          counts = "No requests made in the last hour";
+        } else {
+          counts = (
+            <div className="counts">
+              Of requests created in the last hour:
+              <span
+                style={{color: 'rgb(222, 68, 68)'}}
+                title={`${this.state.counts.failed} out of ${total}`}
+              >
+                {this.state.counts.failed / total * 100}&#37; failed
+              </span>
+              <span
+                style={{color: 'green'}}
+                title={`${this.state.counts.succeeded} out of ${total}`}
+              >
+                {this.state.counts.succeeded / total * 100}&#37; succeeded
+              </span>
+              <span
+                style={{color: 'rgb(98, 98, 179)'}}
+                title={`${this.state.counts.new} out of ${total}`}
+              >
+                {/* .new was throwing off my syntax higlighting, so ignoring linter*/}
+                {this.state.counts['new'] / total * 100}&#37; are still new
+              </span>
+            </div>
+          )
+        }
+      }
+
+      let details = "Loading..."
+      if (this.state.syncbackRequests) {
+        let reqs = JSON.parse(this.state.syncbackRequests);
+        if (this.state.statusFilter !== 'all') {
+          reqs = reqs.filter((req) => req.status === this.state.statusFilter);
+        }
+        let rows = [];
+        if (reqs.length === 0) {
+          rows.push(<tr><td>No results</td><td>-</td><td>-</td></tr>);
+        }
+        for (let i = reqs.length - 1; i >= 0; i--) {
+          const req = reqs[i];
+          const date = new Date(req.createdAt);
+          rows.push(<tr key={req.id} title={`id: ${req.id}`}>
+            <td> {req.status} </td>
+            <td> {req.type} </td>
+            <td> {date.toLocaleTimeString()}, {date.toLocaleDateString()} </td>
+          </tr>)
+        }
+        details = (
+          <table>
+            <tbody>
+              <tr>
+                <th>
+                  Status:&nbsp;
+                  <Dropdown
+                    options={['all', 'FAILED', 'NEW', 'SUCCEEDED']}
+                    defaultOption="all"
+                    onSelect={(status) => this.setStatusFilter.call(this, status)}
+                  />
+                </th>
+                <th> Type </th>
+                <th> Created At </th>
+              </tr>
+              {rows}
+            </tbody>
+          </table>
+        );
+      }
+
+      return (
+        <div>
+          <span className="action-link">Syncback Request Details </span>
+          <div className="modal-bg">
+            <div className="modal">
+              <div className="modal-close" onClick={() => this.close.call(this)}>
+                X
+              </div>
+              <div id="syncback-request-details">
+                {counts}
+                {details}
+              </div>
+            </div>
+          </div>
+        </div>
+      );
+    }
+    // else, the modal isn't open
+    return (
+      <div>
+        <span className="action-link" onClick={() => this.open.call(this)}>
+          Syncback Request Details
+        </span>
+      </div>
+    );
+  }
+}
+
+SyncbackRequestDetails.propTypes = {
+  accountId: React.PropTypes.number,
+}
+
+window.SyncbackRequestDetails = SyncbackRequestDetails;

packages/nylas-dashboard/routes/syncback-requests.js

@@ -0,0 +1,81 @@
+const Joi = require('joi');
+const {DatabaseConnector} = require(`nylas-core`);
+
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/syncback-requests/{account_id}',
+    config: {
+      description: 'Get the SyncbackRequests for an account',
+      notes: 'Notes go here',
+      tags: ['syncback-requests'],
+      validate: {
+        params: {
+          account_id: Joi.number().integer(),
+        },
+      },
+      response: {
+        schema: Joi.string(),
+      },
+    },
+    handler: (request, reply) => {
+      DatabaseConnector.forAccount(request.params.account_id).then((db) => {
+        const {SyncbackRequest} = db;
+        SyncbackRequest.findAll().then((syncbackRequests) => {
+          reply(JSON.stringify(syncbackRequests))
+        });
+      });
+    },
+  });
+
+  server.route({
+    method: 'GET',
+    path: '/syncback-requests/{account_id}/counts',
+    config: {
+      description: 'Get stats on the statuses of SyncbackRequests',
+      notes: 'Notes go here',
+      tags: ['syncback-requests'],
+      validate: {
+        params: {
+          account_id: Joi.number().integer(),
+        },
+        query: {
+          since: Joi.date().timestamp(),
+        },
+      },
+      response: {
+        schema: Joi.string(),
+      },
+    },
+    handler: (request, reply) => {
+      DatabaseConnector.forAccount(request.params.account_id).then((db) => {
+        const {SyncbackRequest} = db;
+
+        const counts = {
+          'new': null,
+          'succeeded': null,
+          'failed': null,
+        }
+
+        const where = {};
+        if (request.query.since) {
+          where.createdAt = {gt: request.query.since};
+        }
+
+        const countPromises = [];
+        for (const status of Object.keys(counts)) {
+          where.status = status.toUpperCase();
+          countPromises.push(
+            SyncbackRequest.count({where: where}).then((count) => {
+              counts[status] = count;
+            })
+          );
+        }
+
+        Promise.all(countPromises).then(() => {
+          reply(JSON.stringify(counts));
+        })
+      });
+    },
+  });
+};

packages/nylas-message-processor/app.js

@@ -1,7 +1,8 @@
-const {PubsubConnector, DatabaseConnector} = require(`nylas-core`)
+const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`)
 const {processors} = require('./processors')
 
 global.Promise = require('bluebird');
+global.Logger = Logger.createLogger('nylas-k2-message-processor')
 
 // List of the attributes of Message that the processor should be allowed to change.
 // The message may move between folders, get starred, etc. while it's being
@@ -11,15 +12,13 @@ const MessageProcessorVersion = 1;
 
 const redis = PubsubConnector.buildClient();
 
-function runPipeline({db, accountId, message}) {
-  console.log(`Processing message ${message.id}`)
+function runPipeline({db, accountId, message, logger}) {
+  logger.info(`MessageProcessor: Processing message`)
   return processors.reduce((prevPromise, processor) => (
     prevPromise.then((prevMessage) => {
-      const processed = processor({message: prevMessage, accountId, db});
-      if (!(processed instanceof Promise)) {
-        throw new Error(`processor ${processor} did not return a promise.`)
-      }
-      return processed.then((nextMessage) => {
+      const processed = processor({message: prevMessage, accountId, db, logger});
+      return Promise.resolve(processed)
+      .then((nextMessage) => {
         if (!nextMessage.body) {
           throw new Error("processor did not resolve with a valid message object.")
         }
@@ -46,26 +45,28 @@ function dequeueJob() {
     try {
       json = JSON.parse(item[1]);
     } catch (error) {
-      console.error(`MessageProcessor Failed: Found invalid JSON item in queue: ${item}`)
+      global.Logger.error({item}, `MessageProcessor: Found invalid JSON item in queue`)
       return dequeueJob();
     }
     const {messageId, accountId} = json;
+    const logger = global.Logger.forAccount({id: accountId}).child({message_id: messageId})
 
-    DatabaseConnector.forAccount(accountId).then((db) =>
-      db.Message.find({
+    DatabaseConnector.forAccount(accountId).then((db) => {
+      return db.Message.find({
         where: {id: messageId},
         include: [{model: db.Folder}, {model: db.Label}],
       }).then((message) => {
         if (!message) {
           return Promise.reject(new Error(`Message not found (${messageId}). Maybe account was deleted?`))
         }
-        return runPipeline({db, accountId, message}).then((processedMessage) =>
+        return runPipeline({db, accountId, message, logger}).then((processedMessage) =>
           saveMessage(processedMessage)
         ).catch((err) =>
-          console.error(`MessageProcessor Failed: ${err} ${err.stack}`)
+          logger.error(err, `MessageProcessor: Failed`)
         )
       })
-    ).finally(() => {
+    })
+    .finally(() => {
       dequeueJob()
     });
 

packages/nylas-message-processor/processors/parsing.js

@@ -11,7 +11,7 @@ function Contact({name, address} = {}) {
 const extractContacts = (values) =>
   (values || []).map(v => Contact(mimelib.parseAddresses(v).pop()))
 
-function processMessage({message}) {
+function processMessage({message, logger}) {
   if (message.snippet) {
     // trim and clean snippet which is alreay present (from message plaintext)
     message.snippet = message.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ')
@@ -24,7 +24,7 @@ function processMessage({message}) {
     // TODO: Fanciness
     message.snippet = message.body.substr(0, Math.min(message.body.length, SNIPPET_SIZE));
   } else {
-    console.log("Received message has no body or snippet.")
+    logger.info("MessageProcessor: Parsing - Received message has no body or snippet.")
   }
 
   message.to = extractContacts(message.headers.to);

packages/nylas-message-processor/processors/threading.js

@@ -9,11 +9,14 @@ class ThreadingProcessor {
     // conversation. Put it back soonish.
 
     // const messageEmails = _.uniq([].concat(message.to, message.cc, message.from).map(p => p.email));
-    // console.log(`Found ${threads.length} candidate threads for message with subject: ${message.subject}`)
+    // this.logger.info({
+    //   num_candidate_threads: threads.length,
+    //   message_subject: message.subject,
+    // }, `Found candidate threads for message`)
     //
     // for (const thread of threads) {
     //   const threadEmails = _.uniq([].concat(thread.participants).map(p => p.email));
-    //   console.log(`Intersection: ${_.intersection(threadEmails, messageEmails).join(',')}`)
+    //   this.logger.info(`Intersection: ${_.intersection(threadEmails, messageEmails).join(',')}`)
     //
     //   if (_.intersection(threadEmails, messageEmails) >= threadEmails.length * 0.9) {
     //     return thread;
@@ -66,7 +69,7 @@ class ThreadingProcessor {
     })
   }
 
-  processMessage({db, message}) {
+  processMessage({db, message, logger}) {
     if (!(message.labels instanceof Array)) {
       throw new Error("Threading processMessage expects labels to be an inflated array.");
     }
@@ -74,6 +77,8 @@ class ThreadingProcessor {
       throw new Error("Threading processMessage expects folder value to be present.");
     }
 
+    this.logger = logger
+
     const {Folder, Label} = db;
     let findOrCreateThread = null;
     if (message.headers['x-gm-thrid']) {

packages/nylas-message-processor/spec/threading-spec.js

@@ -43,7 +43,6 @@ it('adds the message to the thread', (done) => {
       },
       create: (message) => {
         message.setThread = (thread) => {
-          console.log("setting")
           message.thread = thread.id
         }
         return Promise.resolve(message)

packages/nylas-sync/app.js

@@ -1,16 +1,18 @@
+// require('newrelic');
 global.Promise = require('bluebird');
-
-const {DatabaseConnector} = require(`nylas-core`)
+const {DatabaseConnector, Logger} = require(`nylas-core`)
 const SyncProcessManager = require('./sync-process-manager');
 
+global.Logger = Logger.createLogger('nylas-k2-sync')
+
 const manager = new SyncProcessManager();
 
 DatabaseConnector.forShared().then((db) => {
   const {Account} = db;
   Account.findAll().then((accounts) => {
     if (accounts.length === 0) {
-      console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`)
-      console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`)
+      global.Logger.info(`Couldn't find any accounts to sync. Run this CURL command to auth one!`)
+      global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`)
     }
     manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => {
       manager.start();

packages/nylas-sync/imap/fetch-folder-list.js

@@ -3,8 +3,9 @@ const {Provider} = require('nylas-core');
 const GMAIL_FOLDERS = ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam'];
 
 class FetchFolderList {
-  constructor(provider) {
+  constructor(provider, logger = console) {
     this._provider = provider;
+    this._logger = logger;
   }
 
   description() {

packages/nylas-sync/imap/fetch-messages-in-folder.js

@@ -11,12 +11,13 @@ const FETCH_MESSAGES_FIRST_COUNT = 100;
 const FETCH_MESSAGES_COUNT = 200;
 
 class FetchMessagesInFolder {
-  constructor(category, options) {
+  constructor(category, options, logger = console) {
     this._imap = null
     this._box = null
     this._db = null
     this._category = category;
     this._options = options;
+    this._logger = logger;
     if (!this._category) {
       throw new Error("FetchMessagesInFolder requires a category")
     }
@@ -86,9 +87,13 @@ class FetchMessagesInFolder {
         }
       })
 
-      console.log(` --- found ${flagChangeMessages.length || 'no'} flag changes`)
+      this._logger.info({
+        flag_changes: flagChangeMessages.length,
+      }, `FetchMessagesInFolder: found flag changes`)
       if (createdUIDs.length > 0) {
-        console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
+        this._logger.info({
+          new_messages: createdUIDs.length,
+        }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
       }
 
       if (flagChangeMessages.length === 0) {
@@ -111,7 +116,9 @@ class FetchMessagesInFolder {
       .filter(msg => !remoteUIDAttributes[msg.folderImapUID])
       .map(msg => msg.folderImapUID)
 
-    console.log(` --- found ${removedUIDs.length} messages no longer in the folder`)
+      this._logger.info({
+        removed_messages: removedUIDs.length,
+      }, `FetchMessagesInFolder: found messages no longer in the folder`)
 
     if (removedUIDs.length === 0) {
       return Promise.resolve();
@@ -148,7 +155,9 @@ class FetchMessagesInFolder {
     }
 
     if (desired.length === 0) {
-      console.warn(`Could not find good part. Options are: ${available.join(', ')}`)
+      this._logger.warn({
+        available_options: available.join(', '),
+      }, `FetchMessagesInFolder: Could not find good part`)
     }
 
     return desired;
@@ -173,7 +182,10 @@ class FetchMessagesInFolder {
         const uids = uidsByPart[key];
         const desiredParts = JSON.parse(key);
         const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
-        console.log(`Fetching parts ${key} for ${uids.length} messages`)
+        this._logger.info({
+          key,
+          num_messages: uids.length,
+        }, `FetchMessagesInFolder: Fetching parts for messages`)
 
         // note: the order of UIDs in the array doesn't matter, Gmail always
         // returns them in ascending (oldest => newest) order.
@@ -258,11 +270,17 @@ class FetchMessagesInFolder {
     )
     .then((message) => {
       if (created) {
-        console.log(`Created message ID: ${message.id}, UID: ${attributes.uid}`)
+        this._logger.info({
+          message_id: message.id,
+          uid: attributes.uid,
+        }, `FetchMessagesInFolder: Created message`)
         this._createFilesFromStruct({message, struct: attributes.struct})
         PubsubConnector.queueProcessMessage({accountId, messageId: message.id});
       } else {
-        console.log(`Updated message ID: ${message.id}, UID: ${attributes.uid}`)
+        this._logger.info({
+          message_id: message.id,
+          uid: attributes.uid,
+        }, `FetchMessagesInFolder: Updated message`)
       }
     })
 
@@ -291,7 +309,9 @@ class FetchMessagesInFolder {
 
     const desiredRanges = [];
 
-    console.log(` - Fetching messages. Currently have range: ${savedSyncState.fetchedmin}:${savedSyncState.fetchedmax}`)
+    this._logger.info({
+      range: `${savedSyncState.fetchedmin}:${savedSyncState.fetchedmax}`,
+    }, `FetchMessagesInFolder: Fetching messages.`)
 
     // Todo: In the future, this is where logic should go that limits
     // sync based on number of messages / age of messages.
@@ -303,18 +323,20 @@ class FetchMessagesInFolder {
       if (savedSyncState.fetchedmax < boxUidnext) {
         desiredRanges.push({min: savedSyncState.fetchedmax, max: boxUidnext})
       } else {
-        console.log(" --- fetchedmax == uidnext, nothing more recent to fetch.")
+        this._logger.info('FetchMessagesInFolder: fetchedmax == uidnext, nothing more recent to fetch.')
       }
       if (savedSyncState.fetchedmin > 1) {
         const lowerbound = Math.max(1, savedSyncState.fetchedmin - FETCH_MESSAGES_COUNT);
         desiredRanges.push({min: lowerbound, max: savedSyncState.fetchedmin})
       } else {
-        console.log(" --- fetchedmin == 1, nothing older to fetch.")
+        this._logger.info("FetchMessagesInFolder: fetchedmin == 1, nothing older to fetch.")
       }
     }
 
     return Promise.each(desiredRanges, ({min, max}) => {
-      console.log(` --- fetching range: ${min}:${max}`);
+      this._logger.info({
+        range: `${min}:${max}`,
+      }, `FetchMessagesInFolder: Fetching range`);
 
       return this._fetchMessagesAndQueueForProcessing(`${min}:${max}`).then(() => {
         const {fetchedmin, fetchedmax} = this._category.syncState;
@@ -326,7 +348,7 @@ class FetchMessagesInFolder {
         });
       })
     }).then(() => {
-      console.log(` - Fetching messages finished`);
+      this._logger.info(`FetchMessagesInFolder: Fetching messages finished`);
     });
   }
 
@@ -350,15 +372,15 @@ class FetchMessagesInFolder {
     let shallowFetch = null;
 
     if (this._imap.serverSupports(Capabilities.Condstore)) {
-      console.log(` - Shallow attribute scan (using CONDSTORE)`)
+      this._logger.info(`FetchMessagesInFolder: Shallow attribute scan (using CONDSTORE)`)
       if (nextHighestmodseq === highestmodseq) {
-        console.log(" --- highestmodseq matches, nothing more to fetch")
+        this._logger.info('FetchMessagesInFolder: highestmodseq matches, nothing more to fetch')
         return Promise.resolve();
       }
       shallowFetch = this._box.fetchUIDAttributes(`1:*`, {changedsince: highestmodseq});
     } else {
       const range = `${this._getLowerBoundUID(SHALLOW_SCAN_UID_COUNT)}:*`;
-      console.log(` - Shallow attribute scan (using range: ${range})`)
+      this._logger.info({range}, `FetchMessagesInFolder: Shallow attribute scan`)
       shallowFetch = this._box.fetchUIDAttributes(range);
     }
 
@@ -372,7 +394,7 @@ class FetchMessagesInFolder {
         this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes)
       ))
       .then(() => {
-        console.log(` - finished fetching changes to messages`);
+        this._logger.info(`FetchMessagesInFolder: finished fetching changes to messages`);
         return this.updateFolderSyncState({
           highestmodseq: nextHighestmodseq,
           timeShallowScan: Date.now(),
@@ -386,7 +408,7 @@ class FetchMessagesInFolder {
     const {fetchedmin, fetchedmax} = this._category.syncState;
     const range = `${fetchedmin}:${fetchedmax}`;
 
-    console.log(` - Deep attribute scan: fetching attributes in range: ${range}`)
+    this._logger.info({range}, `FetchMessagesInFolder: Deep attribute scan: fetching attributes in range`)
 
     return this._box.fetchUIDAttributes(range)
     .then((remoteUIDAttributes) => {
@@ -401,7 +423,7 @@ class FetchMessagesInFolder {
         })
       ))
       .then(() => {
-        console.log(` - Deep scan finished.`);
+        this._logger.info(`FetchMessagesInFolder: Deep scan finished.`);
         return this.updateFolderSyncState({
           highestmodseq: this._box.highestmodseq,
           timeDeepScan: Date.now(),

packages/nylas-sync/newrelic.js

@@ -0,0 +1,24 @@
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+  /**
+   * Array of application names.
+   */
+  app_name: ['Nylas K2 Sync'],
+  /**
+   * Your New Relic license key.
+   */
+  license_key: 'e232d6ccc786bd87aa72b86782439710162e3739',
+  logging: {
+    /**
+     * Level at which to log. 'trace' is most useful to New Relic when diagnosing
+     * issues with the agent, 'info' and higher will impose the least overhead on
+     * production applications.
+     */
+    level: 'info',
+  },
+}

packages/nylas-sync/sync-process-manager.js

@@ -40,10 +40,11 @@ class SyncProcessManager {
     this._workers = {};
     this._listenForSyncsClient = null;
     this._exiting = false;
+    this._logger = global.Logger.child({identity: IDENTITY})
   }
 
   start() {
-    console.log(`ProcessManager: Starting with ID ${IDENTITY}`)
+    this._logger.info(`ProcessManager: Starting with ID`)
 
     this.unassignAccountsAssignedTo(IDENTITY).then(() => {
       this.unassignAccountsMissingHeartbeats();
@@ -63,12 +64,14 @@ class SyncProcessManager {
     client.setAsync(key, Date.now()).then(() =>
       client.expireAsync(key, HEARTBEAT_EXPIRES)
     ).then(() =>
-      console.log("ProcessManager: 💘")
+    this._logger.info({
+      accounts_syncing_count: Object.keys(this._workers).length,
+    }, "ProcessManager: 💘")
     )
   }
 
   onSigInt() {
-    console.log(`ProcessManager: Exiting...`)
+    this._logger.info(`ProcessManager: Exiting...`)
     this._exiting = true;
 
     this.unassignAccountsAssignedTo(IDENTITY).then(() =>
@@ -85,7 +88,7 @@ class SyncProcessManager {
 
     let unseenIds = [].concat(accountIds);
 
-    console.log("ProcessManager: Starting scan for accountIds in database that are not present in Redis.")
+    this._logger.info("ProcessManager: Starting scan for accountIds in database that are not present in Redis.")
 
     return forEachAccountList((foundProcessIdentity, foundIds) => {
       unseenIds = unseenIds.filter((a) => !foundIds.includes(`${a}`))
@@ -94,7 +97,10 @@ class SyncProcessManager {
       if (unseenIds.length === 0) {
         return;
       }
-      console.log(`ProcessManager: Adding account IDs ${unseenIds.join(',')} to ${ACCOUNTS_UNCLAIMED}.`)
+      this._logger.info({
+        unseen_ids: unseenIds.join(', '),
+        channel: ACCOUNTS_UNCLAIMED,
+      }, `ProcessManager: Adding unseen account IDs to ACCOUNTS_UNCLAIMED channel.`)
       unseenIds.map((id) => client.lpushAsync(ACCOUNTS_UNCLAIMED, id));
     });
   }
@@ -102,7 +108,7 @@ class SyncProcessManager {
   unassignAccountsMissingHeartbeats() {
     const client = PubsubConnector.broadcastClient();
 
-    console.log("ProcessManager: Starting unassignment for processes missing heartbeats.")
+    this._logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.")
 
     Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
       const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, '');
@@ -125,12 +131,15 @@ class SyncProcessManager {
       )
 
     return unassignOne(0).then((returned) => {
-      console.log(`ProcessManager: Returned ${returned} accounts assigned to ${identity}.`)
+      this._logger.info({
+        returned,
+        assigned_to: identity,
+      }, `ProcessManager: Returned accounts`)
     });
   }
 
   update() {
-    console.log(`ProcessManager: Searching for an unclaimed account to sync.`)
+    this._logger.info(`ProcessManager: Searching for an unclaimed account to sync.`)
 
     this.acceptUnclaimedAccount().finally(() => {
       if (this._exiting) {
@@ -170,7 +179,7 @@ class SyncProcessManager {
           if (this._exiting || this._workers[account.id]) {
             return;
           }
-          console.log(`ProcessManager: Starting worker for Account ${accountId}`)
+          this._logger.info({account_id: accountId}, `ProcessManager: Starting worker for Account`)
           this._workers[account.id] = new SyncWorker(account, db, () => {
             this.removeWorkerForAccountId(accountId)
           });
@@ -187,7 +196,8 @@ class SyncProcessManager {
       if (didRemove) {
         PubsubConnector.broadcastClient().rpushAsync(dst, accountId)
       } else {
-        throw new Error("Wanted to return item to pool, but didn't have claim on it.")
+        this._logger.error("Wanted to return item to pool, but didn't have claim on it.")
+        return
       }
       this._workers[accountId] = null;
     });

packages/nylas-sync/sync-worker.js

@@ -11,8 +11,8 @@ const {
 
 const {CLAIM_DURATION} = SchedulerUtils;
 
-const FetchFolderList = require('./imap/fetch-category-list')
-const FetchMessagesInFolder = require('./imap/fetch-messages-in-category')
+const FetchFolderList = require('./imap/fetch-folder-list')
+const FetchMessagesInFolder = require('./imap/fetch-messages-in-folder')
 const SyncbackTaskFactory = require('./syncback-task-factory')
 
 
@@ -24,6 +24,7 @@ class SyncWorker {
     this._startTime = Date.now();
     this._lastSyncTime = null;
     this._onExpired = onExpired;
+    this._logger = global.Logger.forAccount(account)
 
     this._syncTimer = null;
     this._expirationTimer = null;
@@ -54,8 +55,11 @@ class SyncWorker {
         this._onAccountUpdated(); break;
       case MessageTypes.SYNCBACK_REQUESTED:
         this.syncNow({reason: 'Syncback Action Queued'}); break;
+      case MessageTypes.ACCOUNT_CREATED:
+        // No other processing currently required for account creation
+        break;
       default:
-        throw new Error(`Invalid message: ${msg}`)
+        this._logger.error({message: msg}, 'SyncWorker: Invalid message')
     }
   }
 
@@ -63,10 +67,14 @@ class SyncWorker {
     if (!this.isWaitingForNextSync()) {
       return;
     }
-    this._getAccount().then((account) => {
+    this._getAccount()
+    .then((account) => {
       this._account = account;
       this.syncNow({reason: 'Account Modification'});
-    });
+    })
+    .catch((err) => {
+      this._logger.error(err, 'SyncWorker: Error getting account for update')
+    })
   }
 
   _onConnectionIdleUpdate() {
@@ -100,7 +108,12 @@ class SyncWorker {
       return Promise.reject(new Error("ensureConnection: There are no IMAP connection credentials for this account."))
     }
 
-    const conn = new IMAPConnection(this._db, Object.assign({}, settings, credentials));
+    const conn = new IMAPConnection({
+      db: this._db,
+      settings: Object.assign({}, settings, credentials),
+      logger: this._logger,
+    });
+
     conn.on('mail', () => {
       this._onConnectionIdleUpdate();
     })
@@ -145,7 +158,7 @@ class SyncWorker {
       )
 
       return Promise.all(categoriesToSync.map((cat) =>
-        this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions))
+        this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions, this._logger))
       ))
     });
   }
@@ -155,10 +168,10 @@ class SyncWorker {
     this._syncTimer = null;
 
     if (!process.env.SYNC_AFTER_ERRORS && this._account.errored()) {
-      console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) is in error state - Skipping sync`)
+      this._logger.info(`SyncWorker: Account is in error state - Skipping sync`)
       return
     }
-    console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) sync started (${reason})`)
+    this._logger.info({reason}, `SyncWorker: Account sync started`)
 
     this.ensureConnection()
     .then(() => this._account.update({syncError: null}))
@@ -174,7 +187,7 @@ class SyncWorker {
   }
 
   onSyncError(error) {
-    console.error(`SyncWorker: Error while syncing account ${this._account.emailAddress} (${this._account.id})`, error)
+    this._logger.error(error, `SyncWorker: Error while syncing account`)
     this.closeConnection()
 
     if (error.source.includes('socket') || error.source.includes('timeout')) {
@@ -189,8 +202,8 @@ class SyncWorker {
   onSyncDidComplete() {
     const {afterSync} = this._account.syncPolicy;
 
-    if (!this._account.firstSyncCompletedAt) {
-      this._account.firstSyncCompletedAt = Date.now()
+    if (!this._account.firstSyncCompletion) {
+      this._account.firstSyncCompletion = Date.now()
     }
 
     const now = Date.now();
@@ -203,21 +216,22 @@ class SyncWorker {
 
     this._account.lastSyncCompletions = lastSyncCompletions
     this._account.save()
-    console.log('Syncworker: Completed sync cycle')
+    this._logger.info('Syncworker: Completed sync cycle')
 
     if (afterSync === 'idle') {
       return this._getIdleFolder()
       .then((idleFolder) => this._conn.openBox(idleFolder.name))
-      .then(() => console.log('SyncWorker: - Idling on inbox category'))
+      .then(() => this._logger.info('SyncWorker: Idling on inbox category'))
     }
 
     if (afterSync === 'close') {
-      console.log('SyncWorker: - Closing connection');
+      this._logger.info('SyncWorker: Closing connection');
       this.closeConnection()
       return Promise.resolve()
     }
 
-    throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`)
+    this._logger.error({after_sync: afterSync}, `SyncWorker.onSyncDidComplete: Unknown afterSync behavior`)
+    throw new Error('SyncWorker.onSyncDidComplete: Unknown afterSync behavior')
   }
 
   isWaitingForNextSync() {
@@ -226,7 +240,7 @@ class SyncWorker {
 
   scheduleNextSync() {
     if (Date.now() - this._startTime > CLAIM_DURATION) {
-      console.log("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool.");
+      this._logger.info("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool.");
       this.cleanup();
       this._onExpired();
       return;
@@ -238,7 +252,10 @@ class SyncWorker {
 
       if (interval) {
         const target = this._lastSyncTime + interval;
-        console.log(`SyncWorker: Account ${active ? 'active' : 'inactive'}. Next sync scheduled for ${new Date(target).toLocaleString()}`);
+        this._logger.info({
+          is_active: active,
+          next_sync: new Date(target).toLocaleString(),
+        }, `SyncWorker: Next sync scheduled`);
         this._syncTimer = setTimeout(() => {
           this.syncNow({reason: 'Scheduled'});
         }, target - Date.now());

pm2-dev.yml

@@ -4,6 +4,7 @@ apps:
 
 
   - script   : packages/nylas-api/app.js
+    watch    : ["packages"]
     name     : api
     env      :
       PORT: 5100
@@ -12,25 +13,26 @@ apps:
       GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com"
       GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF"
       GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback"
-
-
+      NODE_ENV: 'development'
   - script   : packages/nylas-sync/app.js
+    watch    : ["packages"]
     name     : sync
     env      :
       DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
       DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-
-
+      NODE_ENV: 'development'
   - script   : packages/nylas-dashboard/app.js
+    watch    : ["packages"]
     name     : dashboard
     env      :
       PORT: 5101
       DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
       DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-
-
+      NODE_ENV: 'development'
   - script   : packages/nylas-message-processor/app.js
+    watch    : ["packages"]
     name     : processor
     env      :
       DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
       DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
+      NODE_ENV: 'development'