Merge branch 'master' of ssh://github.com/nylas/k2

Conflicts:
	Dockerfile

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.gitignore
+README.md
+Procfile*
+*node_modules*
+docs

Dockerfile

@@ -1,9 +1,25 @@
+# This Dockerfile builds a production-ready image of K2 to be used across all
+# services. See the Dockerfile documentation here:
+# https://docs.docker.com/engine/reference/builder/
+
+# Use the latest Node 6 base docker image
 # https://github.com/nodejs/docker-node
 FROM node:6
-RUN mkdir -p /usr/src/app
+
-WORKDIR /usr/src/app
+# Copy everything (excluding what's in .dockerignore) into an empty dir
-COPY package.json /usr/src/app/
+COPY . /home
-RUN npm install
+WORKDIR /home
-COPY . /usr/src/app
+
-EXPOSE 8080
+RUN npm install --production
+
+# This will do an `npm install` for each of our modules and then link them all
+# together. See more about Lerna here: https://github.com/lerna/lerna We have
+# to run this separately from npm postinstall due to permission issues.
+RUN node_modules/.bin/lerna bootstrap
+
+# External services run on port 5100. Expose it.
+EXPOSE 5100
+
+# We use a start-aws command that automatically spawns the correct process
+# based on environment variables (which changes instance to instance)
 CMD [ "./node_modules/pm2/bin/pm2", "start", "./pm2-prod-${AWS_SERVICE_NAME}.yml"]

README.md

@@ -1,23 +1,33 @@
 # K2 - Sync Engine Experiment
 
-# Initial Setup
+# Initial Setup:
 
-1. Download https://toolbelt.heroku.com/
+## New Computer (Mac):
 
-```
+1. Install [Homebrew](http://brew.sh/)
-brew install redis
+1. Install [VirtualBox 5+](https://www.virtualbox.org/wiki/Downloads)
-nvm install 6
+1. Install [Docker for Mac](https://docs.docker.com/docker-for-mac/)
-npm install
+1. Install [NVM](https://github.com/creationix/nvm) `brew install nvm`
-```
+1. Install Node 6+ via NVM: `nvm install 6`
 
-# Running locally
+## New to AWS:
+
+1. Install [Elastic Beanstalk CLI](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html#eb-cli3-install-osx): `brew install awsebcli`
+1. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli`
+  1. Add your AWS IAM Security Credentials to `aws configure`.
+  1. These are at Console Home -> IAM -> Users -> {{Your Name}} -> Security
+     Credentials. Note that your private key was only shown unpon creation. If
+     you've lost your private key you have to deactivate your old key and
+     create a new one.
+1. Get the K2 team private SSH key. (Ignore this when we have a Bastion Host). Ask someone on K2 for a copy of the private SSH key. Copy it to your ~/.ssh folder.
+  1. `chmod 400 ~/.ssh/k2-keypair.pem`
+  1. `ssh i ~/.ssh/k2-keypair.pem some-ec2-box-we-own.amazonaws.com`
+1. Connect to Elastic Beanstalk instances: `eb init`. Select correct region. Select correct application.
+
+# Developing Locally:
 
 ```
 npm start
 ```
 
-## Auth an account
+# Deploying
-
-```
-curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-```

package.json

@@ -5,13 +5,13 @@
   "main": "",
   "dependencies": {
     "bluebird": "3.x.x",
+    "lerna": "2.0.0-beta.23",
     "mysql": "^2.11.1",
     "newrelic": "^1.28.1",
     "pm2": "^1.1.3",
     "redis": "2.x.x",
     "rx": "4.x.x",
     "sequelize": "3.x.x",
-    "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz",
     "underscore": "1.x.x"
   },
   "devDependencies": {
@@ -22,7 +22,7 @@
     "eslint-plugin-jsx-a11y": "1.x",
     "eslint-plugin-react": "5.x",
     "eslint_d": "3.x",
-    "lerna": "2.0.0-beta.23"
+    "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz"
   },
   "scripts": {
     "start": "./node_modules/pm2/bin/pm2 start ./pm2-dev.yml  --no-daemon",

packages/nylas-api/app.js

@@ -31,30 +31,38 @@ const plugins = [Inert, Vision, HapiBasicAuth, HapiBoom, {
 }];
 
 let sharedDb = null;
-const {DatabaseConnector, SchedulerUtils} = require(`nylas-core`)
-DatabaseConnector.forShared().then((db) => {
-  sharedDb = db;
-});
 
 const validate = (request, username, password, callback) => {
-  const {AccountToken} = sharedDb;
+  const {DatabaseConnector, SchedulerUtils} = require(`nylas-core`);
 
-  AccountToken.find({
+  let getSharedDb = null;
-    where: {
+  if (sharedDb) {
-      value: username,
+    getSharedDb = Promise.resolve(sharedDb)
-    },
+  } else {
-  }).then((token) => {
+    getSharedDb = DatabaseConnector.forShared()
-    if (!token) {
+  }
-      callback(null, false, {});
+
-      return
+  getSharedDb.then((db) => {
-    }
+    sharedDb = db;
-    token.getAccount().then((account) => {
+    const {AccountToken} = db;
-      if (!account) {
+
+    AccountToken.find({
+      where: {
+        value: username,
+      },
+    }).then((token) => {
+      if (!token) {
         callback(null, false, {});
-        return;
+        return
       }
-      SchedulerUtils.markAccountIsActive(account.id)
+      token.getAccount().then((account) => {
-      callback(null, true, account);
+        if (!account) {
+          callback(null, false, {});
+          return;
+        }
+        SchedulerUtils.markAccountIsActive(account.id)
+        callback(null, true, account);
+      });
     });
   });
 };

packages/nylas-core/models/shared/account.js

@@ -33,6 +33,7 @@ module.exports = (sequelize, Sequelize) => {
           sync_error: this.syncError,
           first_sync_completed_at: this.firstSyncCompletedAt,
           last_sync_completions: this.lastSyncCompletions,
+          created_at: this.createdAt,
         }
       },
 

packages/nylas-dashboard/public/css/app.css

@@ -1,6 +1,9 @@
 body {
+  background-image: url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg);
+  background-image: -moz-linear-gradient(top, rgba(232, 244, 250, 0.2), rgba(231, 231, 233, 1)), url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg);
   background-image: -webkit-linear-gradient(top, rgba(232, 244, 250, 0.2), rgba(231, 231, 233, 1)), url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg);
-  background-size: cover;
+  background-size: 100vw auto;
+  background-attachment: fixed;
   font-family: Roboto, sans-serif;
   font-size: 12px;
 }
@@ -51,6 +54,7 @@ pre {
 #set-all-sync {
   display: block;
   margin-bottom: 10px;
+  color: #ffffff;
 }
 
 .action-link {
@@ -83,3 +87,13 @@ pre {
   background-color: rgba(0, 0, 0, 0.3);
   padding-top: 10%;
 }
+
+.sync-graph {
+  margin-top: 3px;
+}
+
+.stats b {
+  display: inline-block;
+  margin-top: 5px;
+  margin-bottom: 1px;
+}

packages/nylas-dashboard/public/index.html

@@ -6,6 +6,7 @@
   <script src="/js/sync-policy.jsx" type="text/babel"></script>
   <script src="/js/set-all-sync-policies.jsx" type="text/babel"></script>
   <script src="/js/account-filter.jsx" type="text/babel"></script>
+  <script src="/js/sync-graph.jsx" type="text/babel"></script>
   <script src="/js/app.jsx" type="text/babel"></script>
   <link rel='stylesheet' type="text/css" href="./css/app.css" />
   <link rel='shortcut icon' href='favicon.png' / >

packages/nylas-dashboard/public/js/app.jsx

@@ -1,7 +1,12 @@
 /* eslint react/react-in-jsx-scope: 0*/
 const React = window.React;
 const ReactDOM = window.ReactDOM;
-const {SyncPolicy, SetAllSyncPolicies, AccountFilter} = window;
+const {
+  SyncPolicy,
+  SetAllSyncPolicies,
+  AccountFilter,
+  SyncGraph,
+} = window;
 
 class Account extends React.Component {
   renderError() {
@@ -14,10 +19,13 @@ class Account extends React.Component {
         stack: stack.slice(0, 4),
       }
       return (
-        <div className="error">
+        <div>
-          <pre>
+          <div className="section">Error</div>
-            {JSON.stringify(error, null, 2)}
+          <div className="error">
-          </pre>
+            <pre>
+              {JSON.stringify(error, null, 2)}
+            </pre>
+          </div>
         </div>
       )
     }
@@ -27,12 +35,18 @@ class Account extends React.Component {
   render() {
     const {account, assignment, active} = this.props;
     const errorClass = account.sync_error ? ' errored' : ''
-    const lastSyncCompletions = []
+
-    for (const time of account.last_sync_completions) {
+    const numStoredSyncs = account.last_sync_completions.length;
-      lastSyncCompletions.push(
+    const oldestSync = account.last_sync_completions[numStoredSyncs - 1];
-        <div key={time}>{new Date(time).toString()}</div>
+    const newestSync = account.last_sync_completions[0];
-      )
+    const avgBetweenSyncs = (newestSync - oldestSync) / (1000 * numStoredSyncs);
+    const timeSinceLastSync = (Date.now() - newestSync) / 1000;
+
+    let firstSyncDuration = "Incomplete";
+    if (account.first_sync_completed_at) {
+      firstSyncDuration = (new Date(account.first_sync_completed_at) - new Date(account.created_at)) / 1000;
     }
+
     return (
       <div className={`account${errorClass}`}>
         <h3>{account.email_address} {active ? '🌕' : '🌑'}</h3>
@@ -42,13 +56,16 @@ class Account extends React.Component {
           stringifiedSyncPolicy={JSON.stringify(account.sync_policy, null, 2)}
         />
         <div className="section">Sync Cycles</div>
-        <div>
+        <div className="stats">
-          <b>First Sync Completion</b>:
+          <b>First Sync Duration (seconds)</b>:
-          <pre>{new Date(account.first_sync_completed_at).toString()}</pre>
+          <pre>{firstSyncDuration}</pre>
+          <b> Average Time Between Syncs (seconds)</b>:
+          <pre>{avgBetweenSyncs}</pre>
+          <b>Time Since Last Sync (seconds)</b>:
+          <pre>{timeSinceLastSync}</pre>
+          <b>Recent Syncs</b>:
+          <SyncGraph id={account.last_sync_completions.length} syncTimestamps={account.last_sync_completions} />
         </div>
-        <div><b>Last Sync Completions:</b></div>
-        <pre>{lastSyncCompletions}</pre>
-        <div className="section">Error</div>
         {this.renderError()}
       </div>
     );

packages/nylas-dashboard/public/js/sync-graph.jsx

@@ -0,0 +1,92 @@
+const React = window.React;
+const ReactDOM = window.ReactDOM;
+
+class SyncGraph extends React.Component {
+
+  componentDidMount() {
+    this.drawGraph();
+  }
+
+  componentDidUpdate() {
+    this.drawGraph(true);
+  }
+
+  drawGraph(isUpdate) {
+    const now = Date.now();
+    const config = SyncGraph.config;
+    const context = ReactDOM.findDOMNode(this).getContext('2d');
+
+    // Background
+    // (This hides any previous data points, so we don't have to clear the canvas)
+    context.fillStyle = config.backgroundColor;
+    context.fillRect(0, 0, config.width, config.height);
+
+    // Data points
+    const pxPerSec = config.width / config.timeLength;
+    context.strokeStyle = config.dataColor;
+    context.beginPath();
+    for (const syncTimeMs of this.props.syncTimestamps) {
+      const secsAgo = (now - syncTimeMs) / 1000;
+      const pxFromRight = secsAgo * pxPerSec;
+      const pxFromLeft = config.width - pxFromRight;
+      context.moveTo(pxFromLeft, 0);
+      context.lineTo(pxFromLeft, config.height);
+    }
+    context.stroke();
+
+    // Tick marks
+    const interval = config.width / config.numTicks;
+    context.strokeStyle = config.tickColor;
+    context.beginPath();
+    for (let px = interval; px < config.width; px += interval) {
+      context.moveTo(px, config.height - config.tickHeight);
+      context.lineTo(px, config.height);
+    }
+    context.stroke();
+
+    // Axis labels
+    if (!isUpdate) { // only draw these on the initial render
+      context.fillStyle = config.labelColor;
+      context.font = `${config.labelFontSize}px sans-serif`;
+      const fontY = config.height + config.labelFontSize + config.labelTopMargin;
+      const nowText = "now";
+      const nowWidth = context.measureText(nowText).width;
+      context.fillText(nowText, config.width - nowWidth - 1, fontY);
+      context.fillText("-30m", 1, fontY);
+    }
+  }
+
+  render() {
+    return (
+      <canvas
+        width={SyncGraph.config.width}
+        height={SyncGraph.config.height + SyncGraph.config.labelFontSize + SyncGraph.config.labelTopMargin}
+        className="sync-graph"
+        syncTimestamps={this.props.syncTimestamps}
+      ></canvas>
+    )
+  }
+
+}
+
+SyncGraph.config = {
+  height: 50, // Doesn't include labels
+  width: 300,
+  // timeLength is 30 minutes in seconds. If you change this, be sure to update
+  // syncGraphTimeLength in sync-worker.js and the axis labels in drawGraph()!
+  timeLength: 60 * 30,
+  numTicks: 10,
+  tickHeight: 10,
+  tickColor: 'white',
+  labelFontSize: 8,
+  labelTopMargin: 2,
+  labelColor: 'black',
+  backgroundColor: 'black',
+  dataColor: 'blue',
+}
+
+SyncGraph.propTypes = {
+  syncTimestamps: React.PropTypes.arrayOf(React.PropTypes.number),
+}
+
+window.SyncGraph = SyncGraph;

packages/nylas-sync/sync-worker.js

@@ -189,11 +189,14 @@ class SyncWorker {
       this._account.firstSyncCompletedAt = Date.now()
     }
 
+    const now = Date.now();
+    const syncGraphTimeLength = 60 * 30; // 30 minutes, should be the same as SyncGraph.config.timeLength
     let lastSyncCompletions = [...this._account.lastSyncCompletions]
-    lastSyncCompletions = [Date.now(), ...lastSyncCompletions]
+    lastSyncCompletions = [now, ...lastSyncCompletions]
-    if (lastSyncCompletions.length > 10) {
+    while (now - lastSyncCompletions[lastSyncCompletions.length - 1] > 1000 * syncGraphTimeLength) {
-      lastSyncCompletions.pop()
+      lastSyncCompletions.pop();
     }
+
     this._account.lastSyncCompletions = lastSyncCompletions
     this._account.save()
     console.log('Syncworker: Completed sync cycle')