From 27ba2544ddd837b1bfa998f720f0708139bcb1c3 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 22 Nov 2016 10:57:52 -0800 Subject: [PATCH] Pull in cloud packages --- README.md | 20 +++++++++++ package.json | 2 -- packages/cloud-metrics/index.js | 51 +++++++++++++++++++++++++++++ packages/cloud-metrics/package.json | 11 +++++++ packages/nylas-metrics/index.js | 4 --- pm2-dev.yml | 23 ++----------- pm2-prod-api.yml | 2 +- pm2-prod-dashboard.yml | 7 ---- pm2-prod-sync.yml | 9 ----- pm2-prod-workers.yml | 5 +++ 10 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 packages/cloud-metrics/index.js create mode 100644 packages/cloud-metrics/package.json delete mode 100644 pm2-prod-dashboard.yml delete mode 100644 pm2-prod-sync.yml create mode 100644 pm2-prod-workers.yml diff --git a/README.md b/README.md index be11dc1c5..6e354aa12 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,26 @@ ``` npm install npm start + +## New to AWS: + +1. Install [Elastic Beanstalk CLI](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html#eb-cli3-install-osx): `sudo pip install awsebcli` + 1. On Linux, you may need to install Python 3's pip via `sudo apt-get install python3-pip` and then run `pip3 install --user awsebcli`. This installs to your home directory and you need to have `~/.local/bin` in your $PATH. +2. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli` on Mac and `pip install --user awscli` on Linux. + 1. Add your AWS IAM Security Credentials to `aws configure`. + 1. These are at Console Home -> IAM -> Users -> {{Your Name}} -> Security + Credentials. Note that your private key was only shown unpon creation. If + you've lost your private key you have to deactivate your old key and + create a new one. +3. Get the K2 team private SSH key. (Ignore this when we have a Bastion Host). Ask someone on K2 for a copy of the private SSH key. Copy it to your ~/.ssh folder. + 1. `chmod 400 ~/.ssh/k2-keypair.pem` + 1. `ssh i ~/.ssh/k2-keypair.pem some-ec2-box-we-own.amazonaws.com` +4. Connect to Elastic Beanstalk instances: `eb init`. Select correct region. Select correct application. + +# Developing the Cloud Components Locally: + +``` +pm2 start ./pm2-dev.yml --no-daemon ``` We use [pm2](http://pm2.keymetrics.io/) to launch a variety of processes diff --git a/package.json b/package.json index 1fe6f6a0f..fccdab67a 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,6 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "pm2 start ./pm2-dev.yml --no-daemon", - "stop": "pm2 kill", "postinstall": "lerna bootstrap" }, "repository": { diff --git a/packages/cloud-metrics/index.js b/packages/cloud-metrics/index.js new file mode 100644 index 000000000..91abbf22a --- /dev/null +++ b/packages/cloud-metrics/index.js @@ -0,0 +1,51 @@ +/* eslint global-require:0 */ +const {env: {NODE_ENV, SIGNALFX_TOKEN}, pid} = process +const os = require('os') +const signalfx = require('signalfx') + +let signalfxClient = null + +const MetricTypes = { + Gauge: 'gauges', + Counter: 'counters', + CumulativeCounter: 'cumulative_counters', +} +const shouldReport = NODE_ENV && NODE_ENV !== 'development' + + +const Metrics = { + + MetricTypes, + + startCapturing(name) { + if (!shouldReport) { return } + signalfxClient = new signalfx.Ingest(SIGNALFX_TOKEN, { + dimensions: { + name, + host: os.hostname(), + pid: pid.toString(), + env: NODE_ENV, + }, + }) + }, + + reportError(error) { + }, + + reportMetric({name, value, type, dimensions = {}} = {}) { + if (!signalfxClient || !shouldReport) { return } + if (!name) { + throw new Error('Metrics.reportMetric requires a metric.name') + } + if (value == null) { + throw new Error('Metrics.reportMetric requires a metric.value') + } + if (!type) { + throw new Error('Metrics.reportMetric requires a metric.type from Metrics.MetricTypes') + } + const metric = {metric: name, value, timestamp: Date.now(), dimensions} + signalfxClient.send({[type]: [metric]}) + }, +} + +module.exports = Metrics diff --git a/packages/cloud-metrics/package.json b/packages/cloud-metrics/package.json new file mode 100644 index 000000000..000860629 --- /dev/null +++ b/packages/cloud-metrics/package.json @@ -0,0 +1,11 @@ +{ + "name": "cloud-metrics", + "version": "0.0.1", + "description": "Metrics package", + "main": "index.js", + "dependencies": { + "signalfx": "3.0.1" + }, + "author": "Nylas", + "license": "ISC" +} diff --git a/packages/nylas-metrics/index.js b/packages/nylas-metrics/index.js index 1dbc70fa7..fcbc3c714 100644 --- a/packages/nylas-metrics/index.js +++ b/packages/nylas-metrics/index.js @@ -2,7 +2,6 @@ const {env: {NODE_ENV, SIGNALFX_TOKEN}, pid} = process const os = require('os') const signalfx = require('signalfx') -let newrelicClient = null let signalfxClient = null const MetricTypes = { @@ -19,7 +18,6 @@ const Metrics = { startCapturing(name) { if (!shouldReport) { return } - newrelicClient = require('newrelic') signalfxClient = new signalfx.Ingest(SIGNALFX_TOKEN, { dimensions: { name, @@ -31,8 +29,6 @@ const Metrics = { }, reportError(error) { - if (!newrelicClient || !shouldReport) { return } - newrelicClient.noticeError(error) }, reportMetric({name, value, type, dimensions = {}} = {}) { diff --git a/pm2-dev.yml b/pm2-dev.yml index 5401f2564..9d50d1fa7 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -1,35 +1,18 @@ apps: - - script : packages/nylas-api/app.js + - script : packages/cloud-api/app.js watch : ["packages"] name : api env : PORT: 5100 - ALLOW_LIST_ACCOUNTS : true DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF" GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" NODE_ENV: 'development' - - script : packages/nylas-sync/app.js + - script : packages/cloud-workers/app.js watch : ["packages"] - name : sync - env : - SYNC_AFTER_ERRORS : true - DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" - DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - NODE_ENV: 'development' - - script : packages/nylas-dashboard/app.js - watch : ["packages"] - name : dashboard - env : - PORT: 5101 - DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" - DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - NODE_ENV: 'development' - - script : packages/nylas-message-processor/app.js - watch : ["packages"] - name : processor + name : workers env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" diff --git a/pm2-prod-api.yml b/pm2-prod-api.yml index e6f679b56..300f35c43 100644 --- a/pm2-prod-api.yml +++ b/pm2-prod-api.yml @@ -1,5 +1,5 @@ apps: - - script : packages/nylas-api/app.js + - script : packages/cloud-api/app.js name : api instances: 0 exec_mode: cluster diff --git a/pm2-prod-dashboard.yml b/pm2-prod-dashboard.yml deleted file mode 100644 index 5f5e4234e..000000000 --- a/pm2-prod-dashboard.yml +++ /dev/null @@ -1,7 +0,0 @@ -apps: - - script : packages/nylas-dashboard/app.js - name : dashboard - instances: 0 - exec_mode: cluster - env : - PORT: 5100 diff --git a/pm2-prod-sync.yml b/pm2-prod-sync.yml deleted file mode 100644 index 1bcb4cfb4..000000000 --- a/pm2-prod-sync.yml +++ /dev/null @@ -1,9 +0,0 @@ -apps: - - script : packages/nylas-sync/app.js - name : sync - instances: 0 - exec_mode: fork - - script : packages/nylas-message-processor/app.js - name : processor - instances: 0 - exec_mode: fork diff --git a/pm2-prod-workers.yml b/pm2-prod-workers.yml new file mode 100644 index 000000000..f9af7e567 --- /dev/null +++ b/pm2-prod-workers.yml @@ -0,0 +1,5 @@ +apps: + - script : packages/cloud-workers/app.js + name : workers + instances: 0 + exec_mode: fork