[cloud-*] Get cloud-api and cloud-workers running on new repo

[dev] remove .tern-project

[cloud-*] move into cloud-core/build

[cloud-core] convert to launch properly via Docker

.dockerignore

@@ -1,20 +1,14 @@
-n1_cloud_dist
+.arc*
-
+.git*
-.git
-.arcconfig
-.arclint
 arclib
+**/node_modules
+packages/client-*
 
 *.swp
 *~
 .DS_Store
-node_modules
+**/npm-debug.log
-**/node_modules
+**/lerna-debug.log
-dump.rdb
-*npm-debug.log
-storage/
-lerna-debug.log
-newrelic_agent.log
 
 # Vim temp files
 *.swp
@@ -25,3 +19,6 @@ newrelic_agent.log
 !.elasticbeanstalk/*.cfg.yml
 !.elasticbeanstalk/*.global.yml
 /packages/client-sync/spec-saved-state.json
+
+# Built cloud files
+n1_cloud_dist

.gitignore

@@ -39,3 +39,6 @@ node_modules
 **/.DS_Store
 **/Thumbs.db
 **/#emacs
+
+# Built cloud files
+n1_cloud_dist

.tern-project

@@ -1,5 +0,0 @@
-{
-    "plugins": {
-        "node": {}
-    }
-}

Dockerfile

@@ -10,20 +10,16 @@ FROM node:6
 COPY . /home
 WORKDIR /home
 
+# This installs global dependencies, then in the postinstall script, runs lerna
+# bootstrap to install and link cloud-api, cloud-core, and cloud-workers
 RUN npm install --production
 
-# This will do an `npm install` for each of our modules and then link them all
-# together. See more about Lerna here: https://github.com/lerna/lerna We have
-# to run this separately from npm postinstall due to permission issues.
-RUN node_modules/.bin/lerna bootstrap
-
 # This uses babel to compile any es6 to stock js for plain node
-RUN npm run build-n1-cloud
+RUN node packages/cloud-core/build/build-n1-cloud
 
 # External services run on port 80. Expose it.
 EXPOSE 5100
 
 # We use a start-aws command that automatically spawns the correct process
 # based on environment variables (which changes instance to instance)
-#CMD ./node_modules/pm2/bin/pm2 start --no-daemon ./pm2-prod-${AWS_SERVICE_NAME}.yml
+CMD packages/cloud-core/_n1cloud_docker_launcher.sh ${AWS_SERVICE_NAME}
-CMD ./_n1cloud_docker_launcher.sh ${AWS_SERVICE_NAME}

_n1cloud_docker_launcher.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-./node_modules/pm2/bin/pm2 start ./pm2-prod-$1.yml
-./node_modules/pm2/bin/pm2 logs --raw

migrations/01-expirationDate-metadata.es6

@@ -1,11 +0,0 @@
-module.exports = {
-  up: async (queryInterface, Sequelize) => {
-    const {sequelize} = queryInterface;
-    console.log("querying db");
-    await sequelize.query("ALTER TABLE metadata ADD COLUMN `expiration` DATETIME");
-  },
-  down: async (queryInterface, Sequelize) => {
-    const {sequelize} = queryInterface;
-    await sequelize.query("ALTER TABLE metadata DROP COLUMN `expiration`");
-  },
-}

package.json

@@ -2,25 +2,31 @@
   "name": "nylas-mail-all",
   "version": "0.0.1",
   "description": "All components required to run Nylas Mail",
-  "devDependencies": {
+  "dependencies": {
     "babel-cli": "6.x.x",
     "babel-core": "6.x.x",
     "babel-eslint": "7.1.0",
     "babel-preset-electron": "1.4.15",
     "babel-preset-react": "6.x.x",
+    "pm2": "2.4.0",
+    "lerna": "2.0.0-beta.37",
+    "fs-plus": "2.x.x",
+    "fs-extra": "2.x.x",
+    "glob": "7.x.x",
+    "request": "2.x.x",
+    "underscore": "1.8.x"
+  },
+  "devDependencies": {
+    "chalk": "1.x.x",
+    "coffeelint-cjsx": "2.x.x",
+    "electron-installer-dmg": "0.2.x",
+    "electron-packager": "8.4.x",
     "eslint": "3.10.1",
     "eslint-config-airbnb": "13.0.0",
     "eslint-plugin-import": "2.2.0",
     "eslint-plugin-jsx-a11y": "2.2.3",
     "eslint-plugin-react": "6.7.1",
     "eslint_d": "4.2.0",
-    "fs-plus": "2.x.x",
-    "lerna": "2.0.0-beta.37",
-    "chalk": "1.x.x",
-    "coffeelint-cjsx": "2.x.x",
-    "electron-installer-dmg": "0.2.x",
-    "electron-packager": "8.4.x",
-    "glob": "7.x.x",
     "grunt": "0.4.x",
     "grunt-cli": "0.1.x",
     "grunt-coffeelint": "git+https://github.com/atom/grunt-coffeelint.git#cfb99aa99811d52687969532bd5a98011ed95bfe",
@@ -30,22 +36,25 @@
     "grunt-contrib-less": "0.8.x",
     "grunt-electron-installer": "2.x.x",
     "grunt-lesslint": "0.13.x",
-    "request": "2.x.x",
     "s3": "4.x.x",
-    "temp": "0.8.x",
+    "temp": "0.8.x"
-    "underscore": "1.8.x"
   },
   "scripts": {
     "start": "npm run start-client",
     "test": "npm run test-client",
+
     "start-client": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --dev",
-    "start-api": "",
-    "build-client": "",
-    "build-api": "",
     "test-client": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --test",
     "test-client-window": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --test=window",
     "test-client-junit": "",
-    "test-api": "",
+    "build-client": "",
+
+    "start-cloud": "pm2 stop all; pm2 delete all; pm2 start packages/cloud-core/pm2-dev.yml --no-daemon",
+    "test-cloud": "",
+    "stop": "npm run stop-cloud",
+    "stop-cloud": "pm2 stop all; pm2 delete all;",
+    "build-cloud": "",
+
     "postinstall": "babel-node scripts/postinstall.js"
   },
   "repository": {

pm2-debug-cloud-api.yml

@@ -1,26 +0,0 @@
-apps:
-  - script   : packages/cloud-api/app.es6
-    interpreter : node_modules/.bin/babel-node
-    interpreter_args: "--inspect"
-    watch    : ["packages"]
-    name     : api
-    env      :
-      PORT: 5100
-      DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
-      DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-      GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com"
-      GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF"
-      GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback"
-      NODE_ENV: 'development'
-      HONEY_DATASET: 'n1-cloud-staging'
-      HONEY_WRITE_KEY: 'XXXXXXXXXXXXX'
-  - script   : packages/cloud-workers/app.es6
-    interpreter : node_modules/.bin/babel-node
-    watch    : ["packages"]
-    name     : workers
-    env      :
-      DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
-      DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-      NODE_ENV: 'development'
-  - script   : redis-server
-    name     : redis

pm2-dev.yml

@@ -1,27 +0,0 @@
-apps:
-  - script   : packages/cloud-api/app.es6
-    interpreter : node_modules/.bin/babel-node
-    watch    : ["packages"]
-    name     : api
-    env      :
-      PORT: 5100
-      DB_NAME : "nylasmailclouddb"
-      DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
-      DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-      GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com"
-      GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF"
-      GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback"
-      NODE_ENV: 'development'
-      HONEY_DATASET: 'n1-cloud-staging'
-      HONEY_WRITE_KEY: 'XXXXXXXXXXXXX'
-  - script   : packages/cloud-workers/app.es6
-    interpreter : node_modules/.bin/babel-node
-    watch    : ["packages"]
-    name     : workers
-    env      :
-      DB_NAME : "nylasmailclouddb"
-      DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
-      DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
-      NODE_ENV: 'development'
-  - script   : scripts/run-redis.sh
-    name     : redis

pm2-prod-api.yml

@@ -1,7 +0,0 @@
-apps:
-  - script   : n1_cloud_dist/cloud-api/app.js
-    name     : api
-    instances: 0
-    exec_mode: cluster
-    env      :
-      PORT: 5100

pm2-prod-workers.yml

@@ -1,5 +0,0 @@
-apps:
-  - script   : n1_cloud_dist/cloud-workers/app.js
-    name     : workers
-    instances: 1
-    exec_mode: fork

scripts/build-n1-cloud.js

@@ -1,52 +0,0 @@
-const fs = require('fs-extra');
-const glob = require('glob');
-const path = require('path');
-const babel = require('babel-core');
-
-fs.removeSync("n1_cloud_dist")
-fs.copySync("packages/cloud-api", "n1_cloud_dist/cloud-api")
-fs.copySync("packages/cloud-workers", "n1_cloud_dist/cloud-workers")
-
-fs.copySync("packages/cloud-core", "n1_cloud_dist/cloud-core")
-fs.copySync("packages/isomorphic-core", "n1_cloud_dist/isomorphic-core")
-
-glob.sync("n1_cloud_dist/**/*.es6", {absolute: true}).forEach((es6Path) => {
-  if (/(node_modules|\.js$)/.test(es6Path)) return
-  const outPath = es6Path.replace(path.extname(es6Path), '.js');
-  console.log(`---> Compiling ${es6Path.slice(es6Path.indexOf("/n1_cloud_dist") + 15)}`)
-
-  const res = babel.transformFileSync(es6Path, {
-    presets: ["electron", "react"],
-    plugins: ["transform-async-generator-functions"],
-    sourceMaps: true,
-    sourceRoot: '/',
-    sourceMapTarget: path.relative("n1_cloud_dist/", outPath),
-    sourceFileName: path.relative("n1_cloud_dist/", es6Path),
-  });
-
-  fs.writeFileSync(outPath, `${res.code}\n//# sourceMappingURL=${path.basename(outPath)}.map\n`);
-  fs.writeFileSync(`${outPath}.map`, JSON.stringify(res.map));
-  fs.unlinkSync(es6Path);
-});
-
-// Lerna bootstrap creates symlinks. Unfortunately it creates absolute
-// path symlinks that reference the pre-copied, uncompiled files. This
-// does a direct copy for each of the leran bootstrap links to ensure we
-// don't encounter symlink path problems on prod
-//
-// Fix cloud-core symlinks
-fs.removeSync("n1_cloud_dist/cloud-core/node_modules/isomorphic-core")
-fs.copySync("n1_cloud_dist/isomorphic-core", "n1_cloud_dist/cloud-core/node_modules/isomorphic-core")
-
-// Fix cloud-api symlinks
-fs.removeSync("n1_cloud_dist/cloud-api/node_modules/isomorphic-core")
-fs.removeSync("n1_cloud_dist/cloud-api/node_modules/cloud-core")
-fs.copySync("n1_cloud_dist/isomorphic-core", "n1_cloud_dist/cloud-api/node_modules/isomorphic-core")
-fs.copySync("n1_cloud_dist/cloud-core", "n1_cloud_dist/cloud-api/node_modules/cloud-core")
-
-// Fix cloud-workers symlinks
-fs.removeSync("n1_cloud_dist/cloud-workers/node_modules/isomorphic-core")
-fs.removeSync("n1_cloud_dist/cloud-workers/node_modules/cloud-core")
-fs.copySync("n1_cloud_dist/isomorphic-core", "n1_cloud_dist/cloud-workers/node_modules/isomorphic-core")
-fs.copySync("n1_cloud_dist/cloud-core", "n1_cloud_dist/cloud-workers/node_modules/cloud-core")
-

scripts/migrate-db.es6

@@ -1,55 +0,0 @@
-import Umzug from 'umzug'
-import {DatabaseConnector} from '../packages/cloud-core'
-
-async function activate() {
-  // Perform migrations before starting sync
-  const db = await DatabaseConnector.forShared();
-
-  const umzug = new Umzug({
-    storage: 'sequelize',
-    storageOptions: {
-      sequelize: db.sequelize,
-      modelName: 'migration',
-      tableName: 'migrations',
-    },
-    migrations: {
-      path: `migrations`,
-      params: [db.sequelize.getQueryInterface(), db.sequelize],
-      pattern: /^\d+[\w-]+\.es6$/,
-    },
-    logging: console.log,
-  });
-
-  return umzug;
-}
-
-async function upgrade() {
-  const umzug = await activate();
-  const pending = await umzug.pending();
-  if (pending.length > 0) {
-    console.log(`Running ${pending.length} migration(s).`)
-  } else {
-    console.log(`No new migrations to run.`)
-  }
-
-  return umzug.up() // run all pending migrations
-}
-
-async function downgrade() {
-  const umzug = await activate();
-  console.log(`Running 1 down migration.`)
-
-  return umzug.down()
-}
-
-async function main() {
-  if (process.argv.length !== 3) {
-    console.log("usage: migrate-db up|down")
-  } else if (process.argv[2] === 'up') {
-    await upgrade();
-  } else if (process.argv[2] === 'down') {
-    await downgrade();
-  }
-}
-
-main();

scripts/postinstall.js

@@ -14,6 +14,10 @@ async function spawn(cmd, args, opts={}) {
 async function installPrivateResources() {
   console.log("\n---> Linking private plugins")
   const privateDir = path.resolve(path.join('packages', 'client-private-plugins'))
+  if (!fs.existsSync(privateDir)) {
+    console.log("\n---> No client app to link. Moving on")
+    return;
+  }
   const unlinkIfExistsSync = (p) => {
     try {
       if (fs.lstatSync(p)) {
@@ -72,6 +76,10 @@ async function npm(cmd, options) {
 }
 
 async function electronRebuild() {
+  if (!fs.existsSync(path.join("packages", "client-app"))) {
+    console.log("\n---> No client app to rebuild. Moving on")
+    return;
+  }
   await npm('rebuild', {cwd: path.join('packages', 'client-app', 'apm'),
                         env: 'apm'})
   await npm('rebuild', {cwd: path.join('packages', 'client-app'),

scripts/run-redis.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# Redis is one of those servers which by default
-# accept connections from everywhere. Luckily,
-# homebrew and presumably debian come with sane
-# defaults. However, they're located in different
-# directories.
-if [[ $(uname) = 'Darwin' ]]; then
-    echo "Running redis from Homebrew..."
-    redis-server /usr/local/etc/redis.conf
-fi
-
-if [[ $(uname) = 'Linux' ]]; then
-    echo "Running redis"
-    redis-server /etc/redis/redis.conf
-fi

test_accounts.txt

@@ -1,19 +0,0 @@
-Yahoo:
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdiddle1861@yahoo.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdiddle1861@yahoo.com",imap_host":"imap.mail.yahoo.com","imap_port":993,"smtp_host":"smtp.mail.yahoo.com","smtp_port":0,"smtp_username":"benbitdiddle1861@yahoo.com", "smtp_password":"EverybodyLovesIMAPv4","imap_password":"EverybodyLovesIMAPv4","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"cypresstest@yahoo.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"cypresstest@yahoo.com","imap_host":"imap.mail.yahoo.com","imap_port":993,"smtp_host":"smtp.mail.yahoo.com","smtp_port":0,"smtp_username":"cypresstest@yahoo.com", "smtp_password":"IHate2Gmail","imap_password":"IHate2Gmail","ssl_required":true}}' "http://localhost:5100.com/auth?client_id=123"
-
-Aol:
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdit@aol.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdit@aol.com","imap_host":"imap.aol.com","imap_port":993,"smtp_host":"smtp.aol.com","smtp_port":0,"smtp_username":"benbitdit@aol.com", "smtp_password":"IHate2Gmail","imap_password":"IHate2Gmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-
-iCloud:
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inbox.systemtest@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inbox.systemtest@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"inbox.systemtest@icloud.com", "smtp_password":"iHate2Gmai","imap_password":"iHate2Gmai","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inbox.watchdog@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inbox.watchdog@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"inbox.watchdog@icloud.com", "smtp_password":"iHate2Gmai","imap_password":"iHate2Gmai","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdiddle@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdiddle@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"benbitdiddle@icloud.com", "smtp_password":"ih4teIMAP","imap_password":"ih4teIMAP","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-
-Others:
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"nylastest@runbox.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"nylastest","imap_host":"mail.runbox.com","imap_port":993,"smtp_host":"mail.runbox.com","smtp_port":0,"smtp_username":"nylastest", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"securemail@defendcyber.space", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"securemail@defendcyber.space","imap_host":"imap.secureserver.net","imap_port":143,"smtp_host":"smtpout.secureserver.net","smtp_port":25,"smtp_username":"securemail@defendcyber.space", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":false}}' "http://localhost:5100/auth?client_id=123"
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest4@gmail.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest4@gmail.com","imap_host":"imap.gmail.com","imap_port":993,"smtp_host":"smtp.gmail.com","smtp_port":465,"smtp_username":"inboxapptest4@gmail.com", "smtp_password":"ihategmail","imap_password":"ihategmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
-
-Fastmail:
-curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"imap.fastmail.com","imap_port":993,"smtp_host":"smtp.fastmail.com","smtp_port":465,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"6e7bucyuxffmg2vj","imap_password":"6e7bucyuxffmg2vj","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"