From b686b969cdf88089c08c2338c3c2339677abe546 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Fri, 17 Feb 2017 10:47:01 -0800 Subject: [PATCH] [cloud-*] Get cloud-api and cloud-workers running on new repo [dev] remove .tern-project [cloud-*] move into cloud-core/build [cloud-core] convert to launch properly via Docker --- .dockerignore | 21 ++++----- .gitignore | 3 ++ .tern-project | 5 --- Dockerfile | 12 ++--- _n1cloud_docker_launcher.sh | 4 -- migrations/01-expirationDate-metadata.es6 | 11 ----- package.json | 39 +++++++++------- pm2-debug-cloud-api.yml | 26 ----------- pm2-dev.yml | 27 ----------- pm2-prod-api.yml | 7 --- pm2-prod-workers.yml | 5 --- scripts/build-n1-cloud.js | 52 --------------------- scripts/migrate-db.es6 | 55 ----------------------- scripts/postinstall.js | 8 ++++ scripts/run-redis.sh | 16 ------- test_accounts.txt | 19 -------- 16 files changed, 48 insertions(+), 262 deletions(-) delete mode 100644 .tern-project delete mode 100755 _n1cloud_docker_launcher.sh delete mode 100644 migrations/01-expirationDate-metadata.es6 delete mode 100644 pm2-debug-cloud-api.yml delete mode 100644 pm2-dev.yml delete mode 100644 pm2-prod-api.yml delete mode 100644 pm2-prod-workers.yml delete mode 100644 scripts/build-n1-cloud.js delete mode 100644 scripts/migrate-db.es6 delete mode 100755 scripts/run-redis.sh delete mode 100644 test_accounts.txt diff --git a/.dockerignore b/.dockerignore index 20e1554ac..1d936cb4a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,14 @@ -n1_cloud_dist - -.git -.arcconfig -.arclint +.arc* +.git* arclib +**/node_modules +packages/client-* *.swp *~ .DS_Store -node_modules -**/node_modules -dump.rdb -*npm-debug.log -storage/ -lerna-debug.log -newrelic_agent.log +**/npm-debug.log +**/lerna-debug.log # Vim temp files *.swp @@ -25,3 +19,6 @@ newrelic_agent.log !.elasticbeanstalk/*.cfg.yml !.elasticbeanstalk/*.global.yml /packages/client-sync/spec-saved-state.json + +# Built cloud files +n1_cloud_dist diff --git a/.gitignore b/.gitignore index 9701a39ef..2d09d0382 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ node_modules **/.DS_Store **/Thumbs.db **/#emacs + +# Built cloud files +n1_cloud_dist diff --git a/.tern-project b/.tern-project deleted file mode 100644 index 2f4ecf04e..000000000 --- a/.tern-project +++ /dev/null @@ -1,5 +0,0 @@ -{ - "plugins": { - "node": {} - } -} diff --git a/Dockerfile b/Dockerfile index b8090c6e9..8fed79375 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,20 +10,16 @@ FROM node:6 COPY . /home WORKDIR /home +# This installs global dependencies, then in the postinstall script, runs lerna +# bootstrap to install and link cloud-api, cloud-core, and cloud-workers RUN npm install --production -# This will do an `npm install` for each of our modules and then link them all -# together. See more about Lerna here: https://github.com/lerna/lerna We have -# to run this separately from npm postinstall due to permission issues. -RUN node_modules/.bin/lerna bootstrap - # This uses babel to compile any es6 to stock js for plain node -RUN npm run build-n1-cloud +RUN node packages/cloud-core/build/build-n1-cloud # External services run on port 80. Expose it. EXPOSE 5100 # We use a start-aws command that automatically spawns the correct process # based on environment variables (which changes instance to instance) -#CMD ./node_modules/pm2/bin/pm2 start --no-daemon ./pm2-prod-${AWS_SERVICE_NAME}.yml -CMD ./_n1cloud_docker_launcher.sh ${AWS_SERVICE_NAME} +CMD packages/cloud-core/_n1cloud_docker_launcher.sh ${AWS_SERVICE_NAME} diff --git a/_n1cloud_docker_launcher.sh b/_n1cloud_docker_launcher.sh deleted file mode 100755 index 84ef79208..000000000 --- a/_n1cloud_docker_launcher.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -./node_modules/pm2/bin/pm2 start ./pm2-prod-$1.yml -./node_modules/pm2/bin/pm2 logs --raw diff --git a/migrations/01-expirationDate-metadata.es6 b/migrations/01-expirationDate-metadata.es6 deleted file mode 100644 index 36690e4c3..000000000 --- a/migrations/01-expirationDate-metadata.es6 +++ /dev/null @@ -1,11 +0,0 @@ -module.exports = { - up: async (queryInterface, Sequelize) => { - const {sequelize} = queryInterface; - console.log("querying db"); - await sequelize.query("ALTER TABLE metadata ADD COLUMN `expiration` DATETIME"); - }, - down: async (queryInterface, Sequelize) => { - const {sequelize} = queryInterface; - await sequelize.query("ALTER TABLE metadata DROP COLUMN `expiration`"); - }, -} diff --git a/package.json b/package.json index f5a27324e..5b529f564 100644 --- a/package.json +++ b/package.json @@ -2,25 +2,31 @@ "name": "nylas-mail-all", "version": "0.0.1", "description": "All components required to run Nylas Mail", - "devDependencies": { + "dependencies": { "babel-cli": "6.x.x", "babel-core": "6.x.x", "babel-eslint": "7.1.0", "babel-preset-electron": "1.4.15", "babel-preset-react": "6.x.x", + "pm2": "2.4.0", + "lerna": "2.0.0-beta.37", + "fs-plus": "2.x.x", + "fs-extra": "2.x.x", + "glob": "7.x.x", + "request": "2.x.x", + "underscore": "1.8.x" + }, + "devDependencies": { + "chalk": "1.x.x", + "coffeelint-cjsx": "2.x.x", + "electron-installer-dmg": "0.2.x", + "electron-packager": "8.4.x", "eslint": "3.10.1", "eslint-config-airbnb": "13.0.0", "eslint-plugin-import": "2.2.0", "eslint-plugin-jsx-a11y": "2.2.3", "eslint-plugin-react": "6.7.1", "eslint_d": "4.2.0", - "fs-plus": "2.x.x", - "lerna": "2.0.0-beta.37", - "chalk": "1.x.x", - "coffeelint-cjsx": "2.x.x", - "electron-installer-dmg": "0.2.x", - "electron-packager": "8.4.x", - "glob": "7.x.x", "grunt": "0.4.x", "grunt-cli": "0.1.x", "grunt-coffeelint": "git+https://github.com/atom/grunt-coffeelint.git#cfb99aa99811d52687969532bd5a98011ed95bfe", @@ -30,22 +36,25 @@ "grunt-contrib-less": "0.8.x", "grunt-electron-installer": "2.x.x", "grunt-lesslint": "0.13.x", - "request": "2.x.x", "s3": "4.x.x", - "temp": "0.8.x", - "underscore": "1.8.x" + "temp": "0.8.x" }, "scripts": { "start": "npm run start-client", "test": "npm run test-client", + "start-client": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --dev", - "start-api": "", - "build-client": "", - "build-api": "", "test-client": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --test", "test-client-window": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --test=window", "test-client-junit": "", - "test-api": "", + "build-client": "", + + "start-cloud": "pm2 stop all; pm2 delete all; pm2 start packages/cloud-core/pm2-dev.yml --no-daemon", + "test-cloud": "", + "stop": "npm run stop-cloud", + "stop-cloud": "pm2 stop all; pm2 delete all;", + "build-cloud": "", + "postinstall": "babel-node scripts/postinstall.js" }, "repository": { diff --git a/pm2-debug-cloud-api.yml b/pm2-debug-cloud-api.yml deleted file mode 100644 index a1e201340..000000000 --- a/pm2-debug-cloud-api.yml +++ /dev/null @@ -1,26 +0,0 @@ -apps: - - script : packages/cloud-api/app.es6 - interpreter : node_modules/.bin/babel-node - interpreter_args: "--inspect" - watch : ["packages"] - name : api - env : - PORT: 5100 - DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" - DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" - GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF" - GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" - NODE_ENV: 'development' - HONEY_DATASET: 'n1-cloud-staging' - HONEY_WRITE_KEY: 'XXXXXXXXXXXXX' - - script : packages/cloud-workers/app.es6 - interpreter : node_modules/.bin/babel-node - watch : ["packages"] - name : workers - env : - DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" - DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - NODE_ENV: 'development' - - script : redis-server - name : redis diff --git a/pm2-dev.yml b/pm2-dev.yml deleted file mode 100644 index 43c88302e..000000000 --- a/pm2-dev.yml +++ /dev/null @@ -1,27 +0,0 @@ -apps: - - script : packages/cloud-api/app.es6 - interpreter : node_modules/.bin/babel-node - watch : ["packages"] - name : api - env : - PORT: 5100 - DB_NAME : "nylasmailclouddb" - DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" - DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" - GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF" - GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" - NODE_ENV: 'development' - HONEY_DATASET: 'n1-cloud-staging' - HONEY_WRITE_KEY: 'XXXXXXXXXXXXX' - - script : packages/cloud-workers/app.es6 - interpreter : node_modules/.bin/babel-node - watch : ["packages"] - name : workers - env : - DB_NAME : "nylasmailclouddb" - DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" - DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - NODE_ENV: 'development' - - script : scripts/run-redis.sh - name : redis diff --git a/pm2-prod-api.yml b/pm2-prod-api.yml deleted file mode 100644 index 4f1005f4c..000000000 --- a/pm2-prod-api.yml +++ /dev/null @@ -1,7 +0,0 @@ -apps: - - script : n1_cloud_dist/cloud-api/app.js - name : api - instances: 0 - exec_mode: cluster - env : - PORT: 5100 diff --git a/pm2-prod-workers.yml b/pm2-prod-workers.yml deleted file mode 100644 index 3098a52ff..000000000 --- a/pm2-prod-workers.yml +++ /dev/null @@ -1,5 +0,0 @@ -apps: - - script : n1_cloud_dist/cloud-workers/app.js - name : workers - instances: 1 - exec_mode: fork diff --git a/scripts/build-n1-cloud.js b/scripts/build-n1-cloud.js deleted file mode 100644 index 08d8499e9..000000000 --- a/scripts/build-n1-cloud.js +++ /dev/null @@ -1,52 +0,0 @@ -const fs = require('fs-extra'); -const glob = require('glob'); -const path = require('path'); -const babel = require('babel-core'); - -fs.removeSync("n1_cloud_dist") -fs.copySync("packages/cloud-api", "n1_cloud_dist/cloud-api") -fs.copySync("packages/cloud-workers", "n1_cloud_dist/cloud-workers") - -fs.copySync("packages/cloud-core", "n1_cloud_dist/cloud-core") -fs.copySync("packages/isomorphic-core", "n1_cloud_dist/isomorphic-core") - -glob.sync("n1_cloud_dist/**/*.es6", {absolute: true}).forEach((es6Path) => { - if (/(node_modules|\.js$)/.test(es6Path)) return - const outPath = es6Path.replace(path.extname(es6Path), '.js'); - console.log(`---> Compiling ${es6Path.slice(es6Path.indexOf("/n1_cloud_dist") + 15)}`) - - const res = babel.transformFileSync(es6Path, { - presets: ["electron", "react"], - plugins: ["transform-async-generator-functions"], - sourceMaps: true, - sourceRoot: '/', - sourceMapTarget: path.relative("n1_cloud_dist/", outPath), - sourceFileName: path.relative("n1_cloud_dist/", es6Path), - }); - - fs.writeFileSync(outPath, `${res.code}\n//# sourceMappingURL=${path.basename(outPath)}.map\n`); - fs.writeFileSync(`${outPath}.map`, JSON.stringify(res.map)); - fs.unlinkSync(es6Path); -}); - -// Lerna bootstrap creates symlinks. Unfortunately it creates absolute -// path symlinks that reference the pre-copied, uncompiled files. This -// does a direct copy for each of the leran bootstrap links to ensure we -// don't encounter symlink path problems on prod -// -// Fix cloud-core symlinks -fs.removeSync("n1_cloud_dist/cloud-core/node_modules/isomorphic-core") -fs.copySync("n1_cloud_dist/isomorphic-core", "n1_cloud_dist/cloud-core/node_modules/isomorphic-core") - -// Fix cloud-api symlinks -fs.removeSync("n1_cloud_dist/cloud-api/node_modules/isomorphic-core") -fs.removeSync("n1_cloud_dist/cloud-api/node_modules/cloud-core") -fs.copySync("n1_cloud_dist/isomorphic-core", "n1_cloud_dist/cloud-api/node_modules/isomorphic-core") -fs.copySync("n1_cloud_dist/cloud-core", "n1_cloud_dist/cloud-api/node_modules/cloud-core") - -// Fix cloud-workers symlinks -fs.removeSync("n1_cloud_dist/cloud-workers/node_modules/isomorphic-core") -fs.removeSync("n1_cloud_dist/cloud-workers/node_modules/cloud-core") -fs.copySync("n1_cloud_dist/isomorphic-core", "n1_cloud_dist/cloud-workers/node_modules/isomorphic-core") -fs.copySync("n1_cloud_dist/cloud-core", "n1_cloud_dist/cloud-workers/node_modules/cloud-core") - diff --git a/scripts/migrate-db.es6 b/scripts/migrate-db.es6 deleted file mode 100644 index 58df2ffac..000000000 --- a/scripts/migrate-db.es6 +++ /dev/null @@ -1,55 +0,0 @@ -import Umzug from 'umzug' -import {DatabaseConnector} from '../packages/cloud-core' - -async function activate() { - // Perform migrations before starting sync - const db = await DatabaseConnector.forShared(); - - const umzug = new Umzug({ - storage: 'sequelize', - storageOptions: { - sequelize: db.sequelize, - modelName: 'migration', - tableName: 'migrations', - }, - migrations: { - path: `migrations`, - params: [db.sequelize.getQueryInterface(), db.sequelize], - pattern: /^\d+[\w-]+\.es6$/, - }, - logging: console.log, - }); - - return umzug; -} - -async function upgrade() { - const umzug = await activate(); - const pending = await umzug.pending(); - if (pending.length > 0) { - console.log(`Running ${pending.length} migration(s).`) - } else { - console.log(`No new migrations to run.`) - } - - return umzug.up() // run all pending migrations -} - -async function downgrade() { - const umzug = await activate(); - console.log(`Running 1 down migration.`) - - return umzug.down() -} - -async function main() { - if (process.argv.length !== 3) { - console.log("usage: migrate-db up|down") - } else if (process.argv[2] === 'up') { - await upgrade(); - } else if (process.argv[2] === 'down') { - await downgrade(); - } -} - -main(); diff --git a/scripts/postinstall.js b/scripts/postinstall.js index 048c76b8a..4f8b71225 100644 --- a/scripts/postinstall.js +++ b/scripts/postinstall.js @@ -14,6 +14,10 @@ async function spawn(cmd, args, opts={}) { async function installPrivateResources() { console.log("\n---> Linking private plugins") const privateDir = path.resolve(path.join('packages', 'client-private-plugins')) + if (!fs.existsSync(privateDir)) { + console.log("\n---> No client app to link. Moving on") + return; + } const unlinkIfExistsSync = (p) => { try { if (fs.lstatSync(p)) { @@ -72,6 +76,10 @@ async function npm(cmd, options) { } async function electronRebuild() { + if (!fs.existsSync(path.join("packages", "client-app"))) { + console.log("\n---> No client app to rebuild. Moving on") + return; + } await npm('rebuild', {cwd: path.join('packages', 'client-app', 'apm'), env: 'apm'}) await npm('rebuild', {cwd: path.join('packages', 'client-app'), diff --git a/scripts/run-redis.sh b/scripts/run-redis.sh deleted file mode 100755 index 59c666484..000000000 --- a/scripts/run-redis.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Redis is one of those servers which by default -# accept connections from everywhere. Luckily, -# homebrew and presumably debian come with sane -# defaults. However, they're located in different -# directories. -if [[ $(uname) = 'Darwin' ]]; then - echo "Running redis from Homebrew..." - redis-server /usr/local/etc/redis.conf -fi - -if [[ $(uname) = 'Linux' ]]; then - echo "Running redis" - redis-server /etc/redis/redis.conf -fi diff --git a/test_accounts.txt b/test_accounts.txt deleted file mode 100644 index 75a3c59e9..000000000 --- a/test_accounts.txt +++ /dev/null @@ -1,19 +0,0 @@ -Yahoo: -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdiddle1861@yahoo.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdiddle1861@yahoo.com",imap_host":"imap.mail.yahoo.com","imap_port":993,"smtp_host":"smtp.mail.yahoo.com","smtp_port":0,"smtp_username":"benbitdiddle1861@yahoo.com", "smtp_password":"EverybodyLovesIMAPv4","imap_password":"EverybodyLovesIMAPv4","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"cypresstest@yahoo.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"cypresstest@yahoo.com","imap_host":"imap.mail.yahoo.com","imap_port":993,"smtp_host":"smtp.mail.yahoo.com","smtp_port":0,"smtp_username":"cypresstest@yahoo.com", "smtp_password":"IHate2Gmail","imap_password":"IHate2Gmail","ssl_required":true}}' "http://localhost:5100.com/auth?client_id=123" - -Aol: -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdit@aol.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdit@aol.com","imap_host":"imap.aol.com","imap_port":993,"smtp_host":"smtp.aol.com","smtp_port":0,"smtp_username":"benbitdit@aol.com", "smtp_password":"IHate2Gmail","imap_password":"IHate2Gmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" - -iCloud: -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inbox.systemtest@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inbox.systemtest@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"inbox.systemtest@icloud.com", "smtp_password":"iHate2Gmai","imap_password":"iHate2Gmai","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inbox.watchdog@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inbox.watchdog@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"inbox.watchdog@icloud.com", "smtp_password":"iHate2Gmai","imap_password":"iHate2Gmai","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdiddle@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdiddle@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"benbitdiddle@icloud.com", "smtp_password":"ih4teIMAP","imap_password":"ih4teIMAP","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" - -Others: -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"nylastest@runbox.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"nylastest","imap_host":"mail.runbox.com","imap_port":993,"smtp_host":"mail.runbox.com","smtp_port":0,"smtp_username":"nylastest", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"securemail@defendcyber.space", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"securemail@defendcyber.space","imap_host":"imap.secureserver.net","imap_port":143,"smtp_host":"smtpout.secureserver.net","smtp_port":25,"smtp_username":"securemail@defendcyber.space", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":false}}' "http://localhost:5100/auth?client_id=123" -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest4@gmail.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest4@gmail.com","imap_host":"imap.gmail.com","imap_port":993,"smtp_host":"smtp.gmail.com","smtp_port":465,"smtp_username":"inboxapptest4@gmail.com", "smtp_password":"ihategmail","imap_password":"ihategmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" - -Fastmail: -curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"imap.fastmail.com","imap_port":993,"smtp_host":"smtp.fastmail.com","smtp_port":465,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"6e7bucyuxffmg2vj","imap_password":"6e7bucyuxffmg2vj","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"