From 25270c0b75f736cfa3939703d7d92ace1e527a06 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Sun, 19 Jun 2016 03:02:32 -0700 Subject: [PATCH 001/800] Initial commit --- .eslintrc | 46 +++ .gitignore | 2 + api/app.js | 71 +++++ api/decorators/connections.js | 10 + api/package.json | 19 ++ api/routes/accounts.js | 24 ++ api/routes/threads.js | 31 ++ api/serialization.js | 25 ++ core/config/development.json | 11 + core/database-connection-factory.js | 78 +++++ core/migrations/20160617002207-create-user.js | 33 ++ core/models/account/category.js | 24 ++ core/models/account/message.js | 21 ++ core/models/account/message_uid.js | 14 + core/models/account/thread.js | 15 + core/models/shared/account-token.js | 18 ++ core/models/shared/account.js | 21 ++ core/package.json | 16 + process.json | 24 ++ storage/a-1.sqlite | Bin 0 -> 293888 bytes storage/shared.sqlite | Bin 0 -> 4096 bytes sync/app.js | 19 ++ sync/package.json | 17 + sync/sync-worker-pool.js | 16 + sync/sync-worker.js | 297 ++++++++++++++++++ 25 files changed, 852 insertions(+) create mode 100644 .eslintrc create mode 100644 .gitignore create mode 100644 api/app.js create mode 100644 api/decorators/connections.js create mode 100644 api/package.json create mode 100644 api/routes/accounts.js create mode 100644 api/routes/threads.js create mode 100644 api/serialization.js create mode 100644 core/config/development.json create mode 100644 core/database-connection-factory.js create mode 100644 core/migrations/20160617002207-create-user.js create mode 100644 core/models/account/category.js create mode 100644 core/models/account/message.js create mode 100644 core/models/account/message_uid.js create mode 100644 core/models/account/thread.js create mode 100644 core/models/shared/account-token.js create mode 100644 core/models/shared/account.js create mode 100644 core/package.json create mode 100644 process.json create mode 100644 storage/a-1.sqlite create mode 100644 storage/shared.sqlite create mode 100644 sync/app.js create mode 100644 sync/package.json create mode 100644 sync/sync-worker-pool.js create mode 100644 sync/sync-worker.js diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 000000000..002cc8677 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,46 @@ +{ + "parser": "babel-eslint", + "extends": "airbnb", + "globals": { + "NylasEnv": false, + "$n": false, + "waitsForPromise": false, + "advanceClock": false, + "TEST_ACCOUNT_ID": false, + "TEST_ACCOUNT_NAME": false, + "TEST_ACCOUNT_EMAIL": false, + "__base": false + }, + "env": { + "browser": true, + "node": true, + "jasmine": true + }, + "rules": { + "arrow-body-style": "off", + "prefer-arrow-callback": ["error", {"allowNamedFunctions": true}], + "eqeqeq": ["error", "smart"], + "id-length": "off", + "object-curly-spacing": "off", + "max-len": "off", + "new-cap": ["error", {"capIsNew": false}], + "no-console": "off", + "no-constant-condition": "off", + "no-loop-func": "off", + "no-shadow": "error", + "no-underscore-dangle": "off", + "object-shorthand": "off", + "quotes": "off", + "global-require": "off", + "quote-props": ["error", "consistent-as-needed", { "keywords": true }], + "no-param-reassign": ["error", { "props": false }], + "semi": "off", + "import/no-unresolved": ["error", {"ignore": ["nylas-exports", "nylas-component-kit", "electron", "nylas-store", "react-dom/server", "nylas-observables", "windows-shortcuts", "moment-round", "chrono-node", "event-kit", "enzyme"]}], + "react/no-multi-comp": "off", + "react/prop-types": ["error", {"ignore": ["children"]}], + "react/sort-comp": "error" + }, + "settings": { + "import/resolver": {"node": {"extensions": [".es6", ".jsx", ".coffee", ".json", ".cjsx", ".js"]}} + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..9daa8247d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +node_modules diff --git a/api/app.js b/api/app.js new file mode 100644 index 000000000..a763f6a9d --- /dev/null +++ b/api/app.js @@ -0,0 +1,71 @@ +const Hapi = require('hapi'); +const HapiSwagger = require('hapi-swagger'); +const HapiBasicAuth = require('hapi-auth-basic'); +const Inert = require('inert'); +const Vision = require('vision'); +const Package = require('./package'); +const fs = require('fs'); +const path = require('path'); +global.__base = path.join(__dirname, '..') + +const server = new Hapi.Server(); +server.connection({ port: 3000 }); + +const plugins = [Inert, Vision, HapiBasicAuth, { + register: HapiSwagger, + options: { + info: { + title: 'Nylas API Documentation', + version: Package.version, + }, + }, +}]; + +let sharedDb = null; +const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +DatabaseConnectionFactory.forShared().then((db) => { + sharedDb = db; +}); + +const validate = (request, username, password, callback) => { + const {AccountToken} = sharedDb; + + AccountToken.find({ + where: { + value: username, + }, + }).then((token) => { + if (!token) { + callback(null, false, {}); + return + } + token.getAccount().then((account) => { + callback(null, true, account); + }); + }); +}; + +const attach = (directory) => { + const routesDir = path.join(__dirname, directory) + fs.readdirSync(routesDir).forEach((filename) => { + if (filename.endsWith('.js')) { + const routeFactory = require(path.join(routesDir, filename)); + routeFactory(server); + } + }); +} + +server.register(plugins, (err) => { + if (err) { throw err; } + + attach('./routes/') + attach('./decorators/') + + server.auth.strategy('api-consumer', 'basic', { validateFunc: validate }); + server.auth.default('api-consumer'); + + server.start((startErr) => { + if (startErr) { throw startErr; } + console.log('Server running at:', server.info.uri); + }); +}); diff --git a/api/decorators/connections.js b/api/decorators/connections.js new file mode 100644 index 000000000..509d969ee --- /dev/null +++ b/api/decorators/connections.js @@ -0,0 +1,10 @@ +/* eslint func-names:0 */ + +const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`); + +module.exports = (server) => { + server.decorate('request', 'getAccountDatabase', function () { + const account = this.auth.credentials; + return DatabaseConnectionFactory.forAccount(account.id); + }); +} diff --git a/api/package.json b/api/package.json new file mode 100644 index 000000000..33e8d55fe --- /dev/null +++ b/api/package.json @@ -0,0 +1,19 @@ +{ + "name": "api", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "hapi": "^13.4.1", + "hapi-auth-basic": "^4.2.0", + "hapi-swagger": "^6.1.0", + "inert": "^4.0.0", + "joi": "^8.4.2", + "vision": "^4.1.0" + } +} diff --git a/api/routes/accounts.js b/api/routes/accounts.js new file mode 100644 index 000000000..6f0ca9d61 --- /dev/null +++ b/api/routes/accounts.js @@ -0,0 +1,24 @@ +const Serialization = require('../serialization'); + +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/account', + config: { + description: 'Returns the current account.', + notes: 'Notes go here', + tags: ['accounts'], + validate: { + params: { + }, + }, + response: { + schema: Serialization.jsonSchema('Account'), + }, + }, + handler: (request, reply) => { + const account = request.auth.credentials; + reply(Serialization.jsonStringify(account)); + }, + }); +}; diff --git a/api/routes/threads.js b/api/routes/threads.js new file mode 100644 index 000000000..d9ccb910f --- /dev/null +++ b/api/routes/threads.js @@ -0,0 +1,31 @@ +const Joi = require('joi'); +const Serialization = require('../serialization'); + +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/threads', + config: { + description: 'Returns threads', + notes: 'Notes go here', + tags: ['threads'], + validate: { + params: { + }, + }, + response: { + schema: Joi.array().items( + Serialization.jsonSchema('Account') + ), + }, + }, + handler: (request, reply) => { + request.getAccountDatabase().then((db) => { + const {Thread} = db; + Thread.findAll({limit: 50}).then((threads) => { + reply(Serialization.jsonStringify(threads)); + }) + }) + }, + }); +}; diff --git a/api/serialization.js b/api/serialization.js new file mode 100644 index 000000000..6e0da9b90 --- /dev/null +++ b/api/serialization.js @@ -0,0 +1,25 @@ +const Joi = require('joi'); + +function replacer(key, value) { + // force remove any disallowed keys here + return value; +} + +function jsonSchema(modelName) { + if (modelName === 'Account') { + return Joi.object().keys({ + id: Joi.number(), + email_address: Joi.string(), + }) + } + return null; +} + +function jsonStringify(models) { + return JSON.stringify(models, replacer, 2); +} + +module.exports = { + jsonSchema, + jsonStringify, +} diff --git a/core/config/development.json b/core/config/development.json new file mode 100644 index 000000000..aab9d4b95 --- /dev/null +++ b/core/config/development.json @@ -0,0 +1,11 @@ +{ + "storage": { + "database": "account-$ACCOUNTID", + "username": null, + "password": null, + "options": { + "dialect": "sqlite", + "storage": "./account-$ACCOUNTID.sqlite" + } + } +} diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js new file mode 100644 index 000000000..d9dd80a9b --- /dev/null +++ b/core/database-connection-factory.js @@ -0,0 +1,78 @@ +const Sequelize = require('sequelize'); +const fs = require('fs'); +const path = require('path'); + +const STORAGE_DIR = path.join(__base, 'storage'); +if (!fs.existsSync(STORAGE_DIR)) { + fs.mkdirSync(STORAGE_DIR); +} + +class DatabaseConnectionFactory { + constructor() { + this._pools = {}; + } + + _readModelsInDirectory(sequelize, dirname) { + const db = {}; + for (const filename of fs.readdirSync(dirname)) { + if (filename.endsWith('.js')) { + const model = sequelize.import(path.join(dirname, filename)); + db[model.name] = model; + } + } + Object.keys(db).forEach((modelName) => { + if ("associate" in db[modelName]) { + db[modelName].associate(db); + } + }); + + return db; + } + + _sequelizeForAccount(accountId) { + const sequelize = new Sequelize(accountId, '', '', { + storage: path.join(STORAGE_DIR, `a-${accountId}.sqlite`), + dialect: "sqlite", + }); + + const modelsPath = path.join(__dirname, 'models/account'); + const db = this._readModelsInDirectory(sequelize, modelsPath) + + db.sequelize = sequelize; + db.Sequelize = Sequelize; + + return sequelize.authenticate().then(() => + sequelize.sync() + ).thenReturn(db); + } + + forAccount(accountId) { + this._pools[accountId] = this._pools[accountId] || this._sequelizeForAccount(accountId); + return this._pools[accountId]; + } + + _sequelizeForShared() { + const sequelize = new Sequelize('shared', '', '', { + storage: path.join(STORAGE_DIR, 'shared.sqlite'), + dialect: "sqlite", + }); + + const modelsPath = path.join(__dirname, 'models/shared'); + const db = this._readModelsInDirectory(sequelize, modelsPath) + + db.sequelize = sequelize; + db.Sequelize = Sequelize; + + return sequelize.authenticate().then(() => + sequelize.sync() + ).thenReturn(db); + } + + forShared() { + this._pools.shared = this._pools.shared || this._sequelizeForShared(); + return this._pools.shared; + } + +} + +module.exports = new DatabaseConnectionFactory() diff --git a/core/migrations/20160617002207-create-user.js b/core/migrations/20160617002207-create-user.js new file mode 100644 index 000000000..1b8f61170 --- /dev/null +++ b/core/migrations/20160617002207-create-user.js @@ -0,0 +1,33 @@ +'use strict'; +module.exports = { + up: function(queryInterface, Sequelize) { + return queryInterface.createTable('Users', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER + }, + first_name: { + type: Sequelize.STRING + }, + last_name: { + type: Sequelize.STRING + }, + bio: { + type: Sequelize.TEXT + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }); + }, + down: function(queryInterface, Sequelize) { + return queryInterface.dropTable('Users'); + } +}; \ No newline at end of file diff --git a/core/models/account/category.js b/core/models/account/category.js new file mode 100644 index 000000000..b56b5c9d9 --- /dev/null +++ b/core/models/account/category.js @@ -0,0 +1,24 @@ +module.exports = (sequelize, Sequelize) => { + const Category = sequelize.define('Category', { + name: Sequelize.STRING, + role: Sequelize.STRING, + syncState: { + type: Sequelize.STRING, + defaultValue: '{}', + get: function get() { + return JSON.parse(this.getDataValue('syncState')) + }, + set: function set(val) { + this.setDataValue('syncState', JSON.stringify(val)); + }, + }, + }, { + classMethods: { + associate: ({MessageUID}) => { + Category.hasMany(MessageUID) + }, + }, + }); + + return Category; +}; diff --git a/core/models/account/message.js b/core/models/account/message.js new file mode 100644 index 000000000..ced3bd00d --- /dev/null +++ b/core/models/account/message.js @@ -0,0 +1,21 @@ +module.exports = (sequelize, Sequelize) => { + const Message = sequelize.define('Message', { + subject: Sequelize.STRING, + snippet: Sequelize.STRING, + body: Sequelize.STRING, + headers: Sequelize.STRING, + date: Sequelize.DATE, + unread: Sequelize.BOOLEAN, + starred: Sequelize.BOOLEAN, + }, { + classMethods: { + associate: ({MessageUID}) => { + // is this really a good idea? + // Message.hasMany(Contact, {as: 'from'}) + Message.hasMany(MessageUID, {as: 'uids'}) + }, + }, + }); + + return Message; +}; diff --git a/core/models/account/message_uid.js b/core/models/account/message_uid.js new file mode 100644 index 000000000..632bef539 --- /dev/null +++ b/core/models/account/message_uid.js @@ -0,0 +1,14 @@ +module.exports = (sequelize, Sequelize) => { + const MessageUID = sequelize.define('MessageUID', { + uid: Sequelize.STRING, + }, { + classMethods: { + associate: ({Category, Message}) => { + MessageUID.belongsTo(Category) + MessageUID.belongsTo(Message) + }, + }, + }); + + return MessageUID; +}; diff --git a/core/models/account/thread.js b/core/models/account/thread.js new file mode 100644 index 000000000..1c63b9ccc --- /dev/null +++ b/core/models/account/thread.js @@ -0,0 +1,15 @@ +module.exports = (sequelize, Sequelize) => { + const Thread = sequelize.define('Thread', { + first_name: Sequelize.STRING, + last_name: Sequelize.STRING, + bio: Sequelize.TEXT, + }, { + classMethods: { + associate: (models) => { + // associations can be defined here + }, + }, + }); + + return Thread; +}; diff --git a/core/models/shared/account-token.js b/core/models/shared/account-token.js new file mode 100644 index 000000000..a9ff17986 --- /dev/null +++ b/core/models/shared/account-token.js @@ -0,0 +1,18 @@ +module.exports = (sequelize, Sequelize) => { + const AccountToken = sequelize.define('AccountToken', { + value: Sequelize.STRING, + }, { + classMethods: { + associate: ({Account}) => { + AccountToken.belongsTo(Account, { + onDelete: "CASCADE", + foreignKey: { + allowNull: false, + }, + }); + }, + }, + }); + + return AccountToken; +}; diff --git a/core/models/shared/account.js b/core/models/shared/account.js new file mode 100644 index 000000000..9195662d9 --- /dev/null +++ b/core/models/shared/account.js @@ -0,0 +1,21 @@ +module.exports = (sequelize, Sequelize) => { + const Account = sequelize.define('Account', { + emailAddress: Sequelize.STRING, + }, { + classMethods: { + associate: ({AccountToken}) => { + Account.hasMany(AccountToken, {as: 'tokens'}) + }, + }, + instanceMethods: { + toJSON: function toJSON() { + return { + id: this.id, + email_address: this.emailAddress, + } + }, + }, + }); + + return Account; +}; diff --git a/core/package.json b/core/package.json new file mode 100644 index 000000000..184b16d50 --- /dev/null +++ b/core/package.json @@ -0,0 +1,16 @@ +{ + "name": "core", + "version": "1.0.0", + "description": "", + "main": "database-connection-factory.js", + "dependencies": { + "mysql": "^2.10.2", + "sequelize": "^3.23.3", + "sqlite3": "^3.1.4" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC" +} diff --git a/process.json b/process.json new file mode 100644 index 000000000..ee13cea43 --- /dev/null +++ b/process.json @@ -0,0 +1,24 @@ +{ + "apps": [ + { + "name": "api", + "script": "api/app.js", + "watch": true, + "instances": "max", + "exec_mode": "cluster", + "env": { + "NODE_ENV": "development" + }, + "env_production": { + "NODE_ENV": "production" + } + }, + { + "name": "sync", + "script": "sync/app.js", + "watch": true, + "instances": "max", + "exec_mode": "cluster" + } + ] +} diff --git a/storage/a-1.sqlite b/storage/a-1.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..2ef02b1ce81263c8c8122f11d48e35a8b448d97d GIT binary patch literal 293888 zcmeFaYm6gVb|#ipIWs+-qm(q#YG);_cA~R2Q&pYGNAS(8%ADpi$xM>Tb+iy@mwCSg+x=Vc5Xhu;c|SEbKr2W5C!yHn96gSlAX| z8D1OkAN}!nZaf%42AQwwndwnwcV&|CIQQOj&pG$pbI+rEl9Pra*OZl>Fy#De-&lM0 zZ0%EyTU%S}uB|=$KK>d%-@pg@XZ$=fKCJlLU32~EpMDb$JX`;BdhFZlfBx)t#q)oG zZhh+;-(sWv-N5<>-}|lKdiJ+o4~1G+%!{fjv_#eT`;9~?6|1DUN-UmBai1HPKj&V4 zE;T;qvV}_OAXVaurEES{y5Npd7hJ4bInEaFUOrW*>~No}quP~NAAZiA#!87ytn|_s z48Ef4eQ7Wd7q8dk#&qG~wkR}2MP0bw5QgICToPTZWb-LN9rYDF`Z-t07B0}cmpp4~ zC@6|Jf3=P$hhig!E*Jo~!f}NwRC76kG$7#4w_g>Xf9reO+s`f-($woNsQ9faUXR3n zU7Y*6ZbRYR&6j6&u%a3dXnN7U;4a!$mI>W_$cR%?)v#% zZ`d7)2Dxpoch~D(efMPJgS8*bLwy>^>W3e!xn>{>ysO-oiuTxuHCG8sPE=AFO@!YbIjLVt=UO-xGm;%MNrP^sYwz&N85Zm6K)Mt$x#nSx|Mq zA#~;bQW`K`bfy90Zt)u*tbIJgY(eD(}T(?711hxG%h{w-?&siT^idcUdz{6)B ztbNY}_-I&Q?hn7SiQnt%|MS}V|GoYv`1AhfJqF%m;OmTmUs_-P_?fOHRSV8?fQSc{$H;DXY2p*`hU3o`|JPi`oFpUZ>;~F_1|0nSJtQN!*zN6YF%9a zeEqZa)AbVu?CXrdue9&K|K+n^Km&g`rdG1qk~kbGeOFN!ws*NVHF*T>_0t~P0!aOq z*l$UF(Os8&`_Iyg{#bbK~4qmc$!zmrLZU z9Q(+j59~Tr3RkBukNYmDE7(@-a#K;=;kGoo#mKo78#)(j^rZfl`Hotb6#zs(%}Z@* zsP1yHT<&e9R4rFhNzTRZaPinhs{GdL#TR#Uhs#v*xwrXLxg0x4(c4~rXLEz&DsR(S zdNGqNzfB*P@^8zT<5Gnl0Ek?ya8SkbWlqFEHGtJJ;J3PRP3STJ<=ETPRH>XjE&zLe z@X$A2;n&&^F}MHE5Af^%`fdFBvybuXPq*;v|M&>M{so6$|C~0~fBe_*>mUDB{QA#- z6Tkl9Z{XK|^6U8ZAN>{l`upF*um9jL(VE48D8;T^&F_*H{4@9r@D|Jm=Zt$*kFAK?#)8~psP^o*wQfA#prMAKAua=~--eBh+2-yC_SJj)-+TKX z_xVr3{zvTcium7V)u-?8f8Jx@JqG?lV1W4l`LjPB$* z+3$VGG_*>a4&7A!&j)tROuzaNt*j|2>hP`ONmUnS0p~k4Nyf2X>YUCTA9*Il)};5^ z=TA=iB>w*n4*J&rtM&T&ci*4?*-ok6pa0>sw*3D5@6ktW>=gTF_56=@j`;uE&oXQ4 ze}6sw{QrIa?>ygo_Gi!j9xlB9c@PGUf7N!#OE#HhRI|kRrTab@cT2zW!P*P+kk=Da z>TOv6p9Zt=-4E8jZ-u$kQ^4qj-L_=-XBcS;;a3fCzrLhzX#_P_#HtwGMqRnqa1OCMRZrb{5I=Fb^aNZF{ zPXzu8OMsW&0dNrcFG&5GJP`(ip{Ne|PmN`FmyVt|I(jlssb&XPbf{ER${JY=FY*;lG}FP^|CA1g>G9`humkpFHDD8 zZk`={q4CA1i+HdpP4wffR2g02qz1>w;@(&q{pN9E^ Nxi?GEw6FJ?TRaSlXoq3k z%T&1n1U$O$9DW?T9~;4us6UkI!cda?TwdzpG|)O2!a7GIipHaS+8Yl=A~AO=5llwd z=n^=~qX*b&-qQYWZrIy#l|*&arGvM5gzDfL9jUiB5LA_W$$NbPS4|pniYjm&ar%nW zTnXKcJ(JWx6UJ3(Z$^QZ%eB2sG=FfBEFC|nh2G*(s->Y7cWZlsr*@f(2g-VT6BnZ^ zmo#AGTAWMpi98z4(P9fMIu@pLBx+#{pGKN5M=G6qR&wPV-tE(lTICuhime;n|uzM{k3mj#)tcO{SnR~MC`)FjFfIfXnU zRX9`9Ggd+Z|8JTH$PTMr@GBX4sfd6dQcORc^D*8v=c-|X?PIbjdn zqj-04)4Dxt)Z+lrj^_&@ z{IG)DmOoYU#+E&V*ZmRxs3&l;mkcK2P2H4?SmxiA&F967rso0^|t3W6IXM5 z;j(*P9ULA3#IzCS`?*}x-<KCJi-k0JnL6(1e+DvU0ZYzT<=~ zoh?#wnRhs5j9y+kX*6GuZ+A=?KxngONY7#--I!i?TH{u_S?zZHozeK#KWJ4dSMfq# zy!6T)`Lj*{OIKh@GAsCR*p? zT1Gijqo<>GMv@1<({U#nj<-wMgVFes2Z-(s?@M^2`DS3K^b+}gaUcfU=~huG=OgjX zd8SznE52^z+!q}bg2PTRfGL};&XADAz;?OwQWqyXngh;A6%|@mqa;%CAs*mk{#eQz z=7Y=ifjJSFhs(U{ROca%C%2Y_&XpBoo>K-_Ldf?8WomhZ*pusFA(jS$GV}4}~J3uscl-Qz%7OQtYCS3R_Tj zIU%0Pc+$gSLq0uKPWCVnn$}5fL3v~S*8G9U6^hFm>lORZ6zbyUMoLj+MRir=)mmzr z>9G~KUZ-KjB-NIyh(eEhqmBjxSsB_?59V`4VvV3WJ+Qe^)*>r*xihhgu8~K#33+pa z-YZPILSGmSSE-Mi8`>~MR~Ft~Z)o>IuJ1lyARGw0Ep+wi0#Txmn;VPUoroJATaFF@ zw9~TQ1^fS!%E1|%lX`>!0g8jv*TR8Fp(M?BDvxjt9_y02xgnBE*bb-4+@LE8DybGi zgTqrDQ5;zUx1r0Xz1oX}_}nB&b2lSZ9C5R40X z^x{V31`7PIYTYR(bgAtr*OG@Afeb}hu|s9u9EKr3L5gdPo`>TVBvU0`H$17o8J z91R#=(C~|H7ZVLCXbV6gjjB1qrGY#KrZEwM#ID>T+D7c#GNkQreHo9aL!k?DCZ3Sdao`HWBehyWT$7tSZemm;Ma4MB zA}AC5LAZjar6ySz_7I{~WVoeWCSb522&Dn40aCObfF{0m6<+1g8k*J6sCV2AL5+@& zrEYg~gN;G&h4vIJOLT6-+MylTSXLT4Tn!>fYSJSw6x>rgVuM;G^WQIT^x##AAvWb5 zY>pK1h02ZFzz81Z`XU|#yO6i$E=@vNR3N(Xk#>zF3*4GoXj51=H;RxY`a{vEpAC>k zzrzJWLH3PM0BR|ha~|w^^hZ;D3V~~IFR8oWXs%p|l`6Q8HMXh`On-FgZX9jvqt>5% zc|q)KpaPIeG9WkQE*wui1MSUD0mK`l(?fn<(H!>%lM;654~K!S3#2<>oEQR9(VzoE!b zCK$mA{VAtOH5g}9^l}-AYJ9D2`&17e?n1^gzNWqwg(-ED!2}C*F;k9=N6(&#TpL1- zO?^YGOAwkS*Foly2*5ND1F=a2g}Avh7$9`Q+T36g8FCk^35eg6kE^7iinBsZj1r8nDEQJ+>N>|f9yX~5ar;p2?vi`r@kqSZt|LuLdlsp<$q2P-1fF_lx>@~GQj=tW}>M`hd|Mp`j;qFbUZD;WkENogPJMwTZIXlFdYM(nh0f#nI%EK>^_Z@6M#G`N^OK zlKbKW+e8%#+Z@HOkeM1mqPL(~EIDkvEvznVVI=8=odhi)_Q6y4sPjzNGJ;jH&_Fl9 zV+Q7kcbJgk*b%jxD{B~KOEO6eXcKZ5ePy5-_CX1{szQ4x0n05-VH1&^{f1RQlOnis z^~9^-bdN64J&d8CT|hq9z&2nzrdWVzZooHMuNj45?o^pP^dXnPm^ANfa=`2u1#Wbq z)SIYxf#je&wuOGjx-b~?41Qw@4V?cu0m~P|;}A%osV5AIP2iA}IKqByBz0*^&oJMF z&I=>Z4hJ0wVXfNZJ(>=-qGP?{V5=e;13f}1V>r|`K5BL*8oCl_Py!@s5bGVA6*NmM zLhoxhpmu5M2nnP#iNF&YGXx0U0}n7vRB+E2Ak)YR1uoYeI~VB9%lA)7Ngc?lL^HMv zWI*a+N{r1_4v`fUap8xmJ-d4EE!Pk%;UTrjwpBr2c6V7cJt8T52&+i@+!5bvE(UM# z=xZ+Sck}+&+zI``8gjoq*XG8X!G112lvGiC<6#%U|G(1Or#Wj8heMVrQ}7aIPQa@&W>;1^#I209CW9`i z3Z#-S;e#pER6`lu44t2-WY#?Ch&yxw~UVxK=qs!T-Pd)NP2ilX}=LM4x8sUIaTd+crdhWZ7oKobQ1m?!STMTHL2C ziT0!KQdVqGj3?&K2E#;vb zm>Es#yBOU#fW?t%le(79#OecfHn2j_4oMfpKkPPWbvE}mLJ!9vw8#0O!fr72>0vt^ z=;e>z=z#y%W}BU-Z*|D__mrIu_lS*-l{}Vic;<B;)$ z@G)B)!x($_h~_piTl;rzir6SMMSr$9JM4H&U7zM~VZ&pcq}XiZW1W_jdmg=M%-Vf+ zKCpK^b>pMypHJQQu#K0~w#UTBV1HrvW8AxQ2c+#4o$deOmG*z*E*SuyJsYgy|Mx#X z0Svs4|EIhFKcDgcZXekL*x*!{nCduF%@+4wm`-TO)s=y@W$y*DDdBU1PL*&#tAC}l z$qdf3L}l*<&Jth1Yz4P0ScmC`cF6=^ZwokTApa}61ZONVc{RxYVwIwKZXCSokAmD; z{-0;hKU~BA?|=RxW8fd1Kd1B&|C@`Dkj+-e{(%1l&A-4!4hA^WWVs1JV;ouD$XJW4 zk<5a3=Th}_W1TM?4LsIZH|WB?{}Zs*Sxt4c;aX;+qa$gXwdP=q!$ec=kncHo{pubX_mL?iq8{TQ?6*4z`<~<=9NG2yjJz{5Hy(DF z-<$a7{CHM72+u#g#EHF-Mw5OE(n!5*s|NqJ|7h(I<;rtuQ%bcsc<{oTP%Tri^zmIpYZ_zv0 zXXkE@*69Y% z#mUXBXH>=Pm2WHCQ(rXJ3f`7Zyr65<;YG3AYy_fb(~B!5APPsYwm^p4%hgZYHs@n1v zlWLGn!dor>!ZlHG(HVXq8G9JX};P_Rdd2wnwR6_LNuGprYa5ZFx)(9 zTp>xI?tdJ?B!2zee6yyHta(^ycr(EFL<_co!@;|7{RM%zEu`|PGqEl8{|yA4^N4Bw zzq7lV{qLWxJ^N=Q3EuzwJYnGXzq?Lb`v2zpG^n4yT%*?g2h45{$~DR)7+E17n^Dx7 z9h6hxID4N2p+92=LQutD*7*PLu08v^L?rKjekL&Rx7yzz#{D~a>Pje1?N0Cb}`39Gf+YcI#_NsX2w1EM`?au z%-tLwshUqXxu}vWC39=1br4Yjj(o;LFfcL@;RZ#-fFTJ^|ccRxX-aDg3XQos8^%t1M=u*PF4%OfOE3Zr0xrBJHFpB&PrP{ z3~lhLUg1}*_sH|3(Ox}TuLYpJF#O9#@2q|S4*{$VZXPcAJi?-H>J}{fu`z?28;Eo0 zkqf6{3sZ8A{|SUv#sAFig24W&x$)B?{(l0tKij?+$Cr<{{Y7?p-!7_$=sBksY=3js zfjg1|%s=g5@gNgHEFB8-A=t+P@E(1PjlkAxL&<-1-%aD+eE08f{EJ19vpW*xL&>B& z3d5$CMi-7Rv;NuozsmY&@6m&7b0$?%%->zszp2zrMJG3<2et47>tFgfzO9bTWCUN| z zSi;&A#%Z16w+v#2g#?|uZs^;%X^^0JiKfFThO;9Pslc&;&6pCL+ zy~~6cj7{n_|G4JQs)0c*cD;QNma0|6_kGy%H>cC%KxddyM1OY3XG53MzWg&{`O~By zX_%DJ;oVvOmQNWlqRpBiJr7z^(cf$cg)M`6=TpS7> z&+U2Xd{hr-2&vbB$#Ff<$Q^s)*(;^lPBep!QGdYuhU#`IdRXq%gv7yRTn#m1SKAkb zv9A`mnVcTx!-eW`;HZsAln)O}XV;29GCHoFmebwwKpanxjt`2{#zi95K9XwV=t1i)0f3kdp2*{` zRRRDC@n7NEZRQ6^(K)6W`GlPdi^yafr2)*bdCjq6F8<%o`}nBaLOKtK{||&#+W*YE z+W!BatUdcDBnjUCeC;ss`#axcM!qWb{O8Qbw~~A<&)#1iudv$AJ0xRmlsDvVU$lm(XPlyi^0WuWVhSps^uj2+bB0i)xUHrrSlxX zN(jSSNkkmI(DfKrIs}C4mP^eLh)Id*_DSQEjRo89auj7dOKhNpjO-%);uNz&P2)wb zm`nrHH!|fa0Av=kIvW_A8EvKQ8A8MJ0k1rApCn)=6fIJDv zV!blA=N-LKS)Bx!X1kZujTV*sn{cXs?g$hu~wp*!t& z@y0+M4wUeNT6lu}u6f*7J^dR=Z}01CzjLP`Oq+3*Sn zlkG_1Kurx3wZrp%|E%Yq6o-5|-mfwnp27R^*r=@x)w6c6T&rR7M6pn)=eHy6?U*Rs zZeK-DJxAU^uTq-ir-7s9jp+BB$4*Clvv;-~i`3InI$x}hyQ5^TB1dGo4iMF7wHgqP zD!Gwd9Zr*cGM&7<6{?=g=5_q~d~j4e9NnY`m4j-zl)qKH^-AnwBJvl7XyULX1Sgc> z^WdyjZf{2?foM5Uznuu@iIZVD@-t<_dy4&z^yYq@va(ted@i3dLModzLxTD{!SZ1x z)Cp`01-TMaGp*C(vsCfIvz_x6x@Y0TLGChI%@z{vai)+iHiB0GF$(&wW5ryorHi9N33Xks8>P{0Q1*|qs&9OH2uR3 zqv3RHc%2TGily*y8m`CB%A@?1Sh`XJVzSxG6mR2)#p|B0sEo+$I#VwoRSYJ+kMo`v z30qi>&u}u3j3YZ^6lT0T7}<~=E6jNJbkBQmhR+Dk`+~j@@8_dIe~6Fr{&2ud^La{<3Nm7d z*ecj;TsTPAJBe&Kte0+oIst=MB>5>e1!@-w(~wTaVTFgsTZttbCez#O(YYgWx_Q6U zoA4YmJsEq)f$iqlQ?4r&q0kCvqQl{BeVi-C<5G5VIPo^bRHZuY31b~AV;w_&Jnm2a=D)>&A-!ae zqO}nLdyFzX$YG}l^kz~*er;x3!oY@SPQcB^*a?nKzvQ*v%ER{Kilz6rw%m`lcr@1Q|lj&=W=tM@aKWWJ}30$*kt)xC+`sH+6#? z=AWglQ!!RdUSFle)Rd+bucQ^wc6*0A5E0DW-{BCy53fmFNREj3sGZUXi5ux2$apN0 z{THS=t@~P%U)bVc!IQ{*2ujs54Kr#y6aVCI{ylV!B?M;a;RM|z%XkP_f!fp~(zeZs z)US6(EN>$S?MBdxyCAjY04bhncH!BHL;;#Pu^~1E=w{P83QTYXPUxGNfKvb!w`bH`b)V#L1j%O z=^iOW;CZnN3@Ux11SE7uCU4^XnM^@4VMD0g2ZemlL!3Z15tq@li!^UYEEh560Mg1u zxu}OO*#PW$#V|=uxRfv(6Kvk%dMs%yrA|doAG%Y!kz|;%>uqK_XCmJm?dFEjCe|pV z0VhnL%80})=bG`JDlgHvVN|CK%Is)>Sd;SXGE%3kq?#!k8FFLO0EBM>th z9*Dl_@J18&MA~e}Lb8f>FRY!<&e$46*Vxp~<_KVhBX>>o)NW5|kyhjAv~{83dBwuu zM0D&LJG8K*^F?aS0qRRI4{M|jyTN({>7xl9q*EvHL49KBTkSJ|bhlaxT7?~Q&{GD& z4a_ldgf&wl{-^8!*r&0KW`r!tL`_m!?)64}qp*|5eIjZ0;zGSch)m*L=A(|5DeD3?0d)m=;!QS!^hL5^EEP3b&zQ?nh+!oK4Qgjk zpRopGjT^iQjsZ&UKt|95OnCX3bP(9PE`p=;e->%uNfYB*)DUKzDn4M2aZU+T;M|E7 zBhU}I*#2G1^2)R~oSYim1D+uR2RRMrP&?@`hmr16u)C#Iq{r#t&P`AZqBq4$0*>WzTn9sU}YJ(kiTGh z8jh@kgNcWC`-9w8hJ-v7@`HXb61R4^7zM>3|1sDA+BSk=UUOeUEaKgsprAmM_BQ8b zmyx@B?z-PQn6O)2$#M(r>2C?90@Folp?cX9Ca$s6piJI@2!{KL`+|~YS(>#$V=!>r9HDvF z%TYT?1dVNTLgkhNxqdTHIw^LcVX?unfEzm~PS- ziAG&Q7aGkjlyQK1&B(z;YWptd^Al9k<6UFDA-`9f<&WEtyHq^}-9ncH^nvQiSbD@z zzt|X3p~{E@W699_Sl_j&Zpt$F{}oevdM&`5Aw)w5$Kx-fBOT2=8?iB1?HkZTWZN}H z#70+>hcF78%}^6=K%Z}e4V9>F)JNq78Wjy&!vpWrr@hVNhDP%)4K9$xdSydWn2=<7 z-^Xxa#Wk(zqQW}W_L(rjM`0Q!@rC7eKC&8 zG9$FbzH2WpLN@5KOqO~H!2sfxbynOyQ6no#V6RD3#pqPctZk@s7ifB3Um~A04sQ;9#~ZMKxdQPgMJIJ86WR$1p-@;Uc0r?5-DW*Vd=`4@dFEj zh)NGrRISnXYTC$=>|rio1wv~k8Lr2m?soAGmC^loc6|pRv;j3Z5j3yb33XT z1keStr7%;9X)f8?H)k*L5PR{`#@XIgZVLLwRoiiH?JqSUECodSJgKjpwr=gS-J^N> zYJJJ0VH=By2bci=Z<|wrb&X1g?QMM}*l+vL9&k^5uWzuyGUk-ObIbGA{yl8pCNmO> zn8hEg_9RO+N907dSzY*$n0SPt8Zq(j!cMJQj8>Sb&Do#H@_tG`l;BSSb*(oCqEJoS^2GR{n9H)C5_4*b8vYDKOvuszD+ytRK{ zW4UfCqNVOGTlwt#&5mT`P=B&%+q6bL-m-1lSs!cIwvaYtq*;mYDfVr4EJ3H4i{@=h zYgUqA;D6Vn?c!#S&8c6l zgv_S-UD?Xn+};mMEA6|w!laqoIYO45wg>y~%69IA><608;dl2=Ea%v(k&*u+a~fGC zGQ-i%a8R-!Q0KdQh3StDE;~igjNM+ZD7dV{5W{dS z5qYerKYHV-LF1oosP_-3!ZDo$j_R-(nsh^lsYnGxTCe7KX!9oA+Fx>mNW=th_D<_c zzfX0Y*{0IiFf8=OY~N6Qhj<=?qI}2ZF;25*wLG9A>n>`SwwPIAzoI?Q`k}&Y0cC?x zVsdW_hPJJJ?r(4~k>Dy}&Dd8NEm37?*eyGB{^y24mo-UR0#0eord`dRXTI^S*?5j# zZ)E$AY@KQ~*RaE7EgL9G=jYpNn(d|a7aq~xpgU6CsAkyeimbV9qrI(}O@}bsG8;Q! z4`XL!nPG#F%$a5eBxbnW+Sj7CU?gFOopgN2JW|MpFr%Iy%}CBV5r$^S`o`d4%PZkv zxf&@ps|P1)`skuGI)}G<;#g^&kT2=hew7TS+H;P*Gy!$Kt`e}B zfT3I~4`#Advfdy`fHz)pInn2)gJqs&<5ftidphtg^MxHR@29kk>aehTGk zJ5_DrQ-Z$6T8UU5f8wb@Rykl-{DnmM^dy^zUDX0r4?P)6oX4|g=lK}onqq1h7jmgq zrtCitN9b-QQ%av@N|l=L(%bN*rJg!990!xxGXBNF>!knyCw|9W8Q9RbK~UnBrpw}R(!srWHvTUOH#Y7hRdVt=&3MmdOX3y zQm&M|$%&`wY3fP}=Bu^VN%!LDBz;=07lq?a@b)l~I4vFruaQkfO&teE!kHX6;rUK8 zcI1&$*TtJ-B_8s$W4ZJ5YCYR759-ZkK2cU0x0lh3PdIM*3TG$5>1jJwO?LdPQgA$} zG$}MBB$o2+R5Hc)`J2!|yq1|J<@m*ooNM1SB~ab46+EwB$F>Xk?bI-Of{Za;_&B8s<;2Np^`@?5{Dli| zyW+i>UP?ESR`0q{ZdFRnt5bhq)HuINRu2{5;dZZHDGU&E7s*BeI&!Fk*>pjAv)M*r?M@X)NcfoK0$AxmH^7UY0_+;lS67 zmbZiHgM;99BNy=GQ>P~oZYlM$bP)$%#U|IeT&mMM7?%p&z>(+ppn6yxq|Y;9DHr1d z2X*oGLOAqFX|-{#3<9xSY&wn|wZeYobXWradoQy2%qWe~_OGu)x219ZY7jrqD<`c| ztTULzOU*`Dyq34Cxoj+VAs&m8oT!dRqtJCv5fu3>*N;UL-Lu1HSn4OvVpqXl&llg^ zh;)wm_<17`>JD0++im5j5%c(O`C%{9&s8pmKL1Vac6`|@`umf>_IA+|E1!)H(~03} z{yfDnmg=S}o$~0Umq>gy3}NUN?-@c{wGt*nB>BtU?c%@9jfJb;?v;FY7*P7xh0)+h zngm1YZLJyish+do@TL_On@Yq};kU73c=-NNQN5tKWor;pA{eXC95dXcX3Q|12w;Xu zD(*dLg{q(#u$9mj#nvQVJ-)ihwok_G*u?^TS-P z+BhlI1HoiR_GAXB!bwc2drmG#Qd}yOQiZVJcXM(Qj2(tf6K5?fXr9xHz*g5PbGFB@6bgF)ymricN9t0AHy_sr0)99af zN;m3-Z*b(##v13>{#K&LhkWJQZAA!Vlcn48nR?v4Jvlw?h!y{Bd%Juk#T8GiSPT{W zD-yFRL%^0D(Xb>UQN!hBB~Xh+qgC~|9`#*xFNG6zkUK1R&jaFZ^7zOLX#94%e9>+O6OC!) zFx+jFjxPH?war%wQDG1X9&DeTv@QZ=zN=?AS}T1C%$s(*py^*;k)p4;|%cfDRZymxs+E+onIMt6<; zAI4oR{?EPk%uSNu{m*I){Mq-uO$hs)|Me9;_b#gBIk-excE~Gt-cCDgsSi)+`5pEU z?q(hqU^a7tT<#NDcxTR!n;U!j59x0<4->j@_AXpYX0b~gd`lSEcjQ#ExncD*c!F1n zHR1YG-Adp-%ABxqq;aQTiH3|74kaMr-^M?GB46cL`2>Vfg81|jrXAEL?8EVtynPU9 zjhGhw#?q1rS_uR%qBT*v1o2ae#M?wBl{n&fH;>Y%m3;1PK2Sty_7-WHCLhKdDydhtrRn0{M~mCI)Mr!A0^-d?P_aKBhipEuwR2i<(Y=MMVK zx~#oP(C==$;e&yQoqofu`QsA`$TVjy*;*}Y^;y&NMO*K#*!t6m^t5DnqNsS4{MB8< z_7!%W9n`NHj;`8`(U5O^S|?LDl1^f{)^rJa-lvc)X|XprpSC@0sm_+mSBgkgHkrP9 z%~eJs@;(K)Lpbl#a)e)~R<;`qYFSpCT5<~g=T>KLN6&i`=WGtv#5a_{-S6d-!C)+y z0=eIvB<7Cp8;KrDn$Z{bPy<~L8-$t0J>>k}{j<6ANnbjeNTq|pw4UH-=@7vG0b>^I z4N%N6!V2Ne-J3Y}=s~%f`b2g_R!CbwN_?!h2dpTqsTB zOA8;7ud+qhuw-*1Wa?Ak?9Ds{$rbU%y?gh94zOPIXh`@7cXR|8OyLSIl9QpN_P7_) zAh2s`((CaBUce_3!K`BA6#}BDgZcr5f)TEoTLb={fkhp~VxM^jimdMI-qyW4zoox_ z59I1k))AwI?Wr`_vkWQdpP}C5wpF4V_*p61IYe7o=7{8)mQl_TIga_X-k~5#9%b{W zXVmIb2r2xpZ}#AzE$k_(;6fA~KG*jsXJ?L8v_aT?^Zi=-%K z>J(Py4XJ6s|7$7NDLf@8Ho$f=lSAbZm*tpg*UaFOe9cLFbL>m=Rcr5M%ifDxI-@AQ zY~FvAE5$>tPEZVII=A&b_K3;dV!PA|M61bWuB)D>{lQT!H@wLlsMiNLgc)6@ZfZ5j zr`#s_+fh27J5C=Yk84-?YP%)LS2xXqcp9IaFk&8MZ-s-?1JzUAo=*IwQhRdR>xXYj z)n;0~jMQ6#_u6-Gk(FA(v+niw(RuRlX395|)7JRr<}90(yJ;yGNn{VMw@fjN#6 zC*kehRd-wqoJ`IeiPjBOYcx>F#rl0{2X*pSDjpO$b;QExh%Ff(a0O%)WccPZ#^7kR z-Qo~qWRrLMsPsZ?%PjVLLHeKD z=Dn0&aaAfJb{6NHX^3W)^dW35_O{;o1b>8&HtMdZU*KJV03UYMyuq5w?-%`1zrWrL zgc@(R=5`t^MGhn}sgg*mitZvsL%%_n6BJ^N>fA%sMd)A!g6w+8xhpM-4GODqQ99B@ zjCY~dy{5hF=Cbsy(Yd;6jCK<_c55^mb$2!8t(+<(H#ZhY8M84Q4t6~rcHHW=h-=T# z({POp@Get1^6XQ3kvAj=;VNJ`r8OS)nUDG!t{9&RqQ+y~I*=dX!1fAW0ecpn!nDq~ z3J0n69wjote+bHm`og?hAEw&`?zSmuc;7kyPcRT%vHzdlrS0tVwSTbo{J+4jpVgn= z`KQ0JK^^_Qf8e1mX0tfKz84v;*a<^k8}Wf=K%go9$n~~~fVNb>E^Fa#*CV)_G+YSD z?z1=GE4j-U+tkhX1Upm~QJiPIX+ z#o@?=I0?47L)<*X4T9?xcypafD4AN0%B{C$OB^R1|ba8&XI_2Xq|hBH?^F~ z@C&4SY6tx%zqF79f5UnIH^}Dc8OV15-5Ap!D2(y zOyS{}Et+yRod!vh#})UP)tjYnX!;&iT zm51y?8O}^;aA947wP6P$AMe81n;1)@S0V{U+*;C!nphV|k0$aIVOK(okmeqPw&6*` zNReWDP`E=S@}|$ELNgw;tlY>PWB;+pY9e8M)nLo?eO3+E5O#QG&3(MQq8MfH^^?Wb zn~##miONMW#j;Cr#cDj4O>kQ-kLS#v@OYAyBzI0m8klRm95XJ0rDclR_jpo;EpAI6 zJXJ%1``Fm;mX(&LQu0iwhdcq$Ke|}#g8nv^E_kE8`N-w^Olor3)Q|UB)HOHhb^9tn z?fH`?9Xg8*B+y2@r1$kKxb^jx z-ujZJ0n2Qi0kt)%2OAQdyy=0zfzIK;h5A|24K-*kBG9PNcg=^2ZZI<}n@lq@-H#kY zO|<5UkRv05sN?CQo+<-V__puV$hoT;W117<4Uu)o)()KzrqH0+Xfb@bT-NUE&9IIm z*a!R{5vg~Sq><7=*g&yxQrtx}7}BU~!JcMM8H%|fhG~wjuBvofkJFa>i0{#&x7a%Q z(h@Klq5xh!Ui%a7Gvs<`0Sdyo6-w%*{FtMlU#+i}=uaQQkGYSKDhVA$UJ=?*;Kvq# z?MKw_{3Bn|bIz?G40y2>3awq;iVxAg^N;AQQ0(b!JQT|8iu~B26$c1&L*bVX#i5a? zQzOyEZq7I2fYRk7;k{0+@OLAOqN(qXFw{eHz!*<)%d6*wcNp-B(fAyjLEQPfZN~4| z3}WB!XvXI-;1#0@I5y)zL>3YN|3@j6!2_ELIyOZS-K*z^avRJ)vSLIb$7UQT)-obC z8izrzXeR8~3?&7)V>AvuS<#H67;s{Y`O*9=Hsf=Y1SHk(LfV5QfzOdeeRs2{|3SRt zbL1V$AaTcN{0==?P69qhf_~bn=CBm_94XCd&EmlS$RT3QVB&uewfP;%fF-9`%{%@F zkpYtHJ1huJ$e-h#fCE7;pC7*?891pF^UXX+uK69wzzOkBY9`<~H%=ItZ^l9NEk|0w z5ov+Dc_;Xwxd}KDb-;m)=0+2A=*jZY1Rdul=)e?n&4eB_nvf&XLJnP>Z^oI@FrZmN z6t86{;udkwqU^pBN1$*vZDSN_SVlR8^mANwin$k`xUjP zZs|fn&v&FlotN0O?*dqR0wnuG=!IL~d2ntmvh3|!ma))P8zGSq(wgkHDGwC-bmxOj z8HpscLF*m+7+Ih2v)E>BXoIYMW$#l%cXedRee->08gUsLO}UX1%8TSbv)dT+sMn{-lgGl3Ih^_G-`UG8pp&6dzpH)M z;58%_Cjn?rqoq1dIRKY4GF`C~;JHO*A@`I#uykdhh+4XzS#PYDScj}wm_25_Y8z~D zk;h{EwkLF#jCZp~X4h=3_$Zs%BDoOu=LckdXKiTwH-m3}XLoUg zj?A!K%bZZ+`ZwA<&E2&1C*)n)AI9mLWx$PWIY`h&so-WkSbCrJh~Cgh!0~0r+jBiz zP{54&KqAoj(P|@|yJ3TeOl!+9KZGz^z@HA$;T9Vo-0>77sx+BF`)+FymAaO-2nG{p z@6tEhn<$yK>`i@$45>Pc%@|Q!|l+Zmb-(y+DTi!N`+82IXz%%_~%!U*SRfpgE} z&Dui;9P*I`HNc<(>zj6b&fl?(*&kh4iVcVc-e&JxXxBjCY}$;+twVQfUo>7ZUa__B zi!P^Y6Px$#f%(E>AHe^6I?o#YSey^r_?A!2+^C%9X*ne=8OyRMn`>)8wh+|FbQ z$GNRbImekJn_J8|F0Cc5bXv|i&Ury+89C8}b@j&A#Z}r=N*vZ{1ITPy28j6u*aYCP z1RDU%B^U3T3oJf6uAnABhc(myXqi6?t6?eu)Ot^>Pl!#-B^I|G*+zTQa?AFjMhJSp z!2kQZ94cDPuOvp_bn#^f2iwcLaHL#V8q60BsbjpV^-5C*bpbyI&0#4tpy{v8_}s*+ z!zyUJWQg7wChUDN?(68WFO>6rTNGVpGsZ|QvoY6@`GjLr^Ydk_e{)@MSmlf^SV!o9 zBZt+@c+#%=;aG&f3sW}k>Pp}VU180v z?g<$MkW;&_RcE+YkAhF=Q5S`EPzGXkpWreyMpcBNEHtU^6mkO-((Xc3=n37@TU^mC z%$v?~)}-TpIyKV|v26=A$pJ%#7+Rl2!!&JaV0!XgqN570GpShh3vCTF&|x`D_d6fE z=+GVSYJ^y${NJBy`Q9}on`)b`kQxGqDV228z$QZ*+g&;@(?iG5#5klFqT(sKf}LmS z>s^|H?OQaca8w{13GQ9GX6^6@y059ZWB+RK{N3Ok9+`nh+pBudI~{%7d++eb3_RLi z1^({NFFr8??hgi50q27bX8yUpFFrB@?+dQ%JMVQk4xR(Q_{a=AABe2#y^~en4t?^q>)vHz&Tpsi?@ z$>kl+&gSs5)DLF9{{biX3pT@r{@)vZ&IYfd3PQC#_=}PILSePn9_w1)S!<2?6ssau6d<{SP?d=qbJ_^Zj?C(Wf}2 z%)t*j;r}V#D0A>bPWX9}8;YhnYYG;rn{;SxRktkbv4g0#3DQMX3sY=?AGN{P6)p>W z2VKDqe?d^#;qzWw1BT9JX~9Y3vNy3X#?2jydUG4{OG6Z$?+ZtRr#IO>9jpDo1l83D}#^Y1i6>g9i4nCa8-1jAl{8o2dZS zCLEM7dlL)8oEt^R-h|HK)+QF{a!&d!F5+`dgzZh}ikP(t2l;6q#X=wEnuyq&(B+`D zi3QT1AH^L_`0VUwsAHBUoRl&9C>Hu)ABE4(8-_+^ZNgzsVryd2(Ri*2JJA|SnYD?# zh;~tDm}|mLQNBfra?;A!D8TM+-gead}{Q|_dk*^za}l>6;dZfIy0WI5?) z_9pI_a=(4b4JFOm#9dRqsGryo<+o3{p{H4!xNFJ-cL))`eaiidQ|_ds*${PC69M~N z7@C@86pMJ6<99{<+p5_d0HXmc8WYJ74 zdfD7rv0Fq&I>a$9_elO!3+Xct;Z&T$(b$Zv{lY+E$%8xyF#M5bXoXX74hGF#9RKg2 zF6R#ZN<(1}M{-h~F5nA3y?QA6KEj#dQA`fSM#%C5ngQ@xOSWdVD8`bbbQ$PNk(XdctY@WSGNs2nsT*1F)v}kXn{F4e_;aAApjN(%&UORymNMR zwO{wK)xPQOPvizf{nA~XF*5yWXTml*wG=pDEx^Aw@VP+(7Fp3(4U)=)8hv7)tso5Z zK3Vqx%j!e^rJJ5MJ-Ur=%z^t0omi0e#bV6Ur-sl5Sfinq-rMZ| zg&VL8!#U#@%NK@blCHb+`ju5~TGlwgqNw99jV&~je{Etu6RJ-mat3wYl4pfFr!*1- zKxd)PT${Mqgy^?GoR1Z>LbPdkfJOb(czH+vyTJXr^x$_0=`(3!e%BBLSP#IFh9Ejn zYx^{~rf|Y{f`z6YjU@)xufYW@L)^9XOOL`8AqueA8K8kMvRN3PEjQ*c$ojE>T)=|Q z0!dfP(qmRA)|W*9{V|L9f5U=tw7NZ5KX9FuG_oW)0F=eGl-Aw3SDB}$9=@}M)E<9b2~+&D2Bd;!gMV5f6EO1H!=)hh_-f2?@voh+L14LBCwj&mWuH3J3s;N z9$J}ohB47czJle=!}L|4c<&mdWngD^*3H-Q?HQpfxU%fJ;(q@GpJQ9x1_zpAg-X)-oNs zZ!arKKa;N<8JAu|TA{n5%h+yGtp01uglW@ZIlAR(2F%&gzRhNfhW-*4wFwJHcy`b` z%ksfYb6$?DHQv{lBWI+mE3KSJ&#-Y1 zv}4kSv>x9Z?K8#Yo|p%4>r%m4Fc1FEG9RQx?E7hc%pfoJO~*c=p4f#CkemR!gd<3H zM%-e`7#sFk1RkTYT}~_QKAR7~a-5Q{Y(6~13m(Q}$geU>CNm>*qo$j~Ok1eH&I+0; z3g1dCLdfE^ZJDEaVTG{^kb0UE81oN*Z6SY6avBq$eVdX0oUkI%!e)S?6q`ua#WJbv zBBPinbX;R7SFh>UlB?{g*~5T^=VxMgUJz@8)q^H1<69vS0n!%TXAz1&a9kZ&RD!(^ z`Z9PgS?M4Y(FF&&I9tZ_-GuYh%`ISrpqIY|ge#j+i`XVlxcf?0P5H2(*2+Rm zx*8lr$pVAX*XWQ{-ooxe@^&U{Nj-FueBl4X1y+Ml2E2Dbn4vIv=L42~A+>Ok&SSLB zn~RgiCWe0?tpWNK!ynLc39A*r!?785A`7*=LHZz5KVcS>02api7Vt7O2LrmkJ3+c$ zhAMz1uOM16DgC+96S^7#$0C*AG_M&GQH#_X^{Dm;)BiczR%o*TSjMWy)->A^r2C;+ zmLBV5YpcISO(JY+${L2W|He}H5eo?etN}FcnS<6k`Y{$v{GehXvKIoDnZQsl7HR%z z^F~T4`%-6{JiA6@r9c|H_I)J=(E~#4)`GgDH?Gr%Fq^xDaBgHK6%``(*R!-^yPO`p zjM!zKvW!tRcF`sEO-&$F3nNV)8nzmQ$oacplP5M`VoC_wHL`rV0L#q9=~B%4n1*7d zi9n(%Qwc;Gyn5%gG5=6g0YbLPKZ_C+rGOgrN=3?R%%_oWj<@wllspYjEbf7h3RoVV z4|pi~pEm^iSQ8~6x|DVAtrn)lvr?fz?~veaY0C+Hgm?mpfd7wB(})Hs`Z!0-g6vv`J-Ymvx>= z5Xu>>xiQ-W%hvwV^qgEVHDZR#GeG%P47uRCy$&%@h_pnOYY@I%2lSXnfn+|6#wT4qjWeKI><6=3TJC zT<6TiiAGK^KZR80a%r!J6D{%&8jm%xoh-hiUA*ng=X8gJ4Uw zL)RAs`+S6`RlEb1Rl+Zb_ohhy!ETAQk|A~%bgIG!8P!RsM_&A=v}(dj?+&ek_)Y)2 zeXpF9QRGpr^L`BvhMPCrIC2Y4(tt%ajR!6&vlV6P*#e0D`=(xHrGJf0j5+m&!H8Lz ztU}7l30~~IS>_j#tOg4}rrqo+@(6wCH;`#{$gwn?yDYOMQtj@laO6f96ym<>imb}b zA?!K5-J4!_`IrIAs42pdYtPnF+puTUu5EJsUd$8p0;R5!<@=Udv!XxPLT;IEm+3OJ zihYc^N$`H+6-Dl_XPQ{nZgzEcl9>sGWwnF_fZhPCkG-jI(wp8whU6w_g06aZs5NX1 zjW-DfQ6wm6)-xa!!Pd|U+rmJEogA=S^w2IEVC2O>4JJF}DZK+0-JyLjz-n6D4A4wd zVjM;DLa=ltfpLhguBB7-pN;kwPG7XUI!6*i0@iElt7X1j$_R?AI1%c@?v&kR{09Ed zD}tnoyWrZbWpoF43t4~Pc(g$=oavZipFutv?bx>0R*`D`{Eo!yOhTfsIC)sy5!kjE z7R2l_JE3Jl2oDB7;3b`_R`+@Cu6g`v+p98j(4)-{mPN?X^k5B!UsyU845Yvk0?(dk za{z6>dM)1o*3)ecKT(6jY%zYI3HI@31>5vE+6%B%1}3p~ng&_OC9=lOcCr-h25Djl zd?W{+Dem(|8psdiVIqCss!1@W*G+L`M*#m9EDJn3U>a*@#sg+A)Z`KO@e&j2f)ODN zSX!nBX0Nd9amXeB%kRuAf0v~z!mL}&(ijLcnzh^IAE__S_8aIboq3>FGdaS92)kyb z7HVoyAJlMfn+*0K+b``~lAXV)DR|13?p(W6o=)DEhn>6OJa1+a-VH)li?Z$u=_Gzg$TpalSn8z3I4}Qj2 zYJTcLvU9Y*oERRx#xnkJTwM*rzDWxX@xgfC#qGn%nSqYn-2HlZ^k-nthA>s0CG{D8ORpbx$BF z%_BUf1C@4u943z1ceVLZsavDhn^gtaE{TxIda zn5N9=yI>L^#bx;})wV^|>puZjw=*jN<+_5o*HRFKHsA9-pBGXLpL2riCoIa$nY3JYvj$SAFX?3PCM*BeQdYEqYF5Y-+5 zJ}us8GlLVeG+FN$#&U8rvqZG(r=RR9V9n}4m{xb|(>)_r+}3Aw zqO1;amXF1~u{0DT{$3?wS+BXpUPrWEqeN`Ef})a-?mLMQ6q@QSbl+aH6-AV~ z$emyvj3%H^^EMif@@a286p6&#sYEatVWUgHL7ITG0W9r5YKoy`DM~#_b;={vx7LQW zBUA_1=tw=S2Qd7Yn2lr}@^>p5C%q0-@4nA&l~ z&5DTmd_1cn_Hv|BMXO4#e8ana*t8GZTwNMa9jjC#nMt`y<(Mm$D*J*V*Fgf+K{GLb zB*4AIcMtw?z^_WRT|azTkYO&MnyGLDQ-)HLW{iqvp)i|@3U+2?Q@u;z|56_$LuH;` zI`Bf7$|YOKf%NDQXoOcfo zn0#LjjN;wFP3!ioQI7*eJDx9u_}jkRybQ%|Z?gvtX;^5ev1#<(F_CUe zuRE=AE8VPiyZ+8-eCr>yDwV5vAunEf<&Jz?FKt)+Q>oI)S8BB|KnVHMQuR{KO{%d; zAnWr+E@J&{Z|3Y&JPgE6PYx5U^KmVsoT<^%Q9C2a1K;Vm6Aj1PrR>3Ie2MzIA9ine zU&0&BHv>bZm&o^v12Nc6w~9(RABlI)GtFvP@pU8TzUZJ39CnK2NHLS>SZaxZ=Y+W~ zPIk1)fmA}1menYURD6gB_?SPI@`m}~Yi&7o&@o73=)6}_U zueD;Y%lz4U%Ljr1e<&0Qh5g}3gzP|DM)}|+Du5|?&Ru2VXY^;6Yn}Rz<%%#&S3=e8 z%ISI5D;<}bmwuVw&iK=jo*c^Eh5{$4ez_aNp|Q~>Qz%7OQtV2Q9Be^F#ou@;<4F&T z4f*s`IoZQRoV)Un=Ts{=Z|IsSQ`bb%NL^~O4^5#iqTU_StSedp;Z<6yR+hPfB9>`E z9a1A4)eGg`=tZ1ust45bS;=Lm2P`$A&qN$a_E?=v1F}}#nQXs(`24Ic?W_$`WJNvS zkk;$PFP{&EJHu`ZU46Phl<4Dv{_o%QMch7A)O6JU=k8Kj-Dl4RYxw{D&rbjYzyIZT zsDSMsj@Q<{{q1blUch!a=`vb$s#t5?M(KJ*z%I5d>QaEIOLteS<>xBa`uq{!LdDu; zii){WgS^n{%qH?xj(y~OZtSLTeIWE)ojytcBdr~I@9lC^5p{vLOm^oqe&=Eh7`7py z^mkAUR)I|&m37TaNX!oZl~^wKwo>^cui{LjG?~d+pRMzxuK20|1eW6%ML+zKqht2Eb|=@LOHECUhBq za_sGCs#MM%7f|ol@AjF!Qc+;Gj-FF3WJCx)0zShNQl~F8$DLPnTqP7MlO}x#gPVrExX9#Cvzy-LxrnZ zU0_sn?ug=m!#0nV3WhpKACPhoTRM_aRkpvuA(uFjR8}dP0pf%P+)Px!HPXf=hgvZ9MX!ZpC zO;U8VO4MdYg?gQ4N_BG;dJQmG;o;E$sg#i$lGQ{mi$fZes;LI(35UYSI@V}a7;t1i z)hcph1SqkrdETmGmz-?qg_zz9?)JqlL)rlb2IMOoI0Y56f%*QAxC0D&_z{;z*b&^t z=uLQ@43$xxHs+X0^v{TG9;2SFbu*19wC=#857?stRNZ8HLKji`8SYV-z|nx=1r5JI z-ELJz^`kaiidj?43uHiElQb4%spD#5TOtzDi#Kb?AV945gEmf2cC@NKz=49IxNNJZ z+&SwrTA=win?rypy2uG*5Iz=Ntct(^D2;RhMzIZ1?EvX!n?&C|8OBJUUCTWU)}cr@ z4k5He+o-+13~4)X{$?nKTc*YnGCB@iVR*EymK3q!=8l^f)ksk>&N1q|cflWoE7ZRo z4X9cz!(KKY(<;W7^OYiCAXu`2Ovj)aAVsqP^w9)xt!?2|s-RBZpbU+A$K6ma(Q&HX zzPZ81p!Y(1iquln^+g{rX0lXJ*vAfsGFBX%0cld%ltl%q8$OcNh-85qd%5`y%jQNAW7Z#vM*VDnH2NJ5 z^~u>cLIH@pT+Vr5_0S(p^}*O^@FjH@9L<$0u~G&1p<`F|f$5K~kaI`^PO|P zbIx~`DOph7ye^}~4(fbS`E!mGgp=H17UOqNB-Rkv!gwA~ro5Psrqdy~dxDCdA=NRl z(?NyPNCKWyQz#P*VVje+z_}XuxrT9ULs3rGI?e>}F!F5A8C}y@E6p_xleGyFz$rk_ zlkn)-3qh`RQQQdfdGwU3fOO&YvFFt6xtv-u&FjEhnNYbS80y?R30AJ zCg3_fc(zjAB)HQ~*k{IbP)uGK5up=U=P+J?QpwClUsc*RCS-x*RpSRUVy3jyz`PDD zU^GBU$u*j&3aVIQs2mO>G-%#$H=6=XcD;zC7$i&RiH;&$I(a39EE| zWMBa3|H9(3<|hLei0{L{QJHAOm7EyAgJ*JtWSa_YIc&Z?>@GB+F>x%?4dpuLqh=U}wbQ0#UVP9O1%ksI?;jd*l$v~K5Jtz)*1B11RB^%g(kKy2FE#ofS4pUBBhI$ z!0acqrMDJ?5t;NDl2c;TZ4{Or60{m-wh#i#4!P z3Inu6ONRZ9tqQs&5@DpuU_kxS(h(3yXi@+t^lA_e-UAIVNYru9m>|>02?RI-vw-4d zxp?z>hy5gta6|Jzl6$mUmXiXGt|;-o3{~5@T7Rs`g$|_Ii5V?^2en2LPGKfgtMdEKrqY* z?|1+OyX;|eU|jGgs&>1Wz6T9p&t}505mX!YV2RZ|$>mHy11>{MN2~KE zs2W@#x3k2DH4PIwefj7CjRQxnGP9llz*p0SHR>m;7uKt~l0o$KLLCF(#diBb6@#tW zb`8S>*6ipJiUu4qSWttRIO*}q#oiHUAKRH-Dk9$r!nzHeLx(0X?1`F(IUTHP&62m+ zKCo2-v15VBfL@?pKm;V=Xj8j*5PW+_+oU@Jm6sAiuzR5nB50uhtCy-FT$6g57NRe+ zT^E65;7~*KN46;==6WL?1_CUm9p#8r68=-;TFMRu#dwnbXUYmJ`8)I!^KnCAfrVMX$ht!5eHVN-1p9@bcdZOV#Z*HaLpL0|!-03zB_!Rx#=mj}^vyNa=Q^c1y6 zc%s%r$nQ6o8l>%^&De?9E~PwGfw6E>*J5(d%;C_MH> zN?3}IeOY$uJbKqywZ~{auy?&w@!|64OZ6V6czLNkCOl?G8`_U???DZS>xwS_KX$^0 zWB@zP|E#<0{O`9udxgJ0{QQdyfxo!?m*`;fpZoa1WS#U6{A~)%zrY7$=p1JnEQ}w* zj6j+Ps*VfWKpoCt$yF`i8Yg9+f|Go1*nL2vaBb`OTR z%=F2E`i*(4%mJtS2fJTz`t9!W4~Cy#!SdYuP|G{XKUN2L1nRn~Sh> z>2|mWo7nOaQV<{G9V}UHz;E?aWXVb0Hg^}sQa&TMd6;Nn!*o9ApOli*PG12ao5o_HusP zj0fwP)ht%5#tYfj>f1%IfYuuxMcQYWMa1wG>bE!uQp);CViNv*;?R z(`Ywya#b01JE8c+`szjvDb3ScF_+yhMBB^$akvty+}?E8u_c=5oXK!EE#|Ms*=(@3 zz6;InGSV=8TkTy%8`t?@BrSI41NkI8ZFKK%X0cLmqEym}@Ist-dhH%HQMpu`*MV~T zI5a8TRc}MLH~pSW*`dA*F2bPySGrgSzwWJW!cTS)-o9jj2wydWg9e8Ea-~=dlr%M- z&6Rt%t^Im#c3c{)n-_zWbR9|dbB*J&6h4p1$+JeRP5!;j*?DQ5s44yQMRIU@wwmTj z{lYjG))G=EmX7Y1ZO>2WEcq>HK6s2oYfZ*zC~Qn)$n-=1}@;G8a>4c#unCKY=#mMwn5@lYn6MY>c` zPHyoNwn*8K3!Zz<<#D1FL#T*NiY5<#ft_dRBJ%cgDQj3HNL#+ZmD|(UvbsDEt>RaS z;(4*r%`^(li>x?JE=%!zA)l#tg7awiv~z=KF5MS6hDEfUof$49-*GqJ{Fr=s=xo|N zEHF+parVrEZE!^VDO`VnAbuOFeC>^FTm66KsJKi2H}7))-@khGqrW0f@Wan<5(Iwv z+iysv|0jRvH`CYHu75n$*O+lI@ISWw|O$32d}R}R9!jc+*p|LLnA{poKK z=6}!U@(2BYnF#lfikxuMN>epGK**xQjC0)6874aBvWz1pYfmzqpy4|YeWcu??#upFQDW2?wxT~g4#K7m?wB{J9!W2BMSU zr!>qb=WdsLp5W1M$`&m8!B{~QvQb({y$C;N%dHg;-dOq|^R544XlkT{OAP`sWz`F6*CjL{GBKnOON4 z>)(3N&BRxC)hG3^!@h?s1Uv4w4zuOtzP2!v5qy7>1AzWp@I(a9b|(mMF5PzPpAB8U zJz_9zeJsv{B`Yv5ZA%XDJo8^P9*w~Kmyn-n*LRS{7Sg6LPP^q#+Ds1cSj(TIyXTse z?4KqJnX35;~}&B8FYV>%Uq3XrRFM@pODS%A~>w} zvZq?TlIh*{2F=jLuo}`%Bcb)`K2o}#-Veu_X{Mg2XA*@*uURdL>wuK0XYR_2s~dTV zFp$qi!_jqU*{CgUS~7|~UWF&($qjt(!)hs+J|CP7BFU@4_&TM86RUPhUbf`C7MisZ zxBJJ@L8F2Ap3i!6cfBr$hVz`N$oaXLk6f=Olr;FeVENnI${gOx?FF-#Tja$2JwFIVQxK;Ztedbwyva|G1e(5l=H zbqeJ`GJm6X`>AfYvzW|8X|C;O;wQCXtC>2!PHK@(;%5JgR+la$9 z+2ad{Mi8IjVbl6;*kW+ur>Iy}{lelb~&Go# zwc)i!aj6s-Pu3@c%`Op|i>Mw7X@lVknlc&7^hj7iTf?FP7#QKp<4k4ms0Ai8cXoLmXYL*$UGG7Y?B&JT_EZP>oyxrJl#p<%8 zTP**fFDUtNvM+L4fKLMaplBvk&awhWyb9k-5G7eu5C+&efVTtLh;YV7!Fb$Ub|8G^ z{1Jm0n{Ts2>-=Svx(Rzc1Sm=`Gyj{6A<`SgL%?tc{1%hP)B`nZ_PuXEQ| zr}O5bn1Y~PpQ6mVkQkHQ?m%eN(!$>rDZhqkrSx6{SP5Zxt15`2MX@TwN{4_@-Ezqd zfmoD;Zl5$x*;ugcfk07q7O=R>I8tpRi%1g_r#}gylFkCscT83&i&#I2mEgm>-p$ zMUM4BXJNB!@_?IX2&6$Q=Ql4)?u2_v?RVa`JLj^Q%BejsEKBSe%!2-VCcb=o)Sw&d>^{Bb zrET^*o~`Ps?t7r>B(NW4ppOl z4X<>(+K-ivwah%#I=P%oF2?ezG8ePSNrTz&4BAg77JYTDUG&4XRtt-#D5X-nxF759 zCzR&>{!RQma2gDa>(y0p9XjpaDRSU4alR0{1_J?zS#&|cW;xom$TE#$>J_Ms~<`inH8?%jeIH9U*<~LN+*1SCKh4oHc<%= zZ=*{8Byt(6R`Yj@6;jg_vYp{cE-;w{E5pdJU%fglRgu^2wo_f)ho|x)uSv`E6G#f5 ziR)@$8INWY^V@8=TB%0o>u5W9QCk#ml7MuG44*Mkl)_R(l*M>hj)-wlj)u%M zpAQV%mejI@EH_yw6C@u(7Dg6_HdAsmWK`ifR44_D(8g&!(r~GCz^~%K9zruoxFm~{ zkgXyjUV*CMuyNraT^}T}VX#s9y!v>tUGiIZK< z$eFQAx^o&e$K}v|cNwU))q1nki{|3<`F(p?s3env{OV*C>?oOfV?8X?)~C#qQJ=zA zqh>5S%D1Qes+u69=C_oibesIY)Bi7K+~xMaqgOvVB2Mtb&(k3AXMx`)^TR*=W{2_3 zzA-c8{q?2s42i*wG@jeofvJ-{)wS-^r}31K#*-)Q6tdM!rkKbVC|&1qCjGIRsT8j2 zWh532#>2#xyi$EWn95W5_2=wF9)b8o=*nbBfl-7UzYNaLv-+5+uy}H^Dbm-F8HrCdWJ7(A0wX_zY(Z=z9RT1wN1SF&Vl>%S9@6$CR+-U&!f46jLCNG}NasGsTriQDKN(0GXw zp!grD)!6WPl27;v!h$y-^C3)xJkuz{#tY@IfA1e+Xe=QpOApo5O|p!J02PQ$ViWJ%I2wqYAvAxmaNHmM|Vr=0HlFh@2jDr*$`&!j#?aGt)U^`Q~ijyf*s8 z9)&c(}Y^Ulk#jbRHv+j+?0(BoP8Wxpbg3<$KYch z+B=|P_y7Li$k!JMeFr_H2c3f|raEO==?Im{%@mB7&~q@Kg9thFISCLMV0VS(ek!!s zih#{wDwffRfJK>TiAzt%?pAV_=tNd9Z2o z?Mx?FJK_;ckYIk*9ORh9lA@&mdX!8DGhu1%SW-+{Ed!S{Hm*ERF)n;Vdk-c7JhFhj zeLoTmP|CkbyZc_tItF#_~P6^{R3M&?MXU5h?gT>6v`kv%W zBxIi-GNsD;y2rj__Q;3F$}ZCXNLvl%{KI^J?c<0xXjAI2i#i;exfEb9ubpPE4*}jH znr3A<(FYZL6n@U+G#punXRCndm&3wdj+i_Z*n@nqQ1;#l2?~lq{!@?vq-_Mlycd1} zwupC)%9ixD5M-B;yVrGHrc$l+#%N%>WxdhFRimbRs3pcFB&WsGpYu|cxY+_h_EbWt z1L5;fC>)L|@3EzKScxzp$i~k8A;eQ5j{1;yd$2(6%aGpl=X%j9YRuyaM+e<#H{Qh* z%?{Y)LNs3b#Ks1ne6V^Zi_%8*)>dOV6r&o+!q3SBVC&Y=8fyc$%^8|^g97zqbR#f0 z@Op$8`bMEsoMsa1!aO51Fnd{50r`F%(=y;@K)OqFBpmfMN04X^AdI7__Y56;tQ4yt z$%DF$StDf%6_?L=5zso!P7BPO!i8rS49rpns~I z0QIAXHZ6c~a0patmEn<5dX>%C=uKqbAOjNGj*J2FH0HZ&xow>;zJjfMk1LVz98on)Y->WrONVW}BeGC{2?n z$@aP!+gJwH0}V(L_63U&QFmo_Ah5^jrfhE+z}sw@A(~@>&6W|Dwb8PHmwnaNGLg4{ z!ctR%`cM|fEczY}{*HwcxU5s30X$k&l#D^Llg}ecG!zpVNajq4H%S#pG-}Q?>OaQg`*WEuwH{% zgS8?#DQDlTm@Y2B@dl|}4)^%HslK&&iIer-5i%vx69z%)g`u}+H^v*R{IS}`OY=GE zF$4k|DUgxe9i$K#j6tGW05aRuKb!Q-TEKgu5U>8bXY4L$l@cLi!_kjCjQY{$cy$)a z=Fk^E>DGki$oxO0w6KmCQzQQ%u3u1z1Ty`wbY;x=fek@Hrgv0%80b2iUWcbeu1`%; zHr9~;me5V$Tm^Z-rq$%mtYofeT(^$sRZLg~1u|9f0t;>%9x!W)A_Al?G7W;spqnhQ!(v1fIvNmB@+9M*Lx(U7fx3Ya z;AUX3%VT?t9^8n+R%R11h|)d;(2Kh-!=7SDbh<$RLm*oUGo_g3lD#8y^-@o<7jNyH z>8{gmQ&(KM#<_R2)r7FMA=2{(6Movdcf_ACG-c_6W!|yHb}-?a)YUE*YuxgdEPsEgzeiTMgkGD_=D75WT_Sioyaz;Gaq6T z&oERYCjL{{sdbCd4l}j6`W8QazO`C-xv`qk+UvE{T()Z6PHh|TbIjG>D#no^oG{5^ z>LyM7YPgIykvYxSc9H}A-!fTdJJ~(Y;=FhCSYx?vE26FLAzQh1{^kTSawxypv~5}= zpKsYV?X1r=Y}-&9JkqSh^%DCwGf&WI=B9bu)|(wA7&>?vxA%@*ws7Ynn3(?bEzQE( zPh$e7a1XN9dsx72lLm+_L-;d|;AGMML^g2S$P82X#Ue0uwj43j{1DZqJTBmF?UM z*iST_qiEPCv7AF!BP0Kh&1Gbjkl6XJ!+!qA9rFPIZ*GmVBg3F>8$sP|S2ypqRFd2c z)gHDhNLhwzfApbsl)`}(3_RLTg|h{DtY|;|5NJW-#~b>KN5@p*luiN{ZB+j<@rDjl zkqU^UUha74@Fv_l+H!+X#BAOi9UsbMLUmo4QfVj*8>2D%H$>k#p2wv8*sjfEobK3e zc|b+R1L&YVW>z?=^XJ((G`KCGY$!_19qz%ibQ{NKTK4p+y10b5daA7U&d8T^@2EirQ~sRi zD2>qu44W7$SjRN5N1ak3(NzjI6A+Yd=fO-Krvo@P;V6mbjhDO*Y)~#bg0w2;BP{tS zGnNHlCL`r=aGoqzm%-EH-ZX)qQmxU?GRT9-Oc5y*^6AY-jc3^9VxrMK1(LAkK=>u zTJpGgc{1uq@pV%!_Rhqj6e+CNH}q2Snu62L`$1#_9$j5lXDunzJ3k)Hug?;zVlB2R zrtq6hM&kvI9v?R{t48&VUQMad-sbO{*Be*+sbzGXf&+9a8f&Bzv0)$^L%>2z44lS~ z(}j30eHOnAoX4l>^LQxG+&@V-_fONk{bpeF=JmcBxZRJX@f)}cl+w#UDG&@OX)&M# zBDfL=geu3%ieQ&G?PO2Njb-92pH7|)LI3m9bRt+u-QOi{2|~$4aFluT`t1BHv<{uu zCFwMsPpq?pLBFp>Ym3|Xd2`(j1i~lPLN$F?P|maK%*`NNY_xi3qpQ=i?0KzSX_klK z`;%1ayiyL|BAbeqDTf!$i)rXg6o=i!X<(YUt=yH>WF*i}6fQ3t?R>vBYj?ZFR88&N zU&nJ&v)q$P7iZ!1c|Xxe59MApyj<106dDpys>OaLoe?MEUF0~~%B|DWzPV z(a<7P+Mo8clBivU<)W8+0_*0z@MZfpv0p0g zXXf!UWQ-ZX$0<{)rOwV9cWpH%m#%{Sdhl+2J-CbY#d-%B{<&77H!vmr*$u>qqVatNq*{6_WDR*71tg zB#}I+Ho_Toc+%_jh`t|rl>8(jHCXp=7LkhW*5yr!2>+=>nze|O!cjJV-@M+qniKSv7cFjTGiHsT;5DhqK4W;r&h_6ewoS&%oR=+I96R3A#$GZVQFX zaD2S1mPVn|K>4_F(wJp0bJ0N|A%>3I%KcUIBsj=wolA8VN)!_7W#Y6KmDTfk74*Nj z$`^BsEM_~oy^Y*gm&Kb|^0KI&^{R>CY?Z8bJ5l9!y5A_|6NM|KtPG~9#&WTU+>X_z zI=v`N67kgN;-niLOi~w#oA7unCEvV`4a;KkvJ;AoX1(G4zIxh81mt^hKF&=F_3ODL z-?i?S*W-#jS%voZD}hAqVsVm9&CiRM83wV;C|e)a7H8vB>f0d*L$>&VAo#A8G6^Cr zUXSlLf8V^`xEdVYOfODC>g2Yxn4Jz*;fQwM>Lw*Ea1oy0^`c5wjRorBK6Z?NI619o zSG2Zl4?;==V;6D}!(GNAhV`lmVpwI8!LweZ0h|F@HT#OvTO}Lio4b7fY}rp-%@FN3 z$c!2%?aN-UIL`%Zrzi5X-cd8x_wCrCF+VRWx9ZKbcTtPnEcP>N`F3zu8BYTp@nSey z7aGg-GL9VoCzau>xr|{~5Vh3hNnzaRoK@SQaC$fmoff%B8Qdb1ZiT_l>LQm&Vs1&S-AoSL>~wcGVb9LI{O;AKQkwpR1w zM5P#rUbeO4Nv>6oDU(WWdUMs4yOmX7F&_>BwMjKMA2M2~HO|Y`({So4pZ~PG1L*%1 z$#_hRGvwbN2!azt)g*}0Ds>v){C)F!^J=(UE+?DSMQPmFKVMyyJMq?He4^|x&Ijd4 zf7DIQr-9S3Rzb44ZtH9vn6#zJ-89^lA!5crC4uE)JVg-_gG{y%Niq*H%@EQW*9#EI zQms$2nf7IIaCR3BAdom3%r%O+&g61fz0=Xz35acpWwKK5HIg@1{b?efx=DsoC!OM8l*ptCx2N^9;xdsK=ljv*ZFvY`x^}%x z=F^FJt=7A*UM}a1fY(NU**dOeY}zF05Co3mK>c>m(8cXeExPh;Z6 zHEKjG@-v9<(Tmege5DT5VQ1RWiWlo*xL$_CefJ`%os>G^abLPAD1}b>^7JOPu7&TF z%YdR(7yZEXezY9B3?s>Ybk>!lfy+Rm+Ez}a+hTB$pD>EmcP|LiPujnDMZZUif1w@q z|6=e_2?h^>K{~wm1tUHr$qmL2WL_Q6yvyVN{I7oGCr|NEtp zZwUJRr*?_7X~;V@Z!ZmdFoFB0Tf?4Hxq230W;sD__lYchGUvxPuMhPflHVL2CUoKA zQ@EB`u}cDcOIWj?$f@MbYrChx3%p9~3D^BEbSr`TD09MwdHzAa5)O0)ftNrCB3curOAtSmN_|Y_GO1HR z^o#Gj0yD{3vzj_jf6SLYrc1Sp!pBk>X{cDRXArN{D%p>fM6G6qe>&2x(c7C<7akYO zDFs9RXxJ}?B!5^&yk!n2BZps%S7F)T_rnK+c~bkiKR$tgOmlb)L4a^0S|5JVsC~hr zO|$BBoP+b3q8@(1AE97*y2`(sK+}DZ;T-$xi1egE=^36VDqbUhb>F;y1Fdt2`c?DA zO~134i=D6AWC};pNzBEbE^4b^y)gJL=yPJ}Z+_lFZeJurMD(Swy;!T{IP zCSj&=k9fcL_-yWCTFS;#nQSI|CdH11n5Jv1^sz#);*lz90bv3 z^O>IznmzVt#OLo}bkoD#48g)%A2<~#8=VqU5`?!lIwD_X8?#~J!UCE4RK#d9PeF1; z{NmB0`+^Rzz8KS#@Ua8kjly@{jumn;9B5gfaPQHB-_zfJ1aS2y+lWzvdODaL+NKniEzlqE z`X{?r3h>#<+66>g+17~o8qX-_2^`OSTAz@SJSRK2cG!knbjL{y{X-)GaTe%ufTSp9 z>J(Py4XJ5B|F1H>b9hRiZrOq3WTt@1BR<C7EmV_{-Y~30bHn@gA$!E6Zlz!Ch2o8Lw=mK!vvPRRD$MV4$J*^N4q+CznY&hN zAgT9h@qUpl7RuSCIiYq?{*LXAGGa`TOSa`LPyg?5|gHwc1~uk0;T) zYNMOgu4C<9Gk7Z*c>UnQ@cXyFbPe<87A(qM?-|nAb(*tpw zrp}`K$(GgiPIuJ_8D#+k1aKaF1 zx7`yEW8~m>N2v5dX=?V8@R~lJ&ZkQ}vo&7>Ofu%$f%`dHf^3cWFC&I4Y;Cj9_rmm7 z*cXG8UU63{B6b$%ooR?x)`~2XnsQ8xciO(V+!TGGkQnu~g5j1=mK8ZJ%k6F`()qaO z(lkg8A|ZrC$rOunM2dzR^y{|q^X4H>lo6@ zK_O4yI>Vcr&f*|dV7C^F#pr+wZ?#M*{pR%sDq}U^!w(mJcHHW>vFpG*(D5yd=6$Ac z6gVP*?*lPHxC+=#X^lrE^HHheONf~;YCI0`EUQITI7n@eDUk{O zASfd)MMb|Zr1{Op{SF}w?{CTf6E`aSc(!sX`mKMAf4-|fzyJUI?SF#? z`uF}k!1ZpNU_S~CcWA=k*G7Dx84zf)KXScA5>{Vk(w_2gx7!8WO*%e=WKUR}48fhK zpS*h++0%vkw1b*d$b-TnomFHC<~w|oa}2nfK}JyL6L8!9V8*>ePL*{F=i+b`K%4|q z?hrQ*c7x!06<)iH#UMe2x}MF!p(#$+;4(iJe)2gIW=T;5?y*ndSPU!{$KEXR9#I1n zA`a^WcQ1knPQeHKZ5DZe0pNlz3SewheI9J=4j~NGi8@WH*sn>?p7`~vBoHpKYJ zN36Y2kw1gtijji@91&Gy7Rr}?p$esl0-Pxpft-&8wAE4OpdJbdMnFH5O9Ze1cp|lj zxDKBEL+_rzieMUKTp{g?8dfGifS>=Pqko+eF8zg`@42cVRCwa6H0R{b!>0Ba)rK)s zCIG>Cfg~vk>6k6%id}&*FKx?D8D(Mu<&)M%DE=wS9cdnn3U5kZ~U!^y)Z%3)F7NW-*MKsSVQFiL9ISEEED(+M-!+xzu}f>KEKiJ=HFkGL52i9RI3 zl;mhM6b*^7csL>l!JwH&%G0O=kW2jjE<-M0Dm?#jQm;|$60@YFQ1)bq&4#dnt{qnl@skgYT{2*7tjQ z?_0V?TW0U9QAekGup!~ehcW0I;GEBVsGl{sqYlkY1QHd-&V8up1~bF5#WW++W#ky@ zqBmcR92r@QI-Gty)~0|Iz8!gWa_MU%nB;_bLu4It^g}0v8FZ+0dJJAZpMCgxH*DYt z_5uC>LZQ)7l154g;b@9Q{o*d7!H`DX$G(~)Wgr&jn5H?qwx-c-Jx+T%L3|I7-eUXY zTU)?rM6%+wzY@Mdu7@64S^1VKCG}E&CQ#5XAFD0;Q$oP8@MEM(;#HVQp}>zL09!`X zultc-&~x6sAPji37YeQ2-HU{1U-u(=FXW(V&4)s|eX*Z;^y1NiYbxUQsd#h}_v$3R zIZSsa9w6O56EW!3i}*0WD4P2507E@A4~U5rx4e5@M2`vYn2qGwjr6Izkv+RX?E3?= zkvt~6V>ThrZak34h614f|A0~%Jh7{=XIB)_y?cEqw}JbS9W#n}cH@Duwi&V6cuaan zH&M@SC@H`Lv+)?oj&3~JfEQu7XY*a`M)KqY#MK`H+LJhev|n%ABsfd2nO*9Q6|Y?D3FfF-BcO*`_F&;ZHxJvIa{ z;Jauie8Q>4T97Hqxo8{S5ZjZzvnYzsJ>!(Vp{)HrDoZ zrhp+)IV`$g+YNNaM? zr#w&?(}NFor${8hC#?_cb7XzO&t{*kxua$NmAy|LJv5Ll_s#d2X~btdMd4mZ#$wIG z`?hWL(^(VgOx8dRKN%JYNCQRi(tPyOz&qhRVa1`)- z+4Ht*WE%pQu^vbS>Ygp1sq2QLJ!D$j2KhOd(FXkV7CmmU`N17eMWRZR82EQbkEqnO ztw#`;V!cb>9B-mz+O{|KDKenyBxVt!PI|^OtB)3#4b*V-BgMD(V@!N^Pv%ozePMwck_PD!_O9~{1U4R;H6I_Dct?;rOfz2Tc~bcD9dz5=4VH=~q%Xe zR)AO54u~>*z7$^z;~Om$<`U$Sjij%o#gCE$7MP-)02)nHfrJn1`jc$KAVJTijE+9@?4^$il8^D*yRhp2O8k89j43P z$1Zy7o_9GSwkZGie@qIy(W)u7=>n<2aF|d@M-6N-_}m_nd6^u#gd`^T@g}ray_aH> zERH@dU`v|@6@dzb(-XR6?ePeP&&AxOyt{ev;pROav6@HUyGAd19eq1T@9~J$Jo?_% z{KJFae8Os64u^I%E`~kK{I0QYK4LX5g?El!40;>~yPDs8#A;p)#deL}%c}2eee(&k zai&T&QBh>TEV>L)*|l~P#Wo-~aO7f_s4k9=iZD1YFIor%TX1q<&vl#Q!3f4nt6qbX zqMe<$Sf(#$lc=Gp4wk6Q0^_*z&I~#6sRD#YcgG66um(-kiC_;nR8(h|NB}u?Hmgos zp#MLF@3}|A^(JzoQ@dvUiCy!d5#m9QcJ}J3(n7v9?0Cf*%33_4J<9CnvcDtSKJ5Zy z#u&hO-ZKSwn;2x{jS-upk!8B)ZaKV77?|Otr57F8up`mV<2bw6>XqYaJ?_!Ehj)s* zb#IXmqN8~}!Nx8IJ(~AoM(*}KCI~dyY~RcM>Lm_1?(us$UQxvdCz5SK-vfl6<3|++ zE;iT}SM)eLbHQh;AB=naA+Po~Y=#@-e{}m{ul7ASs(bt%Q}l#Sgtrg+|4$qQKs7xc zSOYJ&2+uyEUUTr+blh|B7=mZ}F|YPL5d4|#qwee`4ta}O4i4evXyG+}$%_VF;@aZE zQOOHOy%x^}A1}8G&++RxOz;%!Rk@zEft?x@*S6t(lz1G(YxUTVg zd1!dSU-nwxCpxQu{{ILS`mmiE1{vo*BYUm!OFUIvFba9CffoSWF!Ep{UN8!I!RRHv zDem!m;pj`8Qe5qaz2N^6ZxmPi5ij_>$PI;y&RoDEagz?M?c$bgKlWhN4o12uYGH|O z?Z+MM>jIapeGgf|+5QHjaJDZ79UT}lm#qgciObo=#vEN;h|Vr_vCH0tha;o2i;Zcz zx{#b*=v;|+0s8;@a8ZX1!bA3YS~s$@8(jdi&%{GEadfdM9Jyu^a(1EPuDuHn3G8WI zP!)L#XHmnOi2(L4JcKZ37aP-b%_8FLLZ@(h7aMr#;(nW(xT}k(vkP4ivv=XaKb^DK z7=x>en6nF=58AufKz;Wt9_T`HlAj@t*}Cu&#+bl9M(JiOk-`Lv*p}Xzc33 ziL{1LX7A!5q}`MmTwORZO4`IIFR6?H3iSV<8s+>pMF&?m&Lx*Nm)uJ@bF9avops)0a?ej4`9Y+yvpw@F*sSVk2bv3EiM^t0zY{ zdlX|&t+FXvo`!oA%Fb2bJ7khun33!DQgR_;dY(WPFhI z4~Ab3mzB{OKRWvO(P|E5Iz1ouev%;?0iFQ7KG1E$o~B$rCgvsFGRmXr>?L~Og=ey`ur%egvlvR2Wa;S7aVsYR;)s~>_I4R3!PUgg|xm_%>1 zrGNE6e~jiAk2d@N+>f>lg9f7)+ZP6A;;sj7`O3~WZF?Nrn!{G|3qyrw{I5^MXSOPF zBxkL)v(P6hHd~Z!ExKa`?JYVaJhVmQzAGdD*_bSVrPH`z{qA{ ze74FRTF+)4@k#Y*~E2 z(l>na1TY7!O?=gC@2E&{>%_`aan?6nyFmJ!b6--?4p(3~(B+CDi0fA0&=!Xr* zgsjdJfhX=d*`^Z(^L>49czFba7z_t1Z_g5w-eTdu{N1|z}`Z#9@K}Y735XR zZD}xUa6p3;CZjJcJsre@xPrn@l%b4YL6a(eDW#5%75b1E1olw0ll8@OaiEyFGs^~F zaz0|`g6N*Kw&l=$ds$I>i@!4C(tB_#3|H|P>L$hNzqc(IUk=;Rtw<|it``4htrkuF zEiUo}i^fED&}(J+V5YgCK-QYzdn^+3b#Ap<@9o>GKr-aJI@;o=-PS(sSa?j{6n1q$ z==GI$E@WWdc?8(8Xmi?+9}RX(aCs!;(YSr95NwbK{r?xt2Wb=fGOdpposo@Y3$qdY~=OCHdCYBS5qeU^IsODpoQX3(bw1 zZXPRb!vZHMa8VS#l{`Yo=Cy5GBR8|c+?(Kf+zE{NhrhR>Kl9r%7l41W(BH+Zh_$d4 zpeV&Il6A36DhJ3YrZk7XC4{T@^v{;7>?!MEw1wv_Htc4^e6o7bgl&F1I3k*KMEBX4 z;_rCw4lH8QIRC%`W~m@8XQ8`~yNxW{r< zE_~QPYbPPbT@4DNWP#!Mdko0VZ((;KerqvX5)ZvNAL#!dY>*n5GTQqDh#85JcRt#3 zZlpFU(rJwLd2@5(WMcRS(jK5+G5i5-m#|x*c{n!1PGrND4@e(m$|uZ%62ivl?g1~8 zE0Ny1uALxVF9Q{tC9fdbF)3tN9XCfDfJ=qSZ<^PPg=ocEopzi*!sLH})C#@|Xv^63 z*q-Ltf^cjOhd5ZERe9u6atY3uRVNktUtt5G%r*NL`_jGk zr9S&-4CYgifopK)=0v7Z{Q7*bz9T6r|2zWXAgQ0py4^bzz)oc3wNIFoMam^*T+}lO z+Bm2#rm4-&U;+C7uaM%Oc2ARG4JHg?jBlYQ8@N!!!2qAJ3$~`Cj!V>_XDmg44%66q zwTQOp41#Ud9z)+??CuCrJ9|f4b`HP6-n$C@f!z{&B?By_2hrezjOrvbq9FdLy;k#W z@Ez@f_?Q0rr7yjNQRGo=i!z6U;pWX0M`6QB8f}qH0d*M zF_+#j7%?l8ok`g#Aqd@@Wqu*fYLEbU+TDRVU0@884l=FI1(v4sfMvEss@($(j@$@? zLfltdoodtW9QK^?!QFVYea>jhs42pdYg>D%W7-zBb0tR(Vx1rtsBMib-}lU#75|+* zLv$^I#^L4rcd|A-M~j zpsT@m)Eg8-<4tOVFcKJKjSK)qur;K@esiY4PL8&G^w0r1VCcn~nyucEr}R6J=sVJb z(JYtZtcEN}iBS}-3$>*S35-H?aV?vnKMvg6IDO%Fb&4c{L|gA^thV*`sS60Q%BtC( zkJjuaqc_n1f2}qLnsNZD-P?wDXm2Cy?}q@N6oZ)#DfSuY!*R!#n$n4QqJAOr`49`KS*RlCRRx@#VPI&@WL4tlov!L|u`njY-I@Ecpl zhJh4NLg3j8Z4SWgFW*n^(ALXs4!=@^gS8ny(FFT^vw~xJJnaQgl>tfYou)w+e2J{F zmU>6gZr~;c!$)#pOK^7^X}~{_hl%8UyClJgUN^;&9Rc+JC$4wP`G+iBF=pLjmc~$&;jGgx|4ez&(l;@}} zQFLNxGmGfY<3C~{N6*Qq4p6C2%BcSMiEKXUzG4VAn{3s?{>HLNIEe{^chZyiWP0K$ z0c;G`O;IK_p)=jD)+rx9hnmpVd;8gnjhZ;|*rXY(bKQSe(zWz&y5!A0YI>g)>Okkt#RIxCfRO^-d@fE^rKR=#m?(q zfK=Qn4~#-vp#N9)MIs7dDB-p}&?XL{+O|jTu*N@rE9D!JU5a5eD15NlD;S5hC-B^5 z@x`3h%;>ve5+K24yO!!uQT6&yXlpdIN?yJ#sgbV(@e^u3s@LGgs2C z^jOCb?5u))^-z0srGwAv!(iOASf0U-4ut-ogF1xoXH%3`VF^~*5!(B^fi`FjMl6FU ze+2Nfd81hdCuV7~-!Y8kE70KV`6GV1^xb#Plekmo~y{C}V}Q(cfF zjKlcz*a}xv0o<#?R|sCn+v|T@cR~LD?bf1mM%6aGbSxEtYb$WRBV#U{OOy%zRp0>S zRV!9>)`|_fYsH4P)QT-4J*p5Cg{;y7w?VB~F?tZf-#7{(g{g>Lm6xtGDmRT+qO0aB zj$4|eJ_`B@--uF_$}sswtD>X(=?WeMzPc;Kiiu<}k%eF+KB`148?~@G>KGq2O2pRc zC@T5vu@f6Xp{dQuI%~F~h|&nT6YP`W3<@=Gst{=Q$D1-5LB%x43)K6ytQ*^)Wjkb z0=-ZfW&hytYp=_#!(6<0e3h=2pVWf~uG%`RMA(7D zY#J)qnUziTA%Xt?y9rQ+$~?XGpoJQhOSYlQW6GRvBZlZsVgL?M3ATuhFPN9Mm0*jQ zbnOX!kNL3ah?!VI0x6&}!Fj@IX?RWJ~5 z?l+o|)BE1>YI>8%*6xRy_@7bJ1R?7mf@EYWlo0z0BOp zCxMHT)OBFBiUcyzyUc0ic0X{ROKJtNbUnIk%uY_x#JUp|Cxt>+?yls?xG=g}Wo}1T znBw!6iG#+yz0!i2Yhu)<^js~x@3~-GSBvC-R+SVhMlWw&G+Hn4w+EICP4HDSxMwAm z?W}Kyy=5=kZHz|paIw6XXT5s;CRr*f*TLyuF)t zNUTD6DHyv-O!kAhi*w~9lsG>-N%bz5t(b@G}*7_j~B~p)ZhJV zbSFxwV7%B3&DC+LIH}B(a6j9tsI_7&IlRnu8&Opn#V)1ztQ4LPE96LFadd38M8Ga) zuCtSOymBCw(4=iOPAnB4k|8l6Co;jP7}lxSV?TAVz+DA-jdji#R0$>z@_r2W&N=FBBX}j#-^dqh(&*nQXrk`26fH?aZgCu%ezHNa_vZ zA4x*t&ZyspS6^=+CC0cR|A!BxxL*uyssHP`OJ#L`^rP7;{QcqQR{()u{^D;^0o(rz zg;)Q^-^k~k1#GvYF2hByinVq%N|!4_PPSzUn<&&nyz1eKwW6zHtt7{!jf%C5U3ds8 zrlJOZ!RySXiVcB%6eK^iDO{g5$G+hNC4iCEj=c8{gtdaYz>1S)I#v6!jV62}>O zJ4jhK1fl*ho2M6Z`P#>9xmx^K%ayBjdH_un5~bq?p0Alyk5B)6Q@g6%8q-7>eLXr;D!K?1q71KA?}M7KkGtAzcd)H{wVP9 zPx1SYP!$FdOQtmiO&}ram??U!elr!-MU7l4w~HeUs(_9xep3M@d#G?Vs|$>Z&O=3+ z2~hJ`sbGkc^Z_Xcq0$kLYB2o$5nTVFhknH zW(%c7nss%PUcv z18uh3v{IU1sMBj`gB2cL%#cbMxgl9i`T`@0^8F@|8T#T(wXelV73>4CfH*;W6Lu~Z3K2A^G@v1(6fofFSRv%n-3m0hp z&F&CjiXjTkB@iE*E_Ox00GLLy0K?dhq74CbvroeB@f5~LfSu=_2I)|wn}85n!fn)E z-v+dIaQhFd15i75sSSYdjkRu5FA;}^Qe0MP0n8tE#$Mitb_8Kp86m&VaP2aGPyv$ofpZ()Rz~zaE3DVv1c<+6E0NUueRT zs?p7);HadJu!v#ePwR)mj$9?Qu4SizzqdWLqMW40)sX9f= zUARdpYG6jCX_K+pu%Y+0ne!^lnI8g&B{mFAj;$=U=7;1r z7!qR{s2DU`L|}+txPW0$M=bIz21f?p#cl%RH~Hf(Zm3s?1}7sx5kGwMdN0WbD*QyK zK!8Rv1?)G?wQx6R3T+S^*i;*oL(Bxzt29S`Di4oq6L6g#JX@)565MGg>@(vzC?>Cr zh|meFa~Ll`sbprOuPSXD6SBbZs_}yvF;m)UU|t6nFdCqw1yw9DR1SxcatC&e zVY1y2NXuD3XUY(GplJ&*9VDH=p_L|R{pn)VVc^ABX+W$!eT%Vgt5nUNnEPBEG(qU7 zi4EK%FpQf>W3R#VgWH&meR=v~D|O}XXm44M?dox3Mq1y%%f z2nHBILAn5cZh>q-c1*AU(on8*K5B+xSUYV>23GJT5GJiV+ulL!m<4W(pwye7rxP9M zj{PQ;>$5fnW1T^7OrU|SRA^%RVsM-T2#85yBT~9}3Cw;%TY76T7?DYjLB0!}7gFIn zIvE|{1ACk;2iwuHUkR{P6OI8N!PFTHb%~FNk5A)FoM27+Nmc*g@M*kuo!1LJ}}QMKE}^gU<*do~k} zjldG5)9N(WB%~^e9xMX-|Fhl^tyzyK9K1|jgO;#zO}q+@1My)<)JiL@tY&bcCh-OW zBQYN=Ar}p&V@^Hu4E8>y->#hc(L8SP{m+twq3(8fi*jNgrWh53>MU2CQf?1azP+Pu z(j9@yO9>&^y-)`cG|>P5@KQB|Yf?|sLiA;}>mra09BPRE$TnrfTyLbqK!C-xqa2Y+ z!hdR9OWC2I7*EpwOj&^?e}|r8K5i&1uuyH+Rg9-iodvkR)huH?Y^p87!y1dQO<57_ zdI~}`2rPgUKtx+Ac%8TA@*sL{S26aEo}%^$PtCFcwbg zT8wTS0Gnc5M3>STTYZ9N11p53+6X@xb!c}s^_%7x$DpL+{6u9pnE3QGO$T!MvlSiC z|0_$`dAX_s`TIqhj_?dc$BrIb6&^Q|kY@8ptlvA@q5>J4*voYsCMWC4;d4|R!x;PU zj9g00-qC|f5t}6!^sUX=L*p@JeOki}g~z@~2}|*@FUwAyNADV|_883v_O6#IK3x8M zsoujBFE6#ngvab?L;ErAJ*WY3UD4(L$4~fB`^nS)&vlob|NYiyukiPWpMQ}d@E4c= zCLK)vb00sLtdstMzfFPp7x-Wdo#RY{h4Dj}VcITPCvZ%}H24Qm_2tUC6pe?TtE|sP zub^(V4m0b>Uv4fzr)j?Qx9*Do)?I7%adW#cb zFfGQjsbDYxoP4Gep*85O{=x3SY#Tc15X?W={eshPcb9)K{7jRnR3aFbV*ZR6&15jS zq_o35*c$#W_h9FMp5(ks_DXibgyV3V*RI!PxCurpY9XNQKdA?gqbplSJMOj)yYaS` z4OG+LbN67-|Np$rMcBD?JKTd!Y!6OmNvbEB`0;;++7$;`HaW{CR*4qoez3> zJCk>CDwc?6;2j)=kIC~1d-rVMWq|YQNk_8tKryo6B5a@T11`d#M!nDH`kvb9>bvD4 z+{(+r(R#ULeJN0R+q)D1Tl|DAQa0p*=bm$UoM^=mDq@qO$-`e@=UKXly!~9t8WsuCmM?JS z_B6JvF3&@&_*J5KUTkzTjY9JxD^8QkQaoSCXX>5cJlZ|&+#s4u_XUn&5p8E@h6~Ae z+|4&XCSM*po6Z{k!^UYQ&YpR&4UULEh3hX6#BW2Duf36NtN$$P4+o9>sR=U_piB9QQc>UO5N@ zH~vRX|9|@GN8}0o!_RjE0)No|w}^26sK^O7tu$5B1B5I(%s9tAonfM5F3UJ#vi2my z2^zlh@Hb*!j5wF+!8r$x9n3Mwbd=^!$J>u5nQ;&P5!}y%xtqcxjr)X>iyFC7F}GIQ z2jLY=4olp@n0!dsKv)xX9QBhMbhE|Y>arL0y%Psy{N3nM_XTtu-@P-=N>Doo4)X-hZ71&meMEtuE)LuA zb}Izwg~5MnjLv>8_>_kEcSpU|8ZYI9Et3IiR9ritBA=q)Zb(k$D_qBzYjNtp5902tHzrqs{ zJlmZhz`1nWt$#Lj`SyqbI6%n(?DjD8(zfIP&olo;XY;|cv&5atw8WXi7&##j@msQGRvPq_cyuB z)wot_u44HK+1xII!)h;ks?{r*-feHt3|$PXA?-90TCeUSrR(YaaGaTD>WO+LQE2p< z)snalNSS)(uDrOqk(USq`D`>CU5A#9+Tx}qquAqBcp{$M!1q3^mXhi7!Py{^yc&$J zQ%X3oYPaNNOU`SdSu1h7e;ge&8i?=ttS5KZ>vCu~)>pNsj(^?E`{gTD)wKhEDs z;H8WXpU(2PeaQe3zG?>dJnjuDa<|hgg>&t@gqFXK!m&J9Sp*xyRPipSHTv;#W!?+~ z?k}sCi*__eK)nsE%I#36P!1&XH)^+^>V`Xu$xM{y+I}W}QX96KspIRU7U?8z_OD7y zsTI0gou3q=rA9e)+J`6lXD8K*TUCxN%8m0{cC?%+%k^pbxU%kCr4s$qL2DU5j-L!; z7)JS`eO(GJ-Il+NIBb(WZZZ0WLt-+9b~EM_LjN;*8%jeap%jx(d*wgdo5K72M^ zeyt;eS3mko;sihZeD5Ie%Xj}aGx9Zl1C4x}{O7xpP{3~f`(E1q$ma*R{^yMT zU;VRJ^!I!C^M^-&o3Q_n*M`>`#ideUJXxO%HoHV&K5WaH$h(R&-=5+~i;+STxny=OKrHoLkAm;_lixdaP{e*>l`ao#+ zKZ@z)X+3y-TukQIx$CRbd2>-rLC~&GQD$98jLB|yAT(-e;qM~19i?g1t(4wt04pI3 zZ&d|xv?x|(Sm_WDs#`9(ArOm_(Cw4PDH{v6JrF3$&H@&98Ak$lWD#j%HRw-5sHC%i z^xc$VmC-VbS+gbv=SGCq!?rj<2r&6M-eH5LfDWPAg@uiWD^iLGyh`Z8AH;&w7h_bu zBlBYoVc-kI_aZnMVa%8xm7PV7^+9K0vuyH!Not~qtUx_QKnqC<{KtX@vM8^Pc+?6! z=n;gEAW^KT)gC;W_2+m68q-?3#r|jR@Lt}*l6e=z^M6nTqtk=Wt`OSuV(Eq{r}7$n zbZ*ZJ1(^tt(i7RTd2Y|3d?3OhnZg}W6u_$|*U|bm6BzUI*~kRC#e65l;?WdkzYLKl z0a>g+Y_)P8y+=sn&QU{LPyf1edq&byCX)?@<6_7k%*JDBnw=cctzi#LV3+;QnF8%e z)-^j0*_`UUF1HSootZYDsnI9(u){+4ko~S(o@jx|-9Yc}`)j}Rw%s|G%~Ve9d0|;% z&tMkx|9_hJ^6gQBZmhHW^qQBp+3$F^s;9c|fvS_hew2lJ$!sE=BW3FptD;;Z@d8&1CIhkCHRh|%hik1C7Ee)1rFL;Y*56Mk&HMeE_<7(o7#i2BtKvFz+Pzccz-8imA$G?X`-xaP zJIEF*?d52ZF4U*7>9ma|8u3OW)I6;h7SqOjofgyC^!0tS5xDN&CT}lir0_T*n*v zQmVhqm9mvi_y$cZ!qRP`5+2@0mHtWOGFGkT?-nbhrYU4Q!;@TKG6`0Okzv1jby}(- zuiI^>y0{NdJT|jn3E6cJiXODBdX58!e=yyW?Et zK6z5P9ZMB;L1tI$gbiH9AYzwJ%y^G< z&wFx)&zLAmVJRZYVmvHI#JDI&LuQ)K2Zn7+YFR>-o2=pIrc98e16|v7Anc) zAip|U1v^Tn-dGO{we>0UWYnjy)upp!NPYG!} zdD2cHTg_yOiF|?5bslHZAFG*4;i_InV$oncOl-+3)#rn$JcVC>9#7_djZ~oXt}lk% zpp=u*_(tsIxhX!k>b2A;F!z7Q??0xjPsr_tEQe#5$q*|@?Av1eE3&VUJp%EG(3Q!M z0;32yei@vfXZ0~tVe#Z-Q>3pUGZqUAX9*kElsAnfU{pra8JWv8mNrY_NuU}iFF|>Q zcs4nf%MG;)+I_TR zk-d5Sl{L|nv5(cTk0C#n@YlchC#*4~mm5>IRz$#-Rn)&FmpXbgJwQ5cW?RC<=9DN2 zP?@vbo?nrDBW&OP^bB9zItA0)QwC|J!Fuo$wT~Pa973=reWSu03acljaALWbS_34! zLVLcU3i$ae3((nGxNWF36y_!H7aDL33$>HZFbC zYv2te2m?85&^BVu`#%#I^*EM{c!KK>MOTG*ffPVf-0v)f|1r;7#TPOiM70 zcqK``w*EWeQo&*7$vXky#E6>2h4_+~kLoEe0o+FSz{YE&0LA}+R%6ZQAfJd6ga>ax z?nAf;InpTO#%tw|fA7zsX$%mQ!9z7Qmn@SZPz7ewm_*7pCWA?Pv}Cw#V6;_}CtU%d z0BMFK*7$(PPM}wEcVb8BHkTtpw44Bujs6Av;sfeL%PDCmb5LueMV?7P9iU%y<^u=1 zA^-oNmYbbehax=*L=j+yDs_aY22^Umf1!ya@>GgzWJ6=L%$E+VKmg4Q+TF; zJ}N5?5A2E$IVoU^G67PBZyDww14%R0hOPuP^Ovyyz-3*4^e$Cm@SHLN1=R_00svis zJ)&9>PqD)B$NUoNdoOByY;HwWO+klxk)f*&{jyw4Q=I~>IIA6FiKJoE&=9Z zfQ?(ictFU31f2*>54zJ@4W@8qxBJX>&Q!k9+qZA@Ix$B92AD#Ak!57SJm=hbPnnmf z-*BoEf-2h@Akie8O~&ekO2}Q=K;Z0S(*k7>njE8#wQ29birqi_3l!@MKwo4$0f`|M zQ%$l=I6`G|H-#c5_8cr05JFCUE&?P5m|fwyp9(EDB2Y7&4#eNIdBfE`u{K+=kglTD z3#;ebYc>bbH8!;77@?TO^0cLht%p)8-0NiOy&F#FBNKxI&+$vdd9vkP5uo*E$S=Vn ztbyUS!h8hn!<7!;lu3P1n;3kfeFOk^$ss6Jw#h+H=>*p)$J7z#%mK;&KLq`Rbs9r7 zB4QB|EotfLc)Xlw37?QDbloSlfl*!eksSlGH0_bF$+t6|VC+aoFhhdHRdbS~6Kgsx z1=1ro4U`FkwPT=|G+H_@%~`wB9AaGfmgXLG0(@i=`|x}ud68ZZ6R~^RbvGdaY$A5^ ziI`>|Mo0$9$p;-Mbj0yQ8#Ifk14h;?m_FU5JH$q!GnD7>04Hz+*#pONDN5w2r&X8 zDB+Pz?a*ZJx~UvZ{?kBgBuh-_QAHSWI?aYL#y%xTfqf?iMy_pgvGqGgd1Tfbc20Ha zfy|H^u~QOENVuYqe42MP2?VG=4Kzs26CLT>u;fDM(lF5TfQ|(4Dv()+zzxC-2C)LI zNM`54DSp^Y2312Fb^wf}>uD3z6X;iFXDF!c zw7Y2;PD&y|7xz(6oLn(Sp->!rLXYKrL1)RFL24ds!vauzdw#Nuoa)ew0P_ez{{LYD zB0aPWG`!NPEWgqEP|soS)iDpIF-vlMhwTY*Cu1 zzP998_J=7)vhW?b04&woN@HcU)hcA%!98CP|Xzbt$|z46FvU7B9O5QxK7N zWp*a8$LXfeS~3xGuVlt(wgEOuM%Jdil6Ai9tCo^Uyagndn(ESrv^eh(oy3wu2RuJE zc>C=fvql%O<06Y0IsD%7sBn?Ev7tDw^L={XlYR=}E8}v`_4aLLjwd9-+Q-xFp z*3F1`G78zN(IVg^a%da9MQ>OKN)rlJAZ}Sl<E5J#)`ew;=aRGrh zfO0uM;r*uk*2X0+)+ZN0N~9-rfzlE~A4WIE8!Y{?+Qv(Z1@bY7;vNJtz}+DV;$RFB zor|#9rl+$>&&UOQ;`j6HzbE?af>H?x8554S<)P<~HrlJHP}YaO^hu{C6i4p=e}rIR zZD&k%{6Sp5rW6U}`eATo%=v){K>^Y`vOEm5HJg@)r$wPpO;=oJ zCRH-QTv52@9nrFwunY?1s^SS2c71r@tSOQR&~lNjAearh@6&q+E*-K3DBKhTbk8yo z5gFkMBU_(2B?YorSNX2nUqa152Iv`e|-j zh|8Zr z>@x;Q+6NJ}?AE3GQ*?_?a|oab^c78gO#(j>$Ka{7Os57d2(^k zg>bMSTIUTWe7E)Ff~_8n-B;^NBBwSc6A_ev{Qo6mC@`;4>Y($J?}hq(FX{vBN$#}; zHbll4a$Tt&Ugl@rS6r%2O>6JCSczQ$D07Uf`(4nfOoPr`9}1 zN8HrL=$rcZ<=$%H_0DR7wbyc~?f9xSKeeU7FL76YuN+5?aN;CWsT(r&tI;yfLS}bk zTSX4?|Bo_T=CIhk%;S7=@l;ZK=)55{w<3oZBZCJHBxHAQ+tf{3*@k+E3#Gr{f;vt#|N% zTRIIAn~v}oI>E`KO&idkzz1$=neGbT8tWMn+U8nnmU3Kv!W zJZr}sVGBqbiWGC_Cvdc#TnN7_z(s4;Lv4Nl9hm6TBB-r zcP#E!J~ArL((9e{YI(0^HXFq>x~pvolCFp z#PsP9WqU?tm+WDeZG?43GDey?keK80Ew&N2~hGFluC>d``Z z?a5sBrZRr=m0CYh>)}(3zRJCroWp<7#4N2|vMauPtWqzfWAa_gUlZv`Io61#uNyfT z`|0w$f(zM1FIDk1LSed_N|lq>sdBaDz4dgw$?a_^8oj-Y3~npY%Vy(h)bU2PO<%58 zl5*Z)cDuc!m!h|HIIZ6I01bF_(Cc`+e5SGMDAVQFZs9rde!UA#MAONIT-Z&^H61ZAE`ID zT~Q2Nm9ypeDy!6!+r-@cv9f9%w4_L?N(jUQhic zNg8(Lj5tl)7gq%}8Wj6-w$Z4y)BVb<-R!OLhXwT(}sH>+v3KMuaz52{*QP>a|r{P5JUS zo_^J{+TIRU;okT@U+GoL-MhNazw9*b;eO5C_ogP7l&F1Ic>G$wNNs$2^s)ZtaH*CY9U;#Tsrgj84E zzga>mwOft5JPH1Dg*0gjDIG`I|9$&r?`l*M)#|deOy2j`ekCMbwc_4pBj*cM>w#Rc z6I>L>Ay;-8DW-fb61#8?C#%^A<1CR)@a#e-FNdQ97ddbBdeef)^^&URwj9hZX5Mb3 z;tC`$F9WVl)-UD~^%9g@Vt!k`i9)XA&3!hT7>+O3<^0H>5et{KtJ*BtNQDMjS@K`D zm4}<=m1mHgcN*%$Qs`aqG9T$C*jo;-eigM+8d6kSU>bXXO zQ7kb^R)>{kX&j4vKLVlK7QYn;K5NAcfrv}Dnp!Hxz8_WnZYIy zoIkX>QSV&54lGu^kkVDdVpVcs#t@}RrZ~T$v1M}*AQALgXh#@U2~!xhnrM6{WX{)(Us4bicIj%QrKe_8TNdwX1fc=gBQno=WD*H?4Nm#O*^nysRzi z1?677oA$0N!Mmj^vGv^#R>kpD>`2$c(KcIK$JY^H{9hG^v*tRCSwWh|8duqIty3zu z{ek#!DyC+Md`VW@V(E4{hz|1QL_Xy6u1ckVd=;$6u6vl!#QN2$+U$8UOSw79r+Uem znA-qyYTBAr?zbap9L~i&ji4{u+AJ>RVonS-+Vjgvs#OgulVWOmchmKCiyLvd7!JhB zq?}p|nJiRl^+Gulh~1>qpLTYD{C`F^9y8-~`}em3!3Cmh2tAJ<&<%}t>bX)VWBifdUP6oUOxH@29HnZUdVu(@ulv=t|9Z*esZbbT-}!;lhj zy&O-GghUsU140r_!%WkSwA$?wLb6t?lVqaZ$PG%XkcflCAy2B7OLZoVVR<#b@y;^7 zwA^Xj`+Bjl6!cbF57nkW9WOsruIGi(L#bXLDplV@-&MIAL{(8P7K6pfdRq^xVei{F zSA(@Y&P<1?+N3*1ev3?KbeodPp7bD{mtnTa>u9mst3~f_`cpX_yNmi`SDoBoBqw6o z`%JZzTg&n|-48|Y3qu&wmD_bR9hVoCO7EfESf{JuSt(eHUaw}J%UCuL?cXdHWd%SD zx3^V)OO8Zp^Flk~y&2s$OY>RwD(`9dm4|pC<3VYXD_ObecLTA`HhdKtb;_CBiFe+Y zs`*HB77kpxu1md}%iLlbmacD+BVw7J!F&&0XF8FMI#7q5X=k3h-sS?;0s`*4*U|Y^ zz7rVty?0q9+bJ|Mcd>0H@SrqAMJX@);;k!G2sZ-05F$sqzL3}u<#JoO^4{k>%k+dv ztTubWkbc(u#WVUn(fsot&j082=_mYX^WN^ED430=7U6rm;N*(Cw7C9K#_6jbu|jWyEX6;UNt!1dEthLs?Elm%fw zC9Z@M<48aWIQj3#?>`aC)r1NmtKk?Wm=6a&DKo+l`3WZoJf&!#GdQ-48wXP+@DvCh zoYq9@5}cok#XiPTiC9LEPNjEF2Qw*HvmC3(Kc@2^maF&qL!7}56pt6JqNcG@(lPRA((pDF73SNstIQ=+TEeKC^ zDt=D!>h4AV4y(=~@>eaEcm2+CA$5MC(#!xT!M!C6tsaBdjh+#?IBBcHfzep z>QVvjlV7+(@IFmP_y^g_&H@3Bvf@;e)6sw7IC?u;KHfN^Iaq;jD94-6BgF#&IgkLm zKMnx(Nb_}|2NPXFC)e{jVdQWOI=}aHGZ4$b#CF71CKMEoyo{zl-@DN|&t#YL;Z~mL6(IDteNIu!!q+qQ|2KLe zMCfy)1nv3xtb0DgJ_t^mEoP_0(Co1%Gd_P0t(%@7R|qEGx}m8c*{GD1k|2C-kt2## zwkR8BE-Zo6r{atzixi|##IK&*y02&h>#H$!2_HLHyV3ETug3}n84l)S;j6*Se`cxD zBYOQ`Atn-sS(VN=I1oh*r1MQUAfuv($HrDK|6He{Mq*{cq5~C%8h3v3~XK z+LLXZQN!|dFgv$&Dd00nf5!8l93Lsr=csBIaN5c;Mx@s`qMRdf9PzY1p(1$+I{5s& zjj-sBT^Rc3IskDJ>~aQB6azYirFjE14dnl`gu9MN3FIw1v+c}ek$J>z*=9O35M1K# zg~9ob_|kZ_`seA2=cbiRsj4@f^JTK-Xs|a7D52Ewp?%ICF{E4RmwWz5E#A$J=8dE; zuxw=)tJLNE{t}xo%lpKt)f#x!hq&~xOy;tMhf81$i=o8PKM9rQ&5#m@nBe zIuc7J2SOgGf{fk*7Yu=B+YCojE9kE7s?#0|q~=`Q zom7+#9jkE@9BE9J+#TI|Im@Q$TW5H8*IAy$vh3D!xg4Ew=>=B2>V#ORGQXH6HbL+_IDiAjf0W+L51O!}bnQ z0p}K(!myfBg^kqqn1D?12Zu5u-jH;v8EL0R;ionu4euYo|A`#M|2FQ@a`tyV`}3du z&Od;EeyBfx=CA!te=9ZgZ~yZmUw2~%`$=qgWF-uJt)CAx4g?zNk3w$&!s<>;+Eaeq z?S6@Hla3olvM21E4AGsWpQ3x2*wcmTw1b>fz(HXromC(O^BKO$H3q`XU?ZrD3AAm0 zFyql78D-nTzBob!a880P?r?4%>IR4FRYdJF6$5|@bvs)?L(@53UCVqS{NzgjW_d$6 zxW_&P!eOXbYB#%03S@w!#qxswbnM z8N6{{<4umKjJQC$Hy@(?6e-Z@Sp*6`7Eo44 zSwMQ|NH7lcV{wTCY#^RQ<>6chNB_{OC$i!&4G>po^+g3M6EMI({r6t{Ed;pq7d5=+ zvVx<+6L+z>pl}}6wWlm?m@;Js5P}!TlAEIh|2TgCfH4<16_Nkg zsh`u?CFV(qA??W!lMQh*9S_H7(U7y@)LEJ$u7q!`(JT|)(ibU=PMr0Kn}0z@r{#wY z)jJ_QX|tMwkpNQDPbN*FvS1e~2xdw`3flvC8?f${@h*bBNw7G-5=$`V=AaWTrQIYu zn%Gl;UkQhPcz6uzMkEbmMLJi61PXIvZ~Dvzo+7j?GdGZ9>^~M=O)RX<8f=<=%BEWJ89;aHQ#|cKKsh>_TJ#OYv~c|jYFr8R15W56j#T5Q2Al+Aw>Lk; zYP^n`fVBE!M0=Jd@H&d9_i+*RJxg}Hjch9NEChq}Z+IS$fUq z$Ob+K!oR8-zvI|AQDnCo2i13&Xnsec`5%{^z_Z52@5t1C2R7R2O~9cghkFxn9Gk%7 zV-tKC>!BIF>1{(m9u@ZjD1=nVz4<=Zl4;K7^Z#Qn({3FE; zwBV*2P9pe7r|SF{@V9{sUF`H7|!&?HKSrIos?v1#3fV%t*{izRGw zAeI4Ji8k$oj-F~O58;%bEc^J8Ar`t@O8^-G*5s^DI8bQQqYrkc0FvOH)*ALDP@nL# zUuSD!E17>~?^8vOHRQm3<9+5DaqCafaW8zGg;fd81c(dXrOLW4J5H-8~u&sY1-UhaXMjEve{{+*ZC8O*U0fciKuwXw^A zas?gfYJb=SVGnz1Ir`|Lv0-|dj`oz=nR&n(~g&hCQmfebscj0u42Kl|tTIPKiD zwI}E~*d8Y7+%s?rv>X6*5fq%+f`j*2i|7r`0*)^`-ri~1o&lzh2Y^6#d&_&ebHi31 zkk$@SegS2)M?bAZhg+fcj=q$O{7da^rqHDMpRA2 zOhME{Pk(0f(Im6Z8n$}8k;C=q9Y0=^@sv|t=;0182&cPrv$oKIhEjOX4A8m2`lg+p zyLW7T_C@xlV!cG2Zsz+I-qi_c4x9eCwd<$+3zrrB6+yikSMiwE*yA>>Ai?2E!m?V0Fiy)GOBOBFC&|AQ)2~17+lsC_KMryDZOI{Gep3 zQK`H{x1~okc$Na(v{}Qov(`BS)RS~0SxuU8!b^On=hoCIf^ce`$_2sb+0JAx2uTiH z=`@`mn$Q^{C#tZ{-uSvdOB+Us!#u4QVou8_Vs`>I3UHW$^#Y737w;PrEIvEVphkfX zbEsaRW&A9nhM`2E)_Y2OLtWgOrr>u8jVHguN~LeT_W! ziE_7X`>M;RM(?R*GI?kvz*=pYY!c0^DFGsehaHZ$rsNb{5y7!N<84ZBpe^~;#W>@uVkt9Qmag&CcZ4s*hRE@+e3 z(W)FrZEp@^RDpg(GU^hg-P-hOp47An%F>#J%qLhy^3^+H8alJVp^~jn;yGiF z@wD3-4ThTijH%n41{waunXptxEcp#KxCcJl*1iVns3cR94 zBP7;A8i?aSrHb~W|#6#0UMZz_86gpuV|K5RbC2cjZ!Qg#+%;sfw=o*%oaEdpvdev&lHOb@X<_5NR-=QKch+;Jh^RHP|KEUU`clec6jd4pnuqMrIZ` z$L)7!z{IBv5DwK{Du}`wG?6EQJz!g++N%TrkqXM4?2`{QdN}}@YXQnz;?@@G{|GMv#`4bYRbqL^F?V@8V!o&Q9qOhteIQQ+7*tHu+#W8v7Hh?UKi#cqe7F zTfRdF!6y6VJB44pCIDx*{Z4^bl<~pNWc$>2AfaRXk%fVq4VJ-`9QMw3=yNa*X1D!* zr}FoFhI{RQa`^$L@*OnlZu=d&=t!R+FCX&%ui6BFa(X;622NoSj&+2b`rt6>?DoN- z36AB5oyvD$@E4YkytDf}zPBQSC(3Tw@^*ZsW)8OsU$0@ABvHcnk z+4$!*!7RJ&cN$+O;_kNpiTvYp8sFE1uI#km=QO@f#NBDXQ-p>S{e4d3`^;b!$p62F z41JhRb%%_7p7A-2@oOSgc4*{x8UrT+?9#|VjX2TB??j{5#HQ@F--$$r_?Nw-6iB|#n|JM*h``g7zvj3Aa|U$Co?WZ+{ZssUl>rLNmmyRXFHRdoT9duv10YUWKLytyS!? z{%$WGslsa)KiwR&RN-Wd*?Y0q2752OcG=J^GHVr&$;E!4@lF+XrqzuyYZZ?%?Y_;h zQ-z(Qy!#yGWR)>OLH_>@J)Pga>9A9ceaOB0L+)gp*~Vkva`B*-_&rvI-#!+) zrDmyOpANfK>}eT0q5O72>BgG1ipL1GA4I%Ug`H{PRNPhQPS%=@qmDQdnPC&KkA-fs zS$g4Qu-U3O;>KlF1nq>T+6S-x!KZ!D<8rvIjLP`d z(Z`QwaY)nY_^|VnjM0ejMBwe2<{Nf&NI_I5c z_G>Y={F@g3L}5VGE-lmm;QK)!W2ATZ_=$ z`{dn6S=JcxU!Ky_Mnt#%jRkOjr!fm)Uo6f%ZD@#WC~LWJ_bLAW-_rLgzvJ6_ES)Va zIFnsU&?hN2O7vMu+&wF3EzxG-p)6`Am*scb|9jG}X%Fcg5#7`##&=F2l=Tc6 zaRSkXTHnjdHIx&+Q(36$#Z+SD`gpl0OILS&?b3^AMTA0GY!6T;82K#p&z2iI6lCq# zBQDB9%mQgw%hY3JDAuM$6#5Hx;{O9XjKlNxVE({$meR-qa)46y=Te$?cV1roya4;sVC%F8p&q^~jX-~8^^>BKhxePC|^tOxnwX#{!JatjQGg$^i? zj>%|4OHX_7Ag&PT%k3IQQ!ngJ6T;Eiv!8b?I;_3$a#zH1ENLJ zT82Z5?PW>nP5sK8OYfnr&|Jl>FE{C|{(H-S@!_y+-AXh9#%S?x=4esZzs5yAV4<+Y zHhN8z59XTluq^hT_ZTD=>)dL!-dm?vfo#ZkRg}edyUlspHt^`YVeD#z(DExC9Y}G} zc>>unXbYN;pG zl`|lUDb1mK4dd!P{d3?dd&+zmW#M^K4eu&q-dXL?gr$E+G$M+$o$j+J#UDG)4(!CF zy$#wl_#|EFpcK&r2fb)cW7=xMdFXZ~FrtuobOQR)LAkO4wWw{1gnO!H<;I7Fv{n&f z+Etez0t*a8-lIWQeG9(}>049Tl6mN)`5^!Q&Yq}2DWkkkz?i`hMdza|`$TGyB2C6< zo;M~ZE+%?>Ak6{V6+Ir%atX5)ibr5G{6rRR`2hGJ!#-gi6h9`$-5T&Rg%W9{YpWB$ z^)gbSSc(dw8Iz6-tK;T~6L76i`c31SJ`k;NtJ99~N0|LD(6WM$0?N{7JvOJ=rXVd2 z%`){^2b<6S7Bh*c$&EF1>EZ)rcr?4p3g!js_l!>ShJJ|&6F93`-!`3IQe$6X}Trfyhq z6-ZoV7=eJnYY*S+;}0_xMNDn_&te2c=s*oxH3#H13Ip0Y`Tuv3#`M7CXB%r83j27! zNKje-j0!)@%#qcKbCyaEq+7?={K=PLuM~PuxP*P?7I`ty<}S)6$oWK|O-f69sPRmL z5N0qBW40-llZ%7!oFC>3bvo%?IKXtGtfNSs45pJDovZ-iC|)PZdKzTs6Yx4wmW~+e z1legKOKPx^Jee=!q$678g`8#@EOs`vPK30@Y|+v9c5?BPNoziPZ?*cYpV634AqG2* zGbSf;jpEnshW8zysQmK?i~~?VLv`=YP#`;rk<&b3NER(F32{+F60~Sg%}i69oxu|F z|L+0fpJq=(VGSmXVoYzLCJVWc#6c&YJ`1*{qt1@0LC=^A0oqJs`_&T4qCE&URXa3& zPqFWw5Vfjzlx5ZMd+NQb&>#3MF;_Ce5RhIf!U&I(guRDyGm&vN6 ztdiit+M6N2kY?3I06J}TrcRe=!=wYG)rG)dI?ougC7^cC<_P4*F({n-YOB-vw7Y;m zXMDCAj}G@4W$8IZ7`V1Mm)g2*aywt-_&gXV*ad2PPM+@v=FN)y_ypK8qY<-ZXcqet zV-u5nBrEE4$e!t9T3elI;xuJ06ozUE4*1PIAp@C04;yrbH%Fx20q zGKeF=L1xQ9P#m^~RoHLN6!^(emYW_rLj{bzSW&ahJBpNk2N8Wo>tGbitvItF(^6s* zMdLzcX+{E*5Y1dmCg_ii_x5&Q_+3pRNgz?ydupp?yxr;&hODw_wilxzL!q@WTGp1soN z0N(!Q{d9%0Uhi}Gl{y^E$@rNr*q6H%Y{TQ|FThe6l*HU=IAo!h$Qx@e@94A}w27|p z0S;^$?z>JJ=nvp9k-cxVB$&`^t~l}|K>q&&%LI=G7|vSLdBFUITptNv9&n-VIT4a5 ziz7WSe}!d@Lp}j`eoeRhW1g-s^KLOuqd&xW*6x>op}lCXH_%kt^FXUiJ;Ia-yJqDU zs%qaJ_54CMK^5r)vAO4jiE`{g$a$f%ainMdE$@&76rR2QHC_3 zGA*vwZXdryno!nz>)wh*nz-;-q>1-iciiv$jZs#}%N~N~9`gU!C4BMx_y;CZyF(9_ z-QoSi%<$qlmdS_X?5aEVZED=K=eXK1W?7KfGD5GLWlN$QF;{P8VOgg7$ZDVO3u{}&_?-T^zkuw*;CCUW4^rzVfi!>3&g{$cQJPI6NoEB_Ji$5 zPBEk`MXIMkn_(zhbHmyRsukB64#qqQ>(>&trrdzL5Y%~77%E#Md28of&xu7W1bLw} z%BO?#pF3S{ou?wX%bR$)@T?jfc-2zTk-M$)rr5g8M*S77zKxsl6=fpjd0uzN5~*bs>T2-@^m2~Qb?Y`&>?ZJ%lSS4cdRKi`Z z$Zk1N@dkqK%UI4`Ny)yjU-%l|Mf?h&@0@bGe*SfS3U>kJOhp)&x)^k6#3*SN60^;b z!OqBRsx=An|3?$B45fMc+Cdg7lrGuAE>B5wdPo>TM}+}AKq=UQ7QJ9xI+TJfXvnoA z^&R@dx+5jRvKOL&%#e>~e|}kS5Tj++fJkWr+A}FOx`kYV8s&3|qB`Adzj@CE(Lw)d z#6zynOd8{l*uW-o-(i8dI+xxH&2vZ>xux;BeU8HjP2jJK+W(Xn)+$r)g+6AQ&$ZN4 zJXsfIJ4BknDa&?f+@YkPKxJ3XC|BAU7Z>cJBX8&DXXC+Gach9qXR0zAZQ&3<({fy% z;Ujiln)4BrYWetHpxyY-Q52Fhzo(3`p}BlJljUJvOeMXw?ld{PX^hjwyJ^;Qjt306 zulSeI(QMUwxbC#0D54+D<%810WZJzA$`23e%g$ht@66?GWc6@W9&PKfjYo_$UA1N~ z^U%B8Oz-4mWwmaKYPOs7kC8j}^GUqiQ{=|Hobe=6p`O1N2o48oyxy5M5)ZyB@%k!u zD{eMHF%eoNGQoS7_>hXuvr_(c)Tqs_GALr(2}zS|w(IL|e3Nl@bhAm^k8aS#mkkpq z&F=I{17?hgo}1EfwD7*;fE^qyvj16DQj8eAd~nccyrAD688Q^XM@`qB#aObly&v}0 zy=1pG8u^CH^@DHLt5)x#`J8g=nGUB9?Xs)t+YYM3T(#8-p@?R#Uas9vvzwZ{@u$6> z@Qplic~aMP<;pMDOINX8W8F%r*Yiky*-s6oGjDx8jD({7a{6+)zD54s&qpiC8}meR zUH?KI$8wY6ObPUpy`oylg`>kps#^=G-ch*Wjm+|a#jr?$6sAVUR7(Q9qs%pR@{VT? zq!gMotwuiXG$m~(21Tt_<#vuD4vG9Lh_9EoC&?Uw=k_X7s+a{=Bm~kY0scg z?%w*Qk}Ks)hR4%j_95smB_@@Tj7?*`PG(Rl?zl3d@)oR?a!}(0>A&uKSc>_|H_&tGPWnM zJ91Im>p$2`y^UTI_ zHGzE;yr)=A;rgsOb`K{=0Ss6>irzaDwhHnBpBUmUaCsNx4jkLiP})0421_OnmUPA? z0J9^0Mb2hFR?D?YH4zuw(mNq4-y|v@arn(GokrdXl-BfPE>Wq-mkD~?L#P{qQ2m%p z(~GHeZXjh3d1=U*9%(67)CtTxZTT26-e9K&*Yp>FlBq?=M~4Rrn$M)E6G0jVgu z!7mxbYniLjak^9{blerPQ^V*&7y=>|aFA>Pb6+y`Sr=>chOT(EN5O~xA%6c8WQDHT6(A|dm<`HLekNR^(~B>JB#s>+T#VjCr(qdr24Sye0w zWJXbw)E9lK6Iu$AD1(KxR-Q)!tf1^4aFhvuE z<{FHTNf)ysXaG(lTYzzFN0|>nbfZq<@9`AQNRXXlPeXL*q?>>vw8Y!Uy?%&j?-2aW zNQ|&dE)!EU9JE6BXjwf_mCmV9=%QCkb&h_nk>`B``5;;$|Lt-{*=iZ}(m9!DjIZRX zMU;WVk{uu&gKIz(?g7B64yKW|gr8Feb&3XMWYijdiX|c%PPyCPzF~dPTEU+pvlMxK z(FXLHJQZ~8kU-dPL6^wO(saetknCq>}o_8jlj}{u>GIj%kK5GIr9jC_l-BY+nVDL}LgYx!G0VQ^j=Zn&xbEY7kp2 zzfB@BhTs;a^T0BNvkyBGFCK$suCtHDQHR$s>+Hr_Q zxmf(8$ocm`%X^hJHMz4Ydr>&}2j?(uZ%~oJ4s;g`WsT7|;My zzt(XiVA*o&=ge?476PktQVR!8v=3%p4O9Yy&)|Q;uYZDJg`3 zs$g=V!YRt8WYQQc0RY;{ie}UYCupV$`A`u5h}u<5)PyZ>RnW8$JUV;gRd9uVqZha_!lK?FJ?-dV?bvTp zx;}Ga(8n3_#ta(xN`)q-FGj}&h=4OmtVK!}FG1N)Xi9G_2P1OnG0Jzb=7m-Gj&?=| z_`n`#!@*{B%vS<@)x=}qM<{hhL(Sr&YE9L^QYj2j5)B#VJ2on)mPCY!Dx(3_OG8IQ zAfrhEolvVTICu{-z$j70J-vf$MNTBZ7MMvCC(p&(H%I&@aU3@^HY9mOyXiT}V(W^4 z_kCDun^&t(b-B=nG~2Nwo0=3jtzf7vXJ=W;6XiSgctgV1N$+Xc`(BW}r=G}rA?iCt z47pIE|1gKdp;hpb^R|oObHNVb>>@P~bT`5~et?2qcJMhcE%+04e!rZ)hYVoOX2!98 zumo^go#vK|R7KGai$MPWxOYKg)*}grE>pLVC5&7XuR`O%d>E3n(nu?t8G@+Eyn(?; z$_GQpO~WaLQp3~-mzdR)UCsJG{a$h&8)-!a@T98SGGl0i5*o#l^`5 zWFOO+)heRc3F5lFHHSvc=dV1B?qFOiroF}HfrBd$D;F3F=oQNgn1Ey)?XNB#CEv-# zA?uDv<+KPP+P!iOBFrKG|M_c|A$%qEyj6%c%+|FCECbszM0@1WV#FA4z+vEk#k8Yb z(2|7zRQM|8$O1)wlKy9l6?pQGtW%7~^#uz&REO6p`qRdm1-yUIEu%kdEL()fS1iJz z#fo5Er{IVNkp+|jiD*iNu5(_wJj$Mjml!7(&sp{e&s_Bo^Pd_+4bgU3%~*xlYNb4P z0b}x{wu;f515i^;i)dClQ>)Kd*}w~7E^UOLj5;(s8|$0q7~7z<#`%fLZZPxdc`F^* zj~77dx+C>cIYf)k;Tr!9vH%9tRgZyG%kWnp0e_R zzw5ONA8voXcHP4kUQWv%gC4Vsz15F?@6i4}!{6Wj z{QsE(f3YFZ#^itL<{Oh$T7TegQ(*B0Za701*wbLg_+iYj)h>A_uua5P@Qlf?Z zP{jYz#rkXn|K6`)wQgOib06AXTmW06)0ULJI{)cUwRJfT*}=ddw{TaH$2X|vz^6eLGf6CT=)#}&W8eoefACU zua5@^?>>DXh|m+4BFUJ?BZDVj7({3``e1zUZetELoz@U6K6v*9d))4^_+Z4DMq@GA z6Yz#l6H+LVK~BjLej^N)oG+Z*((;9c@pvK=ODhg+g{oh~CxFjQ2t;@tJD8XUH+ zEEOHO+dA(?+ViAXj{lm62SfhNaL8oYEiVlv2+yjC431@}UPt=)YjL>SVj z)%nucQ>$G4utJ1eX`knEF>9|E_sxi>n%K<3xpE|%Y&C9ph_EL6S$Q}LG?dNiL0r}_ zdXNjHUukCUrBu?D4Yk+(%Rtdz zyua&i!)p}LDftj?noHk~lSxlyyYkOg3GXm|U+&$6YPV@mFfMf#1K(9(TI)XC&BA%l zL@CDQz*1UtdhH$+QEaHqTd~l-^iQ&@^1c85uHW+!I@Av#L>Th_nP%1@uKQq^@UugN z4-XkA!beTlpyIG!DCTlvey&E6sY36*<*KG;m-)fAc|C}EZ-dc(s&-lM2I^s7v{VbX zDZaNktLL|JMd`<{qk~LoGfn0D*>Nf`m%aXQJmf0eE2%=awzykd)#g1xwFk#shlsCGPyP&d=L!)Y!p7C4MS zwCtVfAtXQWFyHi;VtHt9+Sn}6cQdi~%n#dOi}+K7{(?bHEv)i$XJ$JX{~HPTj;#O9 zyFC8)k3Rd+ACV^b?ayx#1%Cf;_=1-7fB8@SX2u#@*B{S~HD(%&qL5!2QEZJ47E}bz z@rdJJs{mo}#(%-(|IdE*qd)tb#QDGGefc~3CnUIkSB?vATxqIC1PFO_m~)OtI>SZB zLYA?`WX?%?5Hw=v5pTqz7_l$YLvRiPJ6K?n5h%@rjt_57GUp!RBY2z#3pYhX8jlI3 z5H$*=VqvW`44*y^V+=w^$()nzYgb0;>)D9DYXj#!{IT#g76 zLTQ68g1oiBU4iZ41D0?(Zc+3xMRc+ti@jE0|6Q+Aiv_f8-@ViKN{~AT0rN!9Lp$%H z)*|s|%ky@m-SWeFVf3Hst+Vb6KBr*bx!rOp<_Qt~hHb%4Kjr(^PleglfwX42nVEKJboI1W79*u z3q=ok|7`6)=KZs`=vlrwgDaon{o4+@iO6PEepU@f{ChwlIC8gjo-9ON^Q8es@aqc> z0QvuK5s3($y$cXvAG*WdKMT8jf5HGBAaDSyKg_sv2pr&L?!QnZ6omUv_Px-rZxf9J zqD{v*t)4&cJ~)82r^h^hw(4H$QnHHL56klx9V-6nBtG>t%0sCodajkoGBE6@-OC~K z{ONLkmrC4>EBWRooSu-+?b^|0)pXM+Ci<{_BBojwf5iD{xL zR}*r!)@zpY(pK~)s)2qv7b*zpho5cP$^1J>CQ+($yVe?*nQ+8m|vZ zgJAS#Fusi`0eRDI`PMC8dhVaKs`%0iDIL7sP6#yk zhv50+`i(4QLUj0ap1;FG28!@e)3xViZ&37gJI#C`)n3W-^lb=%<(}fwQya!|tJJ*K zj}(fFrYJr%%8g|^lp>)-&T=x7yozP?v|;boy#uOvt7nX=wy3YW!g=O!liGlSMT zav8ZAhS7||b^A8&S?_xO_Re7&>hXZnFW{G=VSodKk+y&becQkRY*MT%jX!y(pRHMT zr(f=NTaC{Asm!%Eak&inlj^K~Rc%#64c|KGjc2^R+h{H^i5BC5spj+x`+O1R^g9Fq zpriw(>H60S0HAvO#BF8C9p^%VwxEoY4_LXd6IpDbG=n=f#{(;N&j0%)uM|18u+Go- z`0lX(FQ`ZOI7J6VjJtgO{|BG_=nqH}{PyS9jsm~`?n~z6tNjK#`S$hC4+T)b>i+w+ z^!-uH4|x5*X7c~pKlK^?{WbjgJ1_n;asS`F)uYzvTq+%mC-0N4X4g2Hi&H&xqz#TM z!~>ACOgjlH%(w8UjG4nIoht(VLC#l&e4&~+6JnNwrdnAzjDUlw0?zu8PYy>!$pOSp za-$G-$aZeJ;fjl`kOeTcjJ;^f`Q^jTHuJ151Kncyhi;G8jh%go%K~B&5C=s)p>&o- z1mYF>T!K@QIR(c60gK3eMA+k_!+6|Xb|(DP{v&y$@Y^?eoD)YkQ{{A}n$9PLN)ebB za4*t1I36b~q~m8oyZ=#&uQOH8?PV^S-llGEGK=Oi7lWZ)og&S;Adks!cP7*-ap6w` z+m2uwH7}*}7{E&i$6Hy!Ia(yE(!F#z5UP1Dxg!vRB5VFheV2_LwmlQ*l$}W|9x@I9 zcc6$gF&gwo7K-sCD7~7}S!I;W&a9aoLvSOG*2A|r!4Y7Jb6jD9rUM;9vkMO!Kdwj+ z5qOo@g+GV^r!RV|+!w~j3XXvNt;DLrng~ zi}Kk+0zKfq^M)g#7@=SKDUyIZ)*lXfIWOMhNaN8~!(5O5Nn?8klP++ak3g{ z_5J$#@0@*i_F*#?r&e582JGocLjM0pgI~TsY0#W?)|g)7(josHN2_{n_#UJ>3hqZ* zsMqu+$TiXd*;_MSB=T! zdhFX27g91hsWBg(F8fh=*;g0y>wcipYGLpcC7*BST;aY;R+LpPd6V1v zGu@Tq6B}}UDRsx!E;-yz4wAWIdp%mlv(;&MI&GtfTBKItdnEICKxp!T^f~D}ev@MJ4NGK^U?vsIXu^d`#L+$8w zWtqEE%6D_W67P;v#fRus@qX+rs!MXan&t(xiY~-Y3EuMnuz_KGhT{Hs6zGf*xbYql zWLAW45!%?T2Mm`=8~iFZ>|r#Mg-fN)OrpN8|s3 z`d#k-yZG!!7o-V(`|~^s{C(n&$^GzGzdho7vrf#6dH>{KJpGbq561J5IxyB`&ke2n z^e~=Yz<6@dP9a%NByw^(OK_c+iTKBIqL{s@762^ji3CV3IYE6s3gs#M$>Vsk=xacM zF1qe8ut5ovQU6Bj<)sjx2iLV!C^+}O$M63{s87Il1IpnTZZez|B=v17{tfw8$RC08 ziC8O>BLz+o3jESFKS%X3ps+YN*%a_K~ zy#(P3aWpxG<%Zk^?LNxUaS*k7Y2O4f*aW$y5Ng{Tz*6w}F(=+ng*>J$4FE=W$nXYg z2%(mV?CqPMnjMYl^H>G*81S)#KmNVHhZP2RxiO)&;sk6#MgCg~siQaJ1Hf@J-x4~u zAfO~rWx=pLe@gz1fOY!Q5Wcu~3a+>3G}45@dh`>O4-5=WA$XI%RpAbW*OMTe7#34& z0KiK=&U^$2CQKjfUwo~H!XYJ3&yF2R`$$MLFM1;)h7rKM9PtzpwB+)dyD{&(8HL?e zQ$d8~M06yULe)J200Vk0lE+}o$VuBI0|6iyL`N1)rx$!GC*i%wmbWL&UZOjz?~_Q$YKoJ~UHv$YJ~$Y}Fin#o$fm1WZdXjd&$VzPA25;Znh2 z=E*w&;KYcU#D(~hn2+iyF9F;}_rS($qyWYLfL3G8=OCYm6NCqEK<-1h2szRy{ zkALsap=k^dl)*zaHJ2=tAy5To)0jlcHYS5fd$eS@ZD6!jlP6sPp#W)yB-Z$V$WEYF za(7}!={A=mLbRL!k&XTZ{Ne-ZM9V2@Cv#A1qeY%cK^>r9bmjvGx*`AnvzD8kScf7# z2}BWKhAMT0s0LJOz<;5MB=ST!454+1-@f^lP(!~D-eEM>7d zi!uRHg>M<=AOlG=)rPJFHS?FS|G;Hkfb=d^V(^?Y0tM9xaRLBcf#gfFZ|W3)iRwz_ zDI}BvB1r=6C%g5k-DG(~W4TE%XV6wi$_;Jhp6UgQ-!Mv25H11cVt|cX!gxT)fdrih zOb@!#S`DUfWw-mxb3{J{0kK83qW6FJOPOz6;n;JOgKViayNw{ zCiWaG77#*CeJ%nd2AEypxt|IxHX=|noDRg_w0XnTJ+U@hv5>B!)eEcV+iNxl(KR-- z<`|)v#qzYJh^>cGE8Od3>b)CI=OYt?1JChG#CfviToIu4X2>tWBCLVow!(Y_?ZcH0 z;FL*yP@5QhqkRMbcgZ0rRkq1NPw52LDaX_i=F9=f|NjF0gmoH2G$LXV5-n-z>3F=H zXbGQ?DRkW@wSiGx_mLd~vo!6Iu*tVGonY)pM=(Qz#Z_~XqZ4a7Ed|meHw}~tgSBIz zm^4~CFU?uI(i~!3_?G4#bOL;268rFcBzciu4->I_+I2S}0c;|6^NE;dA4W(9$;k&D zD0IZ}L>n}Vr~^jUESNs*AP|=3BxYgoQdsMD=X3i&(&$B-?I95|j(1s%I%+1=1*!t( z3UHzgF@p95*eIro)>K2xWDr7_Nx_4B@9A4sq1;*FRY(j-@(3{kA}HaJOzqHQ@4BfR zP5#qBY$Que=ut%&aXQV0F~&Y6NP&GP21c%Ja4D6U8nIInOh~w* zkbIhVH3sSg5dZ8B;7!a9GEL=aoEiR)<-)D!4eW@jj3_ghV{!grF~Ib3 zK@+rLby!Uu&duBk(3RIwx7P=emx!WS8cx(f2_J>;n4N|#>%eRyN~gYnaFQY=PYL#5 zA1sxVcY;iZVt{`NF@Uv=!!YlKUqUV7T|KiUy)AgyWnlO2T=!9`R(fMJu-r1=XyU4# z)7{^aB3^*g;^{AWs!H5!K_Gi7vDBIHrQaV2gp~J~(mO0gnBZY;Xa5l6sW3;~z}+4! zfqm)MYW`A7T1AC9JYlP#8|p^7=%P^p>ztq3OP^TV5R(s8&umeesJ^!3SoVi0N3!r8 zxd1HH+Dc<(;I`32!3SJ+T%2EigwTgTmKFVMAp?4{J1e(fkD-}z8?r&GxP|HjN7q;~zzBtD4ANJ6Pr86WAXS6Pqs+9dW3HXyO> zNbeErU2D34Q`o46syGF?ck5tXC9YdefV@t8k%j;5jZx~n&Xf0lL38o+-@5<~Dn>>Nbn)KtM;R|NO8=?v(8Z~Eg%nkad_lDx(^;&Yyhp9p;1M6l)JQ;=T)o2lL z5;?Su-l8|G1EmQCD-gG=qjKt1I9m~b^%lw+sukd*Tz#`-y10PA8$h|7pYVQDerw|r z7weM?ASKchxrlC}^!)qbW>P${Rr|BPyR>UsDfYtiqnAWG~95jmR_x zfzOR)v=$X{hZl3Pk{?AMe58DV!D}Z7%g(~rQG^4^>Vc)sd;K&wEyU&3URuJj-gDAg z*y8oRT1#OcmLrnT(SU@KgN%QPHH7X6)EtZ;HzR}99y?+3u*)cHWcC?@B<+KUT6XKw z{VBRdr#S@B1oEXYSBl{-Ik_-KFZLXN@xjX3+I8A(Y71As;yk%H=t4MH5UukD6TaJe za=})Q#_p^2C6QAblZgmQK>q&^V<<4MQR<-clkbK4eJ|<*?Md#n1vW&+7;;^y9$x7` z)AM|C@dV$uA&f*K=JAK9y~hy=^{de`&O&B)V_QWI^8bG_TjsFX zz0Bi$a`9AWx#laPsqQgfxw-#lCo(XUU+vm9ypb>WY#V;ompZmBtPLG$Wa4^_f19}{ zXg71;y=|$@$`XtnoSfSy7dyUi`yd#c{`@J;s zW&X1dVha{O-q2pWxTFlHv=g{&Bm0-ZH?)}wC?K+WdElWfn(*Y}zzt#%qj;lryeX3j z<#lC?N`1kw*BYaK-Sl1Hd35R&)3vdU(;Ztq56H-PhBau9xfL#|{CU=nIl>l@HWVr5 z&QIWIJGl^kSAdHISE=**x=OE!JQwg=4r%}I|As@CRY_I?c4>{O-QBUcTlvVSJWH>4 z(vug~PPtlJSmCmobrL0WyY+RA`jXlUFQ{)eTFy_A&8#<4SamMFz7x}@LzL|qm0hxj zUA7U{8Oa!F=0IYO%aaR!YKv{7v#m51w)&aLa@K*+Eko8eRvwnT18gkU z!o_awvNTU-ZpzCBqSa#swO688(vyoCIZXL;j;+*NE9yQmmavYkz@D^A1)!@Kd?pYm z_tA}+G>`sHYGTaq`qN_dlt;Wrr#MY3~xd|XRxYUL8W8dF2P{ol85 z_OAA0>(Dla0O(jKT#L)$p_mNgz(QCOGm*=9Hj;{$A`P(~na1l8zu0tL#hb27yyt3) zqqlEds(9}T$MGw!#C&`$=0%UF#3fM?gSaA!{^F&wA==5APV%ZyTg#<%JX#t;{{Np$ zT=o=W4=eeeC=``Fqr}@crFzM~_1CLjZzi6Wx5>ev-=Bvn%lk;Zx$TN#;HsQ0$5&aU zp4=wx27z3y)hmr|GNoj_(k?a&!@$E;ELJZT0{1{unI{T?W%GLKFGyX6c4*8euFLkiKJe7Y9+G@&|zwz{|p4IktunPCa z_xVb%TJGM}eg0*qaTl*$sopEsxLwW9aOOx{dKQtXE9+C+p~6EvFl^5Ytwv}b_AjfA zkS`qW2Up_8l^VqS-gLQjxuG$UqgUlxAfXPgdc7XW_Y=30pCzQa>i*3VQmNf)+~rB| zpDUzEQ%LDJ%Kq=$H+xs3lBiafrDgKIzxFF3>8cg?HXAu#s9F!?ik;x1I1ah8%SbWh zbCKAEb2wSeMi^&_Y=UPOI(a!9CAi3WtJj+rM6Q=qJ-6jxb}{pIBNbO5d3hOdb+Udj zm#CMZ+!FKK@=X+SC2#Jt*~D;sxi05N{)||-tXyw?x(I= zHZ5mwl!7vt#%k;3GI&2$o9gsBJCP%?(e+g~G?>J$<-5Rm?2W#C6CM_%Xrto~j%K~# zgGV=f98WyXP3p0i&O)|y#4UH|Eg8+%3&q6!Guu7Q1u-!Bv44XvMQ|bk4;2DTjv#%(< zO|(|HTc!J@bzi=j;k4f%F{)j)8$C~Mk@8eBSH5Yrqb6=2+TmqwQ7>fN+=T?yVT zU5Tyley}Qzr(#FC9*(xz+B&|D0OS9vIGi=tVay8BJl43%j%%G#x$O_chf^^%OXN$k z+7?T<%RzLIFDLRLpLbO%1>~z>J$Bv0geKOnR@G+DlUd5mQ9jj6&cxgXkW43m%lx?Js&8%z zok(jrzEWJv`k)Z(kGiqNRLlhCMS#t9TcxcyX?u&SX`t(ai5Z5Ji0kEeiXdw2 zXc}gkZlu+2mk^S*TAd^l?M7};T7^U$Bo28}wOpz*X$;G&`Hgp$@ulTXjPhj2=q$`cSF*9{R4z-5{!pa@Ow}gc zG4fkvLZjQ1T=t{~>AVcHODXP=AG_-01|vBU%id?IrQBMU$LW43 zdS4jAn6BKeqv^Q3s8o6n<;FT)4bMu!TJ(A~^IXQVfoT6`xhN|DYPh|v`de}&Qkxgr z5%10Dwpp6bvR8Re!>>HV3mFeelU&KlO}`t6b++NF(5O?++)ljnzEsUenzL}=(sfE=sMGhY}A1|>`XiJ-1RmWs1^`#-@T5`ukxM1xbMBoD%nn< zk-3X)|G)OGtT~P&Ig?V@D{C#^_S)+sT41b5F3~g7-E&|D09O2HRI3ElYSeYlpSu}dC& zOBn2Xaw_@cW8Kr>5?&>G!u8T6-Adp-%1+pe_>~A^6gZTClmGk5m8&0YJbdQuQf4)T zQG)pp@JW>sj!>U)2**?M_G!Vel|0BTnIK9aR1mF+(j|zW+Su6JxVN=&-wR5?vMVr? zoHd_pe7(81^Js7L(e5`7_8xtSG*leyS-~f}kGJ<8-`U+w!at3)YxH%t>cV2NoMEL} zYSc%38IW z$g<+X;aZRf5f0?M`hA>^?sM)STw4L=S0p2}Q2+bs|aQ zUUz@DcsBQ7Gu&=&Y;D(S+cLpXZWl2BKUAY&BA}RKgcX8gkI$SfI#;fyA+a5?RhkM4 zAuof`=a~nc_4^NIlcP6xh6({(f4l4Cwd~zE;eT%`AVlc(1cH2ja(L-xx8P>?ugDh}Rcx z-E}&^x_(Su!mmB--6(wL#&JwehW*L0cfEg7ZD}Yg<*<4kK9LA!je9o{5Je57_$CC% zjM2lQu~qQb6%{3k;~{$o#+)^7eQDvwFUjv0Ag+9~ix@R*Px~k9T9;~3O8SnZ{9HrS6@T8{`%nc>FGB+o1?*Y z|G~<}&fQnHzQUr1$#H+Q`sadm9I*RIO2w(z`PpLJjDWztZEokG;u^ z!{}&Q@;vY!A*&$M+Y=MRqt$lFLyVCj?{1^g3#F-9`T}0lcSqCFYrK=2uK^~RWb7dQ z3`o$e5&xxPxN7M#TWPjx^q+Sts8D*vqEtkD7U#}1L?fFFunf2P3uiE>oFO$y%{2|~CEXs5L5)i8NA?D=4ZBAiW(^-Xs)b24QGQl4NWi-P^P?AboDFI$Glr(&wlmDl&Qnde1p3-*q!z+J( z<%3^c`6d2&uRnKheqX@*=%bw-();{M-AAn+V#`VkWu}7&tXY9BSSfz`i~soH4Ql8= z{jx0f-8jKshz$#N!jRW0J}?OgOvE3#-XaOBzcuWRM7Z0l8Qe{JK0>mG94A9`r`b>5 zy*%yd#?w&`HK~vXg(IEE$P_GA__mN3a5saFFrE$}Z4dh=!aL-C{HB9*akvT~PJ%ag zh?|GFL2&&TUb`$}kf37x=41*9O>w$P%6#g5b`1%$!Uh8O__x~13Pdc9y*ct8Q3Dhr z4(o()FM6 zx;q;qe+I=BBL@d0BC5#D;+rM!2{uI(;7qXy!8FLYLc1>l91o$Y{?5PNzD^03{!Qk4ei9>8c<4WFPsyE!ySB*IhJ`6B zKyY3lO^QM~sJ@K8fk|%V<)>`0nn3xaCj%7!jG|sRtu(wF8*j!)pAeOhvdHhC%b|bJ zpZdzl!w;fb5M|OZ=Ij2HcC}FBsFuB*VV5!0n>CcC9}e5z?v!ur!kK9s zv#{HTwE_F?JNOpP-ZZfUzYH zZ*g`h@A0!Q9_(y*OMbchO|(%iZ$91hp5J@=@Bz#<6_1UJn9`#m(pQ(uTaT7J4tt}h zH_!p@W3QvqXnat9`lNhDEer^N{MF~!1^G3WE_kD@ed_z)_V>M=trxdB>e@Ry9tG5O)O5y6Z3(0f;*&WuL5O0XALk1o?A>0DOxOc$x@_oJe5)C&T!9JM( z-;1erl%$c;K^UMooD@$H4Td!8K3^s+WfFT+bTjE)cQT>JGERFmM0}5k-r{xgh88ed z4-vmAD-I-07m{YSnKlv! zN%MOWR9sks`2?eA>ct6$dT0)c1r)bjJT8Hw!v(zwooEnu{(jL!PBe&ppMxfJbhw~5 zRVNw;6KO1f`Tr56GB_uyniCa8bQg~gYNKR%I@ z4O}XPjpkf(EpoC!d<>v6DGa?VlgOxSFj`(1dde^QgEj?@!l=?eLqal!@MAG?4JExl#16eb6)${y z=0oQn#6GeGw|y9iU?UxkCof=c{g$#p{DatD43^A0ljF&f6bfh(mBW%yZHZ0$E`aTh z#@H<3kORI9I7&366FORwI}fiUBFpynIAfvTnIVx8(wejmDGwCdH21~*5fVv=PRoW} zL)It!WN~(;24r!`-xH#F4as?)e9xv4U%f@)UP#80_QU;HYyIe?jdUh&z#1hoEE19W zu~(v2zM~;mIiV^@{*yjNpJ&HI8obf#_6rYZzVezsJ^(qHjPj|t7Vvug2~GllPw-L~ zryPLmsZ3YdVe`C9?y1p<)|8WREYkg?t|^UXO@X90 z7!zzyXf7G=(pS=Z29}VrnPu69sBd>Dxn>+<{-yAfYkms8M`qZ8HYP}1fA#xfoUDgN zd+L?k_V7#>mVpwoO#A$sn!`jHuL=hY=GjPQRrq<4crG%llGxk%=m&SUM5q z)Kl-A?WJT^tYP2@oB4QD$BQu~Z@F-xhZZk*C7W(~3mr5JRx)aU;sX6klb-eyqt8(@ zvlJ_cif-xm8t*Csro*OQ*SlU47hFBuW=3OE&TJ1FUD1XV!vwI0#nX#jjW ztP#>#b}mWkNwSf&CaIhVi08E2rZUCzO8Qdnc}dT##oY6@MdC`=a-O%Kb4E^}&{uC< zW>@J%DRHdR3P`#v1BAT*CjcBvumX@Qx%fU=VDYdLj2C1d1R%@X_xX%=Zg^6~<0 zu&HYFD_O{_*JTVjt2M{wRY8uo7J5z{aM~EY!3i zV992o@(Fg4V)qW8hTcg(53>Fy-b>CHOZ%Nc|48c3Sh~|?kn>NpyxDlb#cy!H-H(i; zeVNoX?vzDk?Q#&o=PUL*Xy0HKvrCX4sidz0&*$M8rD(Zns|??8QO=b*GRUn20uyQ1weR zOJXc&7RF6ZIX&rkM5ku*5ZhR&Ne(C(qO?9u!%S31OnUNMqN9q6n^koELac#`9NJ-e zJV2HlW(|K7BeGN^_E8$D*UL}@? zZL&D}LctER1{IGAgwq>Rvv$0K<_k6VbyN&Km=E6ZN(vshi&`JJj=s&-J6=h_19uVl z`Hj!Mkphou)gs_Q&0*%Z+MazS1s~Q5+a6RL$H5l(>?Mw`ex zoaxI1i5jZo{%cfbfpOe?a)O-rQ~|<)?rsb(tbQAHBKQSki)x}o0?3gGc0B4}{{Int z&mDlDwvij1;3oAaiuyw>#ET9%OHapf2l>{p;yv6&S&IeWQD!&G{*G+>vLj(`aRGn}+!(SeK|iB=xt>>}4I$AaE;KzDej*wEc7A5_QW ze1h8^R2<-4!pMg2=pfi63*Y5_br}a7yZtW5E2{WlGFg`T4iY-sk17m8ZO{fcaGaf4 z^vU&uvD;sD!Oz$XGwoj(e$55nAyMu2JG$tkPhG&r{QqNP0#HqlIb+~*i*Vv-xccB& zbnHGjn&5=L;)3sB@C(96-PtS;xm7JgLbx0)Tte7aI9Mn-{Sl{l&92B8B{ggVUp>Qc< zW-l^rFnbZ2b3}&xV4{f3Ay*ojMl6?pW}=ugiMnzkoyRR|Yb1bhNjZf*bblf@AZnL%bwW1GhQ}SY$;n z>Lis3Rc+$mbr8zDPu6|F(tXIkE79ADN4L6S2kv*0S&;TcGv>*mA+iD1Z0aA9|Nmd8 zv&whkSdYuu(xyX}i0yW}tMZ8~=W0G=sDP)Op#YU2QvF|f2&w?*e-5v5;WtdGH(<$2 z2m7Nn-z{wRe^&x5WzgXHqFpeVNxHUd{Ysac+8PH~Q`kzrQ(I`3e}^>r5~!hIa)xSK z@^q+{(nt^hokO3!Lz=}2#7Ki^#|r8Y4Gj;lsGVXiFUx;3vtLS&V3~-XN|WST5D2i& zK_fvRI#4?d1>8h9;hJCp)rG0Va3cX1u#~tvl(#NKD4g4~V_ zaRCcH3nX2&r6;LSESE(9eZdj`YaEOt>h@s$z#p@FdwR<7oh2ks zL2YDOkb^_rYqq?xb#~K#F4MODIgovHwd+@W0A1fG!$%67Nq9lQ{AvPXw+>tyP4F-vJ7E z9|KF;7)F;x@bP1{?Coo+aD@b-mvn7!+2K6}t zk9n-+iPG-rcmS4jNdC{pqfB$5%ySI+RnlZKDVwXBZjO=GxWJqWLKTH?rHByHytcJ5 zvV|4;-iFi@PGIaGzN)c5`)wryAg)sOw}cgm79IhLQrt(fF3zOVLPoK;edND}a;DWzdAyzXFK}kVbT$CKP|I)(I8#k!tO%y zHWjv{9=aqS=Ko)1rW%AY;JpXNtT)IzAF#}YRGUR|j?p@wEKXvYDE~lO1LPg$52)S3 zY6b9cY=)gkUB`DnN`3`KKvC zO$w+%t0qW!4aGp^n+uE<#r<-xeCLZYbX3492SK$=$^R;K*vIx!0%AZ}_x41XQoyA` zLEb6BTN3Lz{R;5}7Qy`gAb>eW8iU{Ywz8ounfx!MSH;1+!b_*Zl0- zN*FX&DKFt6n?)|1Xw%NJ334u&Xp_p)<~5!r2;~eGZp?;YS-PE@o^x$79gl_ww;gFZ z0jn@mC#UHoLnj093iEXW)?z6;7s%HMSSn+vD`jVzSyHBz6vcd*lM1BD1vxEMSWGt6 z7eYE>)?_rUmTrGG>`YEpb*)eT9gTS()4*z6vN(}x6n}jm)^{XD6~7O_I7sT3h;Dm@ z0@-OAxz>qfX3^%7GA_zYf|>-CYTCHn>(4O%{~9U&Y4uD5R)5G8W4Q%P8o5x!L6J|b zf}PQzXQ>+WPP!4G!!$mx4gibJAb6>EG(98O?Fdm_yaSdl;WOfWKc+vhTVkzb!cuyW z34D-IorGFc!JqL#r+uTcOsgRNPydy|n=WM(c~rYWBW)W~(f$z}q)hN2h>wd7Hzp)!>jW#^*G_Ufir;43E=ZfUPnp ziM2B^$U-iWH8$PeQM4PRi4yoo4xB3Pwvh(%19_N8-`6z>p7hcbM|K3v|G(81cr+j} z)~3b-wigO{EI-fEwZhgdwlr27%(JFl{(|~q zy5B%k>C6MIO63R(5x%E$3!%#DgM#+PVDO8)U%IU&J74RJ@s^S9T-@5V6Q0`=MZz<+ zW^vQKE8}exofv9n5&1s;ArW%>A{o^|s`0rpsxRJ=?3?x-rPyS}s>A-q*(A)#gvOim zDLD6h5-9;R0c+2wWHter^s6=X>FY{)#rrRNLE>pSR0|st+Iec@{$(X zORutkMJEBeKy&r>{aN-6Ourucy)i=0g%;xhLhwSHY{TELaX!^1c{K&L%ZmVcR0>#p zUUvzm5>|O|6kws4$`yT(Xa+D#*tQ2+#GzDcYh-~ne(|MLEJV5%!*kGE)9e*2Vf6%_ zMHXN5=?xowGbRC2T=I9R#uio9e*&z*QCb4Ze+hH1RuF_X-*(GwA;t8x1lO-wlqDmn zZ+g;W2tKReOUJfHZaPG-*80Z>smN2}sO25@d#FRWdNM+36;80qM`)|3{Yk&mA8-ay z@e1gvd84HUC$=={Pn5Bo9L=+ zf9gI67$pY*iSeR5Djl$Io@QJRz-W9-Ll4doJikMMB+Y&S>cS*Cbh1$ZDGQ(N$%z7G^GiXrSsqR%8t)SKl8*|VEjj#Ys z`ZO+O(1d;^@WQASge{Z-F78bV$mGBp_a=QB`#EUBR&}LS0Xr5FEvPq&dXqejm-Epy xTGhF|snm7Bb literal 0 HcmV?d00001 diff --git a/storage/shared.sqlite b/storage/shared.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..60b0d053d6888e58dfeaa081eb714239b65f4a39 GIT binary patch literal 4096 zcmeH}K~LK-6vyqj1663N;QzOjD$@i{Q2NqZs3fgrXR1cED_A>8$@( zbn>WIg{}o{W->7?o%VIwW_e`L%Ow;MwWqAqH#hNc&L^3S#rprA@JYrapFbOt5~F9y zR^)|zOH``YyLh$UXyBRtlzD+WQA&3yNEf2t1eu54XiS4($`eud?^vt(-o)fL5{EQN zKoVF+0_*r4(rUF+9-YN$e7Sd)zfC^-aj0qe{#VEyBKM+^CJ9Ia%SGU$LYmdm&)pg( ijUtjPzW)!1JS^9= { + const {Account} = db + Account.findAll().then((accounts) => { + accounts.forEach((account) => { + workerPool.addWorkerForAccount(account); + }); + }); +}); + +global.workerPool = workerPool; diff --git a/sync/package.json b/sync/package.json new file mode 100644 index 000000000..07c308f66 --- /dev/null +++ b/sync/package.json @@ -0,0 +1,17 @@ +{ + "name": "imap-experiment", + "version": "1.0.0", + "description": "", + "main": "app.js", + "dependencies": { + "bluebird": "^3.4.1", + "imap": "^0.8.17" + }, + "devDependencies": {}, + "scripts": { + "start": "node app.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC" +} diff --git a/sync/sync-worker-pool.js b/sync/sync-worker-pool.js new file mode 100644 index 000000000..8423d0ceb --- /dev/null +++ b/sync/sync-worker-pool.js @@ -0,0 +1,16 @@ +const SyncWorker = require('./sync-worker'); +const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) + +class SyncWorkerPool { + constructor() { + this._workers = {}; + } + + addWorkerForAccount(account) { + DatabaseConnectionFactory.forAccount(account.id).then((db) => { + this._workers[account.id] = new SyncWorker(account, db); + }); + } +} + +module.exports = SyncWorkerPool; diff --git a/sync/sync-worker.js b/sync/sync-worker.js new file mode 100644 index 000000000..429dfd928 --- /dev/null +++ b/sync/sync-worker.js @@ -0,0 +1,297 @@ +const {inspect} = require('util'); +const Promise = require('bluebird'); +const Imap = require('imap'); + +const State = { + Closed: 'closed', + Connecting: 'connecting', + Open: 'open', +} + +const Capabilities = { + Gmail: 'X-GM-EXT-1', + Quota: 'QUOTA', + UIDPlus: 'UIDPLUS', + Condstore: 'CONDSTORE', + Search: 'ESEARCH', + Sort: 'SORT', +} + +class SyncIMAPConnection { + constructor(settings) { + this._queue = []; + this._current = null; + this._state = State.Connecting; + this._capabilities = []; + + this._imap = Promise.promisifyAll(new Imap(settings)); + + this._imap.once('ready', () => { + this._state = State.Open; + for (const key of Object.keys(Capabilities)) { + const val = Capabilities[key]; + if (this._imap.serverSupports(val)) { + this._capabilities.push(val); + } + } + this.processNextOperation(); + }); + this._imap.once('error', (err) => { + console.log(err); + }); + this._imap.once('end', () => { + this._state = State.Closed; + console.log('Connection ended'); + }); + this._imap.connect(); + } + + queueOperation(op) { + this._queue.push(op); + if (this._state === State.Open && !this._current) { + this.processNextOperation(); + } + } + + processNextOperation() { + if (this._current) { return; } + + this._current = this._queue.shift(); + + if (this._current) { + console.log(`Starting task ${this._current.constructor.name}`) + + const result = this._current.run(this._imap); + if (result instanceof Promise === false) { + throw new Error(`processNextOperation: Expected ${this._current.constructor.name} to return promise.`); + } + result.catch((err) => { + this._current = null; + console.error(err); + }); + result.then(() => { + console.log(`Finished task ${this._current.constructor.name}`) + this._current = null; + this.processNextOperation(); + }); + } + } +} + +class SyncMailboxOperation { + constructor(db, {role} = {}) { + this._db = db; + this._category = null; + this._box = null; + } + + _fetch(imap, range) { + return new Promise((resolve, reject) => { + const f = imap.fetch(range, { + bodies: ['HEADER', 'TEXT'], + }); + f.on('message', (msg, uid) => this._receiveMessage(msg, uid)); + f.once('error', reject); + f.once('end', resolve); + }); + } + + _unlinkAllMessages() { + const {MessageUID} = this._db; + return MessageUID.destroy({ + where: { + categoryId: this._category.id, + }, + }) + } + + _receiveMessage(msg, uid) { + let attributes = null; + let body = null; + let headers = null; + + msg.on('attributes', (attrs) => { + attributes = attrs; + }); + msg.on('body', (stream, type) => { + const chunks = []; + stream.on('data', (chunk) => { + chunks.push(chunk); + }); + stream.once('end', () => { + const full = Buffer.concat(chunks).toString('utf8'); + if (type === 'TEXT') { + body = full; + } + if (type === 'HEADERS') { + headers = full; + } + }); + }); + msg.once('end', () => { + this._processMessage(attributes, headers, body, uid); + }); + } + + _processMessage(attributes, headers, body) { + console.log(attributes); + const {Message, MessageUID} = this._db; + + return Message.create({ + unread: attributes.flags.includes('\\Unseen'), + starred: attributes.flags.includes('\\Flagged'), + date: attributes.date, + body: body, + }).then((model) => { + return MessageUID.create({ + MessageId: model.id, + CategoryId: this._category.id, + uid: attributes.uid, + }); + }); + } + + // _flushProcessedMessages() { + // return sequelize.transaction((transaction) => { + // return Promise.props({ + // msgs: Message.bulkCreate(this._processedMessages, {transaction}) + // uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction}) + // }) + // }).then(() => { + // this._processedMessages = []; + // this._processedMessageUIDs = []; + // }); + // } + + run(imap) { + const {Category} = this._db; + + return Promise.props({ + box: imap.openBoxAsync('INBOX', true), + category: Category.find({name: 'INBOX'}), + }) + .then(({category, box}) => { + if (this.box.persistentUIDs === false) { + throw new Error("Mailbox does not support persistentUIDs.") + } + + this._category = category; + this._box = box; + + if (box.uidvalidity !== category.syncState.uidvalidity) { + return this._unlinkAllMessages(); + } + return Promise.resolve(); + }) + .then(() => { + const lastUIDNext = this._category.syncState.uidnext; + const currentUIDNext = this._box.uidnext + + if (lastUIDNext) { + if (lastUIDNext === currentUIDNext) { + return Promise.resolve(); + } + + // just request mail >= UIDNext + return this._fetch(imap, `${lastUIDNext}:*`); + } + return this._fetch(imap, `1:*`); + }); + } +} + +class RefreshMailboxesOperation { + constructor(db) { + this._db = db; + } + + _roleForMailbox(box) { + for (const attrib of (box.attribs || [])) { + const role = { + '\\Sent': 'sent', + '\\Drafts': 'drafts', + '\\Junk': 'junk', + '\\Flagged': 'flagged', + }[attrib]; + if (role) { + return role; + } + } + return null; + } + + _updateCategoriesWithBoxes(categories, boxes) { + const {Category} = this._db; + + const stack = []; + const created = []; + const next = []; + + Object.keys(boxes).forEach((name) => { + stack.push([name, boxes[name]]); + }); + + while (stack.length > 0) { + const [name, box] = stack.pop(); + if (box.children) { + Object.keys(box.children).forEach((subname) => { + stack.push([`${name}/${subname}`, box.children[subname]]); + }); + } + + let category = categories.find((cat) => cat.name === name); + if (!category) { + category = Category.build({ + name: name, + role: this._roleForMailbox(box), + }); + created.push(category); + } + next.push(category); + } + + // Todo: decide whether these are renames or deletes + const deleted = categories.filter(cat => !next.includes(cat)); + + return {next, created, deleted}; + } + + run(imap) { + return imap.getBoxesAsync().then((boxes) => { + const {Category, sequelize} = this._db; + + return sequelize.transaction((transaction) => { + return Category.findAll({transaction}).then((categories) => { + const {created, deleted} = this._updateCategoriesWithBoxes(categories, boxes); + + let promises = [Promise.resolve()] + promises = promises.concat(created.map(cat => cat.save({transaction}))) + promises = promises.concat(deleted.map(cat => cat.destroy({transaction}))) + return Promise.all(promises) + }); + }); + }); + } +} + +class SyncWorker { + constructor(account, db) { + this._db = db + this._conns = [] + + const main = new SyncIMAPConnection({ + user: 'inboxapptest1@fastmail.fm', + password: 'trar2e', + host: 'mail.messagingengine.com', + port: 993, + tls: true, + }) + main.queueOperation(new RefreshMailboxesOperation(db)); + main.queueOperation(new SyncMailboxOperation(db, { + role: 'inbox', + })); + this._conns.push(main); + } +} + +module.exports = SyncWorker; From 3dc03688e8fe931ecd8793a3602097d59fced7c8 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 20 Jun 2016 00:19:16 -0700 Subject: [PATCH 002/800] Cleaner sync state machine --- core/models/account/message_uid.js | 15 + sync/app.js | 1 + sync/imap/discover-messages-operation.js | 131 +++++++++ sync/imap/refresh-mailboxes-operation.js | 87 ++++++ sync/imap/scan-uids-operation.js | 98 +++++++ sync/sync-worker.js | 347 ++++++++--------------- 6 files changed, 443 insertions(+), 236 deletions(-) create mode 100644 sync/imap/discover-messages-operation.js create mode 100644 sync/imap/refresh-mailboxes-operation.js create mode 100644 sync/imap/scan-uids-operation.js diff --git a/core/models/account/message_uid.js b/core/models/account/message_uid.js index 632bef539..d9174061f 100644 --- a/core/models/account/message_uid.js +++ b/core/models/account/message_uid.js @@ -1,7 +1,22 @@ module.exports = (sequelize, Sequelize) => { const MessageUID = sequelize.define('MessageUID', { uid: Sequelize.STRING, + flags: { + type: Sequelize.STRING, + get: function get() { + return JSON.parse(this.getDataValue('flags')) + }, + set: function set(val) { + this.setDataValue('flags', JSON.stringify(val)); + }, + }, }, { + indexes: [ + { + unique: true, + fields: ['uid', 'MessageId', 'CategoryId'] + } + ], classMethods: { associate: ({Category, Message}) => { MessageUID.belongsTo(Category) diff --git a/sync/app.js b/sync/app.js index 8420d1e73..5481a9411 100644 --- a/sync/app.js +++ b/sync/app.js @@ -2,6 +2,7 @@ const path = require('path'); global.__base = path.join(__dirname, '..') global.config = require(`${__base}/core/config/${process.env.ENV || 'development'}.json`); +global.Promise = require('bluebird'); const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) const SyncWorkerPool = require('./sync-worker-pool'); diff --git a/sync/imap/discover-messages-operation.js b/sync/imap/discover-messages-operation.js new file mode 100644 index 000000000..008aa1345 --- /dev/null +++ b/sync/imap/discover-messages-operation.js @@ -0,0 +1,131 @@ +class SyncMailboxOperation { + constructor(category) { + this._category = category; + if (!this._category) { + throw new Error("SyncMailboxOperation requires a category") + } + } + + description() { + return `SyncMailboxOperation (${this._category.name})`; + } + + _fetch(imap, range) { + return new Promise((resolve, reject) => { + const f = imap.fetch(range, { + bodies: ['HEADER', 'TEXT'], + }); + f.on('message', (msg, uid) => this._receiveMessage(msg, uid)); + f.once('error', reject); + f.once('end', resolve); + }); + } + + _unlinkAllMessages() { + const {MessageUID} = this._db; + return MessageUID.destroy({ + where: { + CategoryId: this._category.id, + }, + }) + } + + _receiveMessage(msg, uid) { + let attributes = null; + let body = null; + let headers = null; + + msg.on('attributes', (attrs) => { + attributes = attrs; + }); + msg.on('body', (stream, info) => { + const chunks = []; + + stream.on('data', (chunk) => { + chunks.push(chunk); + }); + stream.once('end', () => { + const full = Buffer.concat(chunks).toString('utf8'); + if (info.which === 'HEADER') { + headers = full; + } + if (info.which === 'TEXT') { + body = full; + } + }); + }); + msg.once('end', () => { + this._processMessage(attributes, headers, body, uid); + }); + } + + _processMessage(attributes, headers, body) { + console.log(attributes); + const {Message, MessageUID} = this._db; + + return Message.create({ + unread: attributes.flags.includes('\\Unseen'), + starred: attributes.flags.includes('\\Flagged'), + date: attributes.date, + headers: headers, + body: body, + }).then((model) => { + return MessageUID.create({ + MessageId: model.id, + CategoryId: this._category.id, + flags: attributes.flags, + uid: attributes.uid, + }); + }); + } + + // _flushProcessedMessages() { + // return sequelize.transaction((transaction) => { + // return Promise.props({ + // msgs: Message.bulkCreate(this._processedMessages, {transaction}) + // uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction}) + // }) + // }).then(() => { + // this._processedMessages = []; + // this._processedMessageUIDs = []; + // }); + // } + + run(db, imap) { + this._db = db; + + return imap.openBoxAsync(this._category.name, true).then((box) => { + this._box = box; + + if (box.persistentUIDs === false) { + throw new Error("Mailbox does not support persistentUIDs.") + } + if (box.uidvalidity !== this._category.syncState.uidvalidity) { + return this._unlinkAllMessages(); + } + return Promise.resolve(); + }) + .then(() => { + const savedSyncState = this._category.syncState; + const currentSyncState = { + uidnext: this._box.uidnext, + uidvalidity: this._box.uidvalidity, + } + + let fetchRange = `1:*`; + if (savedSyncState.uidnext) { + if (savedSyncState.uidnext === currentSyncState.uidnext) { + return Promise.resolve(); + } + fetchRange = `${savedSyncState.uidnext}:*` + } + + return this._fetch(imap, fetchRange).then(() => { + this._category.syncState = currentSyncState; + return this._category.save(); + }); + }) + } +} + +module.exports = SyncMailboxOperation; diff --git a/sync/imap/refresh-mailboxes-operation.js b/sync/imap/refresh-mailboxes-operation.js new file mode 100644 index 000000000..de531bbab --- /dev/null +++ b/sync/imap/refresh-mailboxes-operation.js @@ -0,0 +1,87 @@ +class RefreshMailboxesOperation { + description() { + return `RefreshMailboxesOperation`; + } + + _roleForMailbox(boxName, box) { + for (const attrib of (box.attribs || [])) { + const role = { + '\\Sent': 'sent', + '\\Drafts': 'drafts', + '\\Junk': 'junk', + '\\Flagged': 'flagged', + }[attrib]; + if (role) { + return role; + } + } + if (boxName.toLowerCase().trim() === 'inbox') { + return 'inbox'; + } + return null; + } + + _updateCategoriesWithBoxes(categories, boxes) { + const {Category} = this._db; + + const stack = []; + const created = []; + const next = []; + + Object.keys(boxes).forEach((boxName) => { + stack.push([boxName, boxes[boxName]]); + }); + + while (stack.length > 0) { + const [boxName, box] = stack.pop(); + if (!box.attribs) { + // Some boxes seem to come back as partial objects. Not sure why, but + // I also can't access them via openMailbox. Possible node-imap i8n issue? + continue; + } + + if (box.children && box.attribs.includes('\\HasChildren')) { + Object.keys(box.children).forEach((subname) => { + stack.push([`${boxName}${box.delimiter}${subname}`, box.children[subname]]); + }); + } + + let category = categories.find((cat) => cat.name === boxName); + if (!category) { + category = Category.build({ + name: boxName, + role: this._roleForMailbox(boxName, box), + }); + created.push(category); + } + next.push(category); + } + + // Todo: decide whether these are renames or deletes + const deleted = categories.filter(cat => !next.includes(cat)); + + return {next, created, deleted}; + } + + run(db, imap) { + this._db = db; + this._imap = imap; + + return imap.getBoxesAsync().then((boxes) => { + const {Category, sequelize} = this._db; + + return sequelize.transaction((transaction) => { + return Category.findAll({transaction}).then((categories) => { + const {created, deleted} = this._updateCategoriesWithBoxes(categories, boxes); + + let promises = [Promise.resolve()] + promises = promises.concat(created.map(cat => cat.save({transaction}))) + promises = promises.concat(deleted.map(cat => cat.destroy({transaction}))) + return Promise.all(promises) + }); + }); + }); + } +} + +module.exports = RefreshMailboxesOperation; diff --git a/sync/imap/scan-uids-operation.js b/sync/imap/scan-uids-operation.js new file mode 100644 index 000000000..615a940db --- /dev/null +++ b/sync/imap/scan-uids-operation.js @@ -0,0 +1,98 @@ +class ScanUIDsOperation { + constructor(category) { + this._category = category; + } + + description() { + return `ScanUIDsOperation (${this._category.name})`; + } + + _fetchUIDAttributes(imap, range) { + return new Promise((resolve, reject) => { + const latestUIDAttributes = {}; + const f = imap.fetch(range, {}); + f.on('message', (msg, uid) => { + msg.on('attributes', (attrs) => { + latestUIDAttributes[uid] = attrs; + }) + }); + f.once('error', reject); + f.once('end', () => { + resolve(latestUIDAttributes); + }); + }); + } + + _fetchMessages(uids) { + if (uids.length === 0) { + return Promise.resolve(); + } + console.log(`TODO! NEED TO FETCH UIDS ${uids.join(', ')}`) + return Promise.resolve(); + } + + _removeDeletedMessageUIDs(removedUIDs) { + const {MessageUID} = this._db; + + if (removedUIDs.length === 0) { + return Promise.resolve(); + } + return this._db.sequelize.transaction((transaction) => + MessageUID.destroy({where: {uid: removedUIDs}}, {transaction}) + ); + } + + _deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs) { + const removedUIDs = []; + const neededUIDs = []; + + for (const known of knownUIDs) { + if (!latestUIDAttributes[known.uid]) { + removedUIDs.push(known.uid); + continue; + } + if (latestUIDAttributes[known.uid].flags !== known.flags) { + known.flags = latestUIDAttributes[known.uid].flags; + neededUIDs.push(known.uid); + } + delete latestUIDAttributes[known.uid]; + } + + return { + neededUIDs: neededUIDs.concat(Object.keys(latestUIDAttributes)), + removedUIDs: removedUIDs, + }; + } + + // _flushProcessedMessages() { + // return sequelize.transaction((transaction) => { + // return Promise.props({ + // msgs: Message.bulkCreate(this._processedMessages, {transaction}) + // uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction}) + // }) + // }).then(() => { + // this._processedMessages = []; + // this._processedMessageUIDs = []; + // }); + // } + + run(db, imap) { + this._db = db; + const {MessageUID} = db; + + return imap.openBoxAsync(this._category.name, true).then(() => { + return this._fetchUIDAttributes(imap, `1:*`).then((latestUIDAttributes) => { + return MessageUID.findAll({CategoryId: this._category.id}).then((knownUIDs) => { + const {removedUIDs, neededUIDs} = this._deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs); + + return Promise.props({ + deletes: this._removeDeletedMessageUIDs(removedUIDs), + changes: this._fetchMessages(neededUIDs), + }); + }); + }); + }); + } +} + +module.exports = ScanUIDsOperation; diff --git a/sync/sync-worker.js b/sync/sync-worker.js index 429dfd928..4850f28cd 100644 --- a/sync/sync-worker.js +++ b/sync/sync-worker.js @@ -1,12 +1,9 @@ -const {inspect} = require('util'); -const Promise = require('bluebird'); const Imap = require('imap'); +const EventEmitter = require('events'); -const State = { - Closed: 'closed', - Connecting: 'connecting', - Open: 'open', -} +const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') +const DiscoverMessagesOperation = require('./imap/discover-messages-operation') +const ScanUIDsOperation = require('./imap/scan-uids-operation') const Capabilities = { Gmail: 'X-GM-EXT-1', @@ -17,40 +14,69 @@ const Capabilities = { Sort: 'SORT', } -class SyncIMAPConnection { - constructor(settings) { +class IMAPConnectionStateMachine extends EventEmitter { + constructor(db, settings) { + super(); + + this._db = db; this._queue = []; this._current = null; - this._state = State.Connecting; this._capabilities = []; - this._imap = Promise.promisifyAll(new Imap(settings)); this._imap.once('ready', () => { - this._state = State.Open; for (const key of Object.keys(Capabilities)) { const val = Capabilities[key]; if (this._imap.serverSupports(val)) { this._capabilities.push(val); } } - this.processNextOperation(); + this.emit('ready'); }); + this._imap.once('error', (err) => { console.log(err); }); + this._imap.once('end', () => { - this._state = State.Closed; console.log('Connection ended'); }); + + this._imap.on('alert', (msg) => { + console.log(`IMAP SERVER SAYS: ${msg}`) + }) + + // Emitted when new mail arrives in the currently open mailbox. + // Fix https://github.com/mscdex/node-imap/issues/445 + let lastMailEventBox = null; + this._imap.on('mail', () => { + if (lastMailEventBox === this._imap._box.name) { + this.emit('mail'); + } + lastMailEventBox = this._imap._box.name + }); + + // Emitted if the UID validity value for the currently open mailbox + // changes during the current session. + this._imap.on('uidvalidity', () => this.emit('uidvalidity')) + + // Emitted when message metadata (e.g. flags) changes externally. + this._imap.on('update', () => this.emit('update')) + this._imap.connect(); } - queueOperation(op) { - this._queue.push(op); - if (this._state === State.Open && !this._current) { - this.processNextOperation(); - } + getIMAP() { + return this._imap; + } + + runOperation(operation) { + return new Promise((resolve, reject) => { + this._queue.push({operation, resolve, reject}); + if (this._imap.state === 'authenticated' && !this._current) { + this.processNextOperation(); + } + }); } processNextOperation() { @@ -58,239 +84,88 @@ class SyncIMAPConnection { this._current = this._queue.shift(); - if (this._current) { - console.log(`Starting task ${this._current.constructor.name}`) - - const result = this._current.run(this._imap); - if (result instanceof Promise === false) { - throw new Error(`processNextOperation: Expected ${this._current.constructor.name} to return promise.`); - } - result.catch((err) => { - this._current = null; - console.error(err); - }); - result.then(() => { - console.log(`Finished task ${this._current.constructor.name}`) - this._current = null; - this.processNextOperation(); - }); + if (!this._current) { + this.emit('queue-empty'); + return; } - } -} -class SyncMailboxOperation { - constructor(db, {role} = {}) { - this._db = db; - this._category = null; - this._box = null; - } + const {operation, resolve, reject} = this._current; - _fetch(imap, range) { - return new Promise((resolve, reject) => { - const f = imap.fetch(range, { - bodies: ['HEADER', 'TEXT'], - }); - f.on('message', (msg, uid) => this._receiveMessage(msg, uid)); - f.once('error', reject); - f.once('end', resolve); - }); - } - - _unlinkAllMessages() { - const {MessageUID} = this._db; - return MessageUID.destroy({ - where: { - categoryId: this._category.id, - }, - }) - } - - _receiveMessage(msg, uid) { - let attributes = null; - let body = null; - let headers = null; - - msg.on('attributes', (attrs) => { - attributes = attrs; - }); - msg.on('body', (stream, type) => { - const chunks = []; - stream.on('data', (chunk) => { - chunks.push(chunk); - }); - stream.once('end', () => { - const full = Buffer.concat(chunks).toString('utf8'); - if (type === 'TEXT') { - body = full; - } - if (type === 'HEADERS') { - headers = full; - } - }); - }); - msg.once('end', () => { - this._processMessage(attributes, headers, body, uid); - }); - } - - _processMessage(attributes, headers, body) { - console.log(attributes); - const {Message, MessageUID} = this._db; - - return Message.create({ - unread: attributes.flags.includes('\\Unseen'), - starred: attributes.flags.includes('\\Flagged'), - date: attributes.date, - body: body, - }).then((model) => { - return MessageUID.create({ - MessageId: model.id, - CategoryId: this._category.id, - uid: attributes.uid, - }); - }); - } - - // _flushProcessedMessages() { - // return sequelize.transaction((transaction) => { - // return Promise.props({ - // msgs: Message.bulkCreate(this._processedMessages, {transaction}) - // uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction}) - // }) - // }).then(() => { - // this._processedMessages = []; - // this._processedMessageUIDs = []; - // }); - // } - - run(imap) { - const {Category} = this._db; - - return Promise.props({ - box: imap.openBoxAsync('INBOX', true), - category: Category.find({name: 'INBOX'}), - }) - .then(({category, box}) => { - if (this.box.persistentUIDs === false) { - throw new Error("Mailbox does not support persistentUIDs.") - } - - this._category = category; - this._box = box; - - if (box.uidvalidity !== category.syncState.uidvalidity) { - return this._unlinkAllMessages(); - } - return Promise.resolve(); + console.log(`Starting task ${operation.description()}`) + const result = operation.run(this._db, this._imap); + if (result instanceof Promise === false) { + throw new Error(`Expected ${operation.constructor.name} to return promise.`); + } + result.catch((err) => { + this._current = null; + console.error(err); + reject(); }) .then(() => { - const lastUIDNext = this._category.syncState.uidnext; - const currentUIDNext = this._box.uidnext - - if (lastUIDNext) { - if (lastUIDNext === currentUIDNext) { - return Promise.resolve(); - } - - // just request mail >= UIDNext - return this._fetch(imap, `${lastUIDNext}:*`); - } - return this._fetch(imap, `1:*`); - }); - } -} - -class RefreshMailboxesOperation { - constructor(db) { - this._db = db; - } - - _roleForMailbox(box) { - for (const attrib of (box.attribs || [])) { - const role = { - '\\Sent': 'sent', - '\\Drafts': 'drafts', - '\\Junk': 'junk', - '\\Flagged': 'flagged', - }[attrib]; - if (role) { - return role; - } - } - return null; - } - - _updateCategoriesWithBoxes(categories, boxes) { - const {Category} = this._db; - - const stack = []; - const created = []; - const next = []; - - Object.keys(boxes).forEach((name) => { - stack.push([name, boxes[name]]); - }); - - while (stack.length > 0) { - const [name, box] = stack.pop(); - if (box.children) { - Object.keys(box.children).forEach((subname) => { - stack.push([`${name}/${subname}`, box.children[subname]]); - }); - } - - let category = categories.find((cat) => cat.name === name); - if (!category) { - category = Category.build({ - name: name, - role: this._roleForMailbox(box), - }); - created.push(category); - } - next.push(category); - } - - // Todo: decide whether these are renames or deletes - const deleted = categories.filter(cat => !next.includes(cat)); - - return {next, created, deleted}; - } - - run(imap) { - return imap.getBoxesAsync().then((boxes) => { - const {Category, sequelize} = this._db; - - return sequelize.transaction((transaction) => { - return Category.findAll({transaction}).then((categories) => { - const {created, deleted} = this._updateCategoriesWithBoxes(categories, boxes); - - let promises = [Promise.resolve()] - promises = promises.concat(created.map(cat => cat.save({transaction}))) - promises = promises.concat(deleted.map(cat => cat.destroy({transaction}))) - return Promise.all(promises) - }); - }); + this._current = null; + console.log(`Finished task ${operation.description()}`) + resolve(); + }) + .finally(() => { + this.processNextOperation(); }); } } class SyncWorker { constructor(account, db) { - this._db = db - this._conns = [] - - const main = new SyncIMAPConnection({ + const main = new IMAPConnectionStateMachine(db, { user: 'inboxapptest1@fastmail.fm', password: 'trar2e', host: 'mail.messagingengine.com', port: 993, tls: true, - }) - main.queueOperation(new RefreshMailboxesOperation(db)); - main.queueOperation(new SyncMailboxOperation(db, { - role: 'inbox', - })); - this._conns.push(main); + }); + + // Todo: SyncWorker should decide what operations to queue and what params + // to pass them, and how often, based on SyncPolicy model (TBD). + + main.on('ready', () => { + main.runOperation(new RefreshMailboxesOperation()) + .then(() => + this._db.Category.find({where: {role: 'inbox'}}) + ).then((inboxCategory) => { + if (!inboxCategory) { + throw new Error("Unable to find an inbox category.") + } + main.on('mail', () => { + main.runOperation(new DiscoverMessagesOperation(inboxCategory)); + }) + main.on('update', () => { + main.runOperation(new ScanUIDsOperation(inboxCategory)); + }) + main.on('queue-empty', () => { + main.getIMAP().openBoxAsync(inboxCategory.name, true).then(() => { + console.log("Idling on inbox category"); + }); + }); + + setInterval(() => this.syncAllMailboxes(), 120 * 1000); + this.syncAllMailboxes(); + }); + }); + + this._db = db; + this._main = main; + } + + syncAllMailboxes() { + const {Category} = this._db; + Category.findAll().then((categories) => { + const priority = ['inbox', 'drafts', 'sent']; + const sorted = categories.sort((a, b) => { + return priority.indexOf(b.role) - priority.indexOf(a.role); + }) + for (const cat of sorted) { + this._main.runOperation(new DiscoverMessagesOperation(cat)); + this._main.runOperation(new ScanUIDsOperation(cat)); + } + }); } } From 3a677a8c2d14f53abbb64176f6a726cb49f38f5c Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 20 Jun 2016 12:19:01 -0700 Subject: [PATCH 003/800] Create README.md --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..d889369c8 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# K2 - Sync Engine Experiment + +# Running the API + +``` +cd api +npm install +node app.js +``` + +# Running the Sync Engine + +``` +cd sync +npm install +node app.js +``` From 6577bd93585aea4000214f5294b8b66169e7c96f Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 20 Jun 2016 13:21:10 -0700 Subject: [PATCH 004/800] Update readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index d889369c8..7c9a6c915 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # K2 - Sync Engine Experiment +# Initial Setup +``` +nvm use 6 +cd core +npm instal +``` + # Running the API ``` From de8e09d6b555f50bddd286de18c8988caed51ab4 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 20 Jun 2016 14:44:02 -0700 Subject: [PATCH 005/800] Unify operations, fix a few bugs, logging issues --- .gitignore | 1 + core/database-connection-factory.js | 3 +- core/models/account/message.js | 5 + core/models/account/message_uid.js | 8 +- storage/a-1.sqlite | Bin 293888 -> 0 bytes sync/imap/connection.js | 179 +++++++++++++++++++++++ sync/imap/discover-messages-operation.js | 131 ----------------- sync/imap/refresh-mailboxes-operation.js | 3 +- sync/imap/scan-uids-operation.js | 98 ------------- sync/imap/sync-mailbox-operation.js | 152 +++++++++++++++++++ sync/package.json | 3 +- sync/sync-worker.js | 125 +--------------- 12 files changed, 353 insertions(+), 355 deletions(-) delete mode 100644 storage/a-1.sqlite create mode 100644 sync/imap/connection.js delete mode 100644 sync/imap/discover-messages-operation.js delete mode 100644 sync/imap/scan-uids-operation.js create mode 100644 sync/imap/sync-mailbox-operation.js diff --git a/.gitignore b/.gitignore index 9daa8247d..4cd9e26d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store node_modules +storage/a-1.sqlite diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index d9dd80a9b..59d021487 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -33,6 +33,7 @@ class DatabaseConnectionFactory { const sequelize = new Sequelize(accountId, '', '', { storage: path.join(STORAGE_DIR, `a-${accountId}.sqlite`), dialect: "sqlite", + logging: false, }); const modelsPath = path.join(__dirname, 'models/account'); @@ -55,6 +56,7 @@ class DatabaseConnectionFactory { const sequelize = new Sequelize('shared', '', '', { storage: path.join(STORAGE_DIR, 'shared.sqlite'), dialect: "sqlite", + logging: false, }); const modelsPath = path.join(__dirname, 'models/shared'); @@ -72,7 +74,6 @@ class DatabaseConnectionFactory { this._pools.shared = this._pools.shared || this._sequelizeForShared(); return this._pools.shared; } - } module.exports = new DatabaseConnectionFactory() diff --git a/core/models/account/message.js b/core/models/account/message.js index ced3bd00d..9418aaba4 100644 --- a/core/models/account/message.js +++ b/core/models/account/message.js @@ -1,3 +1,5 @@ +const crypto = require('crypto'); + module.exports = (sequelize, Sequelize) => { const Message = sequelize.define('Message', { subject: Sequelize.STRING, @@ -14,6 +16,9 @@ module.exports = (sequelize, Sequelize) => { // Message.hasMany(Contact, {as: 'from'}) Message.hasMany(MessageUID, {as: 'uids'}) }, + hashForHeaders: (headers) => { + return crypto.createHash('sha256').update(headers, 'utf8').digest('hex'); + }, }, }); diff --git a/core/models/account/message_uid.js b/core/models/account/message_uid.js index d9174061f..0c7a2352d 100644 --- a/core/models/account/message_uid.js +++ b/core/models/account/message_uid.js @@ -1,6 +1,7 @@ module.exports = (sequelize, Sequelize) => { const MessageUID = sequelize.define('MessageUID', { uid: Sequelize.STRING, + messageHash: Sequelize.STRING, flags: { type: Sequelize.STRING, get: function get() { @@ -14,13 +15,12 @@ module.exports = (sequelize, Sequelize) => { indexes: [ { unique: true, - fields: ['uid', 'MessageId', 'CategoryId'] - } + fields: ['uid', 'CategoryId', 'messageHash'], + }, ], classMethods: { - associate: ({Category, Message}) => { + associate: ({Category}) => { MessageUID.belongsTo(Category) - MessageUID.belongsTo(Message) }, }, }); diff --git a/storage/a-1.sqlite b/storage/a-1.sqlite deleted file mode 100644 index 2ef02b1ce81263c8c8122f11d48e35a8b448d97d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 293888 zcmeFaYm6gVb|#ipIWs+-qm(q#YG);_cA~R2Q&pYGNAS(8%ADpi$xM>Tb+iy@mwCSg+x=Vc5Xhu;c|SEbKr2W5C!yHn96gSlAX| z8D1OkAN}!nZaf%42AQwwndwnwcV&|CIQQOj&pG$pbI+rEl9Pra*OZl>Fy#De-&lM0 zZ0%EyTU%S}uB|=$KK>d%-@pg@XZ$=fKCJlLU32~EpMDb$JX`;BdhFZlfBx)t#q)oG zZhh+;-(sWv-N5<>-}|lKdiJ+o4~1G+%!{fjv_#eT`;9~?6|1DUN-UmBai1HPKj&V4 zE;T;qvV}_OAXVaurEES{y5Npd7hJ4bInEaFUOrW*>~No}quP~NAAZiA#!87ytn|_s z48Ef4eQ7Wd7q8dk#&qG~wkR}2MP0bw5QgICToPTZWb-LN9rYDF`Z-t07B0}cmpp4~ zC@6|Jf3=P$hhig!E*Jo~!f}NwRC76kG$7#4w_g>Xf9reO+s`f-($woNsQ9faUXR3n zU7Y*6ZbRYR&6j6&u%a3dXnN7U;4a!$mI>W_$cR%?)v#% zZ`d7)2Dxpoch~D(efMPJgS8*bLwy>^>W3e!xn>{>ysO-oiuTxuHCG8sPE=AFO@!YbIjLVt=UO-xGm;%MNrP^sYwz&N85Zm6K)Mt$x#nSx|Mq zA#~;bQW`K`bfy90Zt)u*tbIJgY(eD(}T(?711hxG%h{w-?&siT^idcUdz{6)B ztbNY}_-I&Q?hn7SiQnt%|MS}V|GoYv`1AhfJqF%m;OmTmUs_-P_?fOHRSV8?fQSc{$H;DXY2p*`hU3o`|JPi`oFpUZ>;~F_1|0nSJtQN!*zN6YF%9a zeEqZa)AbVu?CXrdue9&K|K+n^Km&g`rdG1qk~kbGeOFN!ws*NVHF*T>_0t~P0!aOq z*l$UF(Os8&`_Iyg{#bbK~4qmc$!zmrLZU z9Q(+j59~Tr3RkBukNYmDE7(@-a#K;=;kGoo#mKo78#)(j^rZfl`Hotb6#zs(%}Z@* zsP1yHT<&e9R4rFhNzTRZaPinhs{GdL#TR#Uhs#v*xwrXLxg0x4(c4~rXLEz&DsR(S zdNGqNzfB*P@^8zT<5Gnl0Ek?ya8SkbWlqFEHGtJJ;J3PRP3STJ<=ETPRH>XjE&zLe z@X$A2;n&&^F}MHE5Af^%`fdFBvybuXPq*;v|M&>M{so6$|C~0~fBe_*>mUDB{QA#- z6Tkl9Z{XK|^6U8ZAN>{l`upF*um9jL(VE48D8;T^&F_*H{4@9r@D|Jm=Zt$*kFAK?#)8~psP^o*wQfA#prMAKAua=~--eBh+2-yC_SJj)-+TKX z_xVr3{zvTcium7V)u-?8f8Jx@JqG?lV1W4l`LjPB$* z+3$VGG_*>a4&7A!&j)tROuzaNt*j|2>hP`ONmUnS0p~k4Nyf2X>YUCTA9*Il)};5^ z=TA=iB>w*n4*J&rtM&T&ci*4?*-ok6pa0>sw*3D5@6ktW>=gTF_56=@j`;uE&oXQ4 ze}6sw{QrIa?>ygo_Gi!j9xlB9c@PGUf7N!#OE#HhRI|kRrTab@cT2zW!P*P+kk=Da z>TOv6p9Zt=-4E8jZ-u$kQ^4qj-L_=-XBcS;;a3fCzrLhzX#_P_#HtwGMqRnqa1OCMRZrb{5I=Fb^aNZF{ zPXzu8OMsW&0dNrcFG&5GJP`(ip{Ne|PmN`FmyVt|I(jlssb&XPbf{ER${JY=FY*;lG}FP^|CA1g>G9`humkpFHDD8 zZk`={q4CA1i+HdpP4wffR2g02qz1>w;@(&q{pN9E^ Nxi?GEw6FJ?TRaSlXoq3k z%T&1n1U$O$9DW?T9~;4us6UkI!cda?TwdzpG|)O2!a7GIipHaS+8Yl=A~AO=5llwd z=n^=~qX*b&-qQYWZrIy#l|*&arGvM5gzDfL9jUiB5LA_W$$NbPS4|pniYjm&ar%nW zTnXKcJ(JWx6UJ3(Z$^QZ%eB2sG=FfBEFC|nh2G*(s->Y7cWZlsr*@f(2g-VT6BnZ^ zmo#AGTAWMpi98z4(P9fMIu@pLBx+#{pGKN5M=G6qR&wPV-tE(lTICuhime;n|uzM{k3mj#)tcO{SnR~MC`)FjFfIfXnU zRX9`9Ggd+Z|8JTH$PTMr@GBX4sfd6dQcORc^D*8v=c-|X?PIbjdn zqj-04)4Dxt)Z+lrj^_&@ z{IG)DmOoYU#+E&V*ZmRxs3&l;mkcK2P2H4?SmxiA&F967rso0^|t3W6IXM5 z;j(*P9ULA3#IzCS`?*}x-<KCJi-k0JnL6(1e+DvU0ZYzT<=~ zoh?#wnRhs5j9y+kX*6GuZ+A=?KxngONY7#--I!i?TH{u_S?zZHozeK#KWJ4dSMfq# zy!6T)`Lj*{OIKh@GAsCR*p? zT1Gijqo<>GMv@1<({U#nj<-wMgVFes2Z-(s?@M^2`DS3K^b+}gaUcfU=~huG=OgjX zd8SznE52^z+!q}bg2PTRfGL};&XADAz;?OwQWqyXngh;A6%|@mqa;%CAs*mk{#eQz z=7Y=ifjJSFhs(U{ROca%C%2Y_&XpBoo>K-_Ldf?8WomhZ*pusFA(jS$GV}4}~J3uscl-Qz%7OQtYCS3R_Tj zIU%0Pc+$gSLq0uKPWCVnn$}5fL3v~S*8G9U6^hFm>lORZ6zbyUMoLj+MRir=)mmzr z>9G~KUZ-KjB-NIyh(eEhqmBjxSsB_?59V`4VvV3WJ+Qe^)*>r*xihhgu8~K#33+pa z-YZPILSGmSSE-Mi8`>~MR~Ft~Z)o>IuJ1lyARGw0Ep+wi0#Txmn;VPUoroJATaFF@ zw9~TQ1^fS!%E1|%lX`>!0g8jv*TR8Fp(M?BDvxjt9_y02xgnBE*bb-4+@LE8DybGi zgTqrDQ5;zUx1r0Xz1oX}_}nB&b2lSZ9C5R40X z^x{V31`7PIYTYR(bgAtr*OG@Afeb}hu|s9u9EKr3L5gdPo`>TVBvU0`H$17o8J z91R#=(C~|H7ZVLCXbV6gjjB1qrGY#KrZEwM#ID>T+D7c#GNkQreHo9aL!k?DCZ3Sdao`HWBehyWT$7tSZemm;Ma4MB zA}AC5LAZjar6ySz_7I{~WVoeWCSb522&Dn40aCObfF{0m6<+1g8k*J6sCV2AL5+@& zrEYg~gN;G&h4vIJOLT6-+MylTSXLT4Tn!>fYSJSw6x>rgVuM;G^WQIT^x##AAvWb5 zY>pK1h02ZFzz81Z`XU|#yO6i$E=@vNR3N(Xk#>zF3*4GoXj51=H;RxY`a{vEpAC>k zzrzJWLH3PM0BR|ha~|w^^hZ;D3V~~IFR8oWXs%p|l`6Q8HMXh`On-FgZX9jvqt>5% zc|q)KpaPIeG9WkQE*wui1MSUD0mK`l(?fn<(H!>%lM;654~K!S3#2<>oEQR9(VzoE!b zCK$mA{VAtOH5g}9^l}-AYJ9D2`&17e?n1^gzNWqwg(-ED!2}C*F;k9=N6(&#TpL1- zO?^YGOAwkS*Foly2*5ND1F=a2g}Avh7$9`Q+T36g8FCk^35eg6kE^7iinBsZj1r8nDEQJ+>N>|f9yX~5ar;p2?vi`r@kqSZt|LuLdlsp<$q2P-1fF_lx>@~GQj=tW}>M`hd|Mp`j;qFbUZD;WkENogPJMwTZIXlFdYM(nh0f#nI%EK>^_Z@6M#G`N^OK zlKbKW+e8%#+Z@HOkeM1mqPL(~EIDkvEvznVVI=8=odhi)_Q6y4sPjzNGJ;jH&_Fl9 zV+Q7kcbJgk*b%jxD{B~KOEO6eXcKZ5ePy5-_CX1{szQ4x0n05-VH1&^{f1RQlOnis z^~9^-bdN64J&d8CT|hq9z&2nzrdWVzZooHMuNj45?o^pP^dXnPm^ANfa=`2u1#Wbq z)SIYxf#je&wuOGjx-b~?41Qw@4V?cu0m~P|;}A%osV5AIP2iA}IKqByBz0*^&oJMF z&I=>Z4hJ0wVXfNZJ(>=-qGP?{V5=e;13f}1V>r|`K5BL*8oCl_Py!@s5bGVA6*NmM zLhoxhpmu5M2nnP#iNF&YGXx0U0}n7vRB+E2Ak)YR1uoYeI~VB9%lA)7Ngc?lL^HMv zWI*a+N{r1_4v`fUap8xmJ-d4EE!Pk%;UTrjwpBr2c6V7cJt8T52&+i@+!5bvE(UM# z=xZ+Sck}+&+zI``8gjoq*XG8X!G112lvGiC<6#%U|G(1Or#Wj8heMVrQ}7aIPQa@&W>;1^#I209CW9`i z3Z#-S;e#pER6`lu44t2-WY#?Ch&yxw~UVxK=qs!T-Pd)NP2ilX}=LM4x8sUIaTd+crdhWZ7oKobQ1m?!STMTHL2C ziT0!KQdVqGj3?&K2E#;vb zm>Es#yBOU#fW?t%le(79#OecfHn2j_4oMfpKkPPWbvE}mLJ!9vw8#0O!fr72>0vt^ z=;e>z=z#y%W}BU-Z*|D__mrIu_lS*-l{}Vic;<B;)$ z@G)B)!x($_h~_piTl;rzir6SMMSr$9JM4H&U7zM~VZ&pcq}XiZW1W_jdmg=M%-Vf+ zKCpK^b>pMypHJQQu#K0~w#UTBV1HrvW8AxQ2c+#4o$deOmG*z*E*SuyJsYgy|Mx#X z0Svs4|EIhFKcDgcZXekL*x*!{nCduF%@+4wm`-TO)s=y@W$y*DDdBU1PL*&#tAC}l z$qdf3L}l*<&Jth1Yz4P0ScmC`cF6=^ZwokTApa}61ZONVc{RxYVwIwKZXCSokAmD; z{-0;hKU~BA?|=RxW8fd1Kd1B&|C@`Dkj+-e{(%1l&A-4!4hA^WWVs1JV;ouD$XJW4 zk<5a3=Th}_W1TM?4LsIZH|WB?{}Zs*Sxt4c;aX;+qa$gXwdP=q!$ec=kncHo{pubX_mL?iq8{TQ?6*4z`<~<=9NG2yjJz{5Hy(DF z-<$a7{CHM72+u#g#EHF-Mw5OE(n!5*s|NqJ|7h(I<;rtuQ%bcsc<{oTP%Tri^zmIpYZ_zv0 zXXkE@*69Y% z#mUXBXH>=Pm2WHCQ(rXJ3f`7Zyr65<;YG3AYy_fb(~B!5APPsYwm^p4%hgZYHs@n1v zlWLGn!dor>!ZlHG(HVXq8G9JX};P_Rdd2wnwR6_LNuGprYa5ZFx)(9 zTp>xI?tdJ?B!2zee6yyHta(^ycr(EFL<_co!@;|7{RM%zEu`|PGqEl8{|yA4^N4Bw zzq7lV{qLWxJ^N=Q3EuzwJYnGXzq?Lb`v2zpG^n4yT%*?g2h45{$~DR)7+E17n^Dx7 z9h6hxID4N2p+92=LQutD*7*PLu08v^L?rKjekL&Rx7yzz#{D~a>Pje1?N0Cb}`39Gf+YcI#_NsX2w1EM`?au z%-tLwshUqXxu}vWC39=1br4Yjj(o;LFfcL@;RZ#-fFTJ^|ccRxX-aDg3XQos8^%t1M=u*PF4%OfOE3Zr0xrBJHFpB&PrP{ z3~lhLUg1}*_sH|3(Ox}TuLYpJF#O9#@2q|S4*{$VZXPcAJi?-H>J}{fu`z?28;Eo0 zkqf6{3sZ8A{|SUv#sAFig24W&x$)B?{(l0tKij?+$Cr<{{Y7?p-!7_$=sBksY=3js zfjg1|%s=g5@gNgHEFB8-A=t+P@E(1PjlkAxL&<-1-%aD+eE08f{EJ19vpW*xL&>B& z3d5$CMi-7Rv;NuozsmY&@6m&7b0$?%%->zszp2zrMJG3<2et47>tFgfzO9bTWCUN| z zSi;&A#%Z16w+v#2g#?|uZs^;%X^^0JiKfFThO;9Pslc&;&6pCL+ zy~~6cj7{n_|G4JQs)0c*cD;QNma0|6_kGy%H>cC%KxddyM1OY3XG53MzWg&{`O~By zX_%DJ;oVvOmQNWlqRpBiJr7z^(cf$cg)M`6=TpS7> z&+U2Xd{hr-2&vbB$#Ff<$Q^s)*(;^lPBep!QGdYuhU#`IdRXq%gv7yRTn#m1SKAkb zv9A`mnVcTx!-eW`;HZsAln)O}XV;29GCHoFmebwwKpanxjt`2{#zi95K9XwV=t1i)0f3kdp2*{` zRRRDC@n7NEZRQ6^(K)6W`GlPdi^yafr2)*bdCjq6F8<%o`}nBaLOKtK{||&#+W*YE z+W!BatUdcDBnjUCeC;ss`#axcM!qWb{O8Qbw~~A<&)#1iudv$AJ0xRmlsDvVU$lm(XPlyi^0WuWVhSps^uj2+bB0i)xUHrrSlxX zN(jSSNkkmI(DfKrIs}C4mP^eLh)Id*_DSQEjRo89auj7dOKhNpjO-%);uNz&P2)wb zm`nrHH!|fa0Av=kIvW_A8EvKQ8A8MJ0k1rApCn)=6fIJDv zV!blA=N-LKS)Bx!X1kZujTV*sn{cXs?g$hu~wp*!t& z@y0+M4wUeNT6lu}u6f*7J^dR=Z}01CzjLP`Oq+3*Sn zlkG_1Kurx3wZrp%|E%Yq6o-5|-mfwnp27R^*r=@x)w6c6T&rR7M6pn)=eHy6?U*Rs zZeK-DJxAU^uTq-ir-7s9jp+BB$4*Clvv;-~i`3InI$x}hyQ5^TB1dGo4iMF7wHgqP zD!Gwd9Zr*cGM&7<6{?=g=5_q~d~j4e9NnY`m4j-zl)qKH^-AnwBJvl7XyULX1Sgc> z^WdyjZf{2?foM5Uznuu@iIZVD@-t<_dy4&z^yYq@va(ted@i3dLModzLxTD{!SZ1x z)Cp`01-TMaGp*C(vsCfIvz_x6x@Y0TLGChI%@z{vai)+iHiB0GF$(&wW5ryorHi9N33Xks8>P{0Q1*|qs&9OH2uR3 zqv3RHc%2TGily*y8m`CB%A@?1Sh`XJVzSxG6mR2)#p|B0sEo+$I#VwoRSYJ+kMo`v z30qi>&u}u3j3YZ^6lT0T7}<~=E6jNJbkBQmhR+Dk`+~j@@8_dIe~6Fr{&2ud^La{<3Nm7d z*ecj;TsTPAJBe&Kte0+oIst=MB>5>e1!@-w(~wTaVTFgsTZttbCez#O(YYgWx_Q6U zoA4YmJsEq)f$iqlQ?4r&q0kCvqQl{BeVi-C<5G5VIPo^bRHZuY31b~AV;w_&Jnm2a=D)>&A-!ae zqO}nLdyFzX$YG}l^kz~*er;x3!oY@SPQcB^*a?nKzvQ*v%ER{Kilz6rw%m`lcr@1Q|lj&=W=tM@aKWWJ}30$*kt)xC+`sH+6#? z=AWglQ!!RdUSFle)Rd+bucQ^wc6*0A5E0DW-{BCy53fmFNREj3sGZUXi5ux2$apN0 z{THS=t@~P%U)bVc!IQ{*2ujs54Kr#y6aVCI{ylV!B?M;a;RM|z%XkP_f!fp~(zeZs z)US6(EN>$S?MBdxyCAjY04bhncH!BHL;;#Pu^~1E=w{P83QTYXPUxGNfKvb!w`bH`b)V#L1j%O z=^iOW;CZnN3@Ux11SE7uCU4^XnM^@4VMD0g2ZemlL!3Z15tq@li!^UYEEh560Mg1u zxu}OO*#PW$#V|=uxRfv(6Kvk%dMs%yrA|doAG%Y!kz|;%>uqK_XCmJm?dFEjCe|pV z0VhnL%80})=bG`JDlgHvVN|CK%Is)>Sd;SXGE%3kq?#!k8FFLO0EBM>th z9*Dl_@J18&MA~e}Lb8f>FRY!<&e$46*Vxp~<_KVhBX>>o)NW5|kyhjAv~{83dBwuu zM0D&LJG8K*^F?aS0qRRI4{M|jyTN({>7xl9q*EvHL49KBTkSJ|bhlaxT7?~Q&{GD& z4a_ldgf&wl{-^8!*r&0KW`r!tL`_m!?)64}qp*|5eIjZ0;zGSch)m*L=A(|5DeD3?0d)m=;!QS!^hL5^EEP3b&zQ?nh+!oK4Qgjk zpRopGjT^iQjsZ&UKt|95OnCX3bP(9PE`p=;e->%uNfYB*)DUKzDn4M2aZU+T;M|E7 zBhU}I*#2G1^2)R~oSYim1D+uR2RRMrP&?@`hmr16u)C#Iq{r#t&P`AZqBq4$0*>WzTn9sU}YJ(kiTGh z8jh@kgNcWC`-9w8hJ-v7@`HXb61R4^7zM>3|1sDA+BSk=UUOeUEaKgsprAmM_BQ8b zmyx@B?z-PQn6O)2$#M(r>2C?90@Folp?cX9Ca$s6piJI@2!{KL`+|~YS(>#$V=!>r9HDvF z%TYT?1dVNTLgkhNxqdTHIw^LcVX?unfEzm~PS- ziAG&Q7aGkjlyQK1&B(z;YWptd^Al9k<6UFDA-`9f<&WEtyHq^}-9ncH^nvQiSbD@z zzt|X3p~{E@W699_Sl_j&Zpt$F{}oevdM&`5Aw)w5$Kx-fBOT2=8?iB1?HkZTWZN}H z#70+>hcF78%}^6=K%Z}e4V9>F)JNq78Wjy&!vpWrr@hVNhDP%)4K9$xdSydWn2=<7 z-^Xxa#Wk(zqQW}W_L(rjM`0Q!@rC7eKC&8 zG9$FbzH2WpLN@5KOqO~H!2sfxbynOyQ6no#V6RD3#pqPctZk@s7ifB3Um~A04sQ;9#~ZMKxdQPgMJIJ86WR$1p-@;Uc0r?5-DW*Vd=`4@dFEj zh)NGrRISnXYTC$=>|rio1wv~k8Lr2m?soAGmC^loc6|pRv;j3Z5j3yb33XT z1keStr7%;9X)f8?H)k*L5PR{`#@XIgZVLLwRoiiH?JqSUECodSJgKjpwr=gS-J^N> zYJJJ0VH=By2bci=Z<|wrb&X1g?QMM}*l+vL9&k^5uWzuyGUk-ObIbGA{yl8pCNmO> zn8hEg_9RO+N907dSzY*$n0SPt8Zq(j!cMJQj8>Sb&Do#H@_tG`l;BSSb*(oCqEJoS^2GR{n9H)C5_4*b8vYDKOvuszD+ytRK{ zW4UfCqNVOGTlwt#&5mT`P=B&%+q6bL-m-1lSs!cIwvaYtq*;mYDfVr4EJ3H4i{@=h zYgUqA;D6Vn?c!#S&8c6l zgv_S-UD?Xn+};mMEA6|w!laqoIYO45wg>y~%69IA><608;dl2=Ea%v(k&*u+a~fGC zGQ-i%a8R-!Q0KdQh3StDE;~igjNM+ZD7dV{5W{dS z5qYerKYHV-LF1oosP_-3!ZDo$j_R-(nsh^lsYnGxTCe7KX!9oA+Fx>mNW=th_D<_c zzfX0Y*{0IiFf8=OY~N6Qhj<=?qI}2ZF;25*wLG9A>n>`SwwPIAzoI?Q`k}&Y0cC?x zVsdW_hPJJJ?r(4~k>Dy}&Dd8NEm37?*eyGB{^y24mo-UR0#0eord`dRXTI^S*?5j# zZ)E$AY@KQ~*RaE7EgL9G=jYpNn(d|a7aq~xpgU6CsAkyeimbV9qrI(}O@}bsG8;Q! z4`XL!nPG#F%$a5eBxbnW+Sj7CU?gFOopgN2JW|MpFr%Iy%}CBV5r$^S`o`d4%PZkv zxf&@ps|P1)`skuGI)}G<;#g^&kT2=hew7TS+H;P*Gy!$Kt`e}B zfT3I~4`#Advfdy`fHz)pInn2)gJqs&<5ftidphtg^MxHR@29kk>aehTGk zJ5_DrQ-Z$6T8UU5f8wb@Rykl-{DnmM^dy^zUDX0r4?P)6oX4|g=lK}onqq1h7jmgq zrtCitN9b-QQ%av@N|l=L(%bN*rJg!990!xxGXBNF>!knyCw|9W8Q9RbK~UnBrpw}R(!srWHvTUOH#Y7hRdVt=&3MmdOX3y zQm&M|$%&`wY3fP}=Bu^VN%!LDBz;=07lq?a@b)l~I4vFruaQkfO&teE!kHX6;rUK8 zcI1&$*TtJ-B_8s$W4ZJ5YCYR759-ZkK2cU0x0lh3PdIM*3TG$5>1jJwO?LdPQgA$} zG$}MBB$o2+R5Hc)`J2!|yq1|J<@m*ooNM1SB~ab46+EwB$F>Xk?bI-Of{Za;_&B8s<;2Np^`@?5{Dli| zyW+i>UP?ESR`0q{ZdFRnt5bhq)HuINRu2{5;dZZHDGU&E7s*BeI&!Fk*>pjAv)M*r?M@X)NcfoK0$AxmH^7UY0_+;lS67 zmbZiHgM;99BNy=GQ>P~oZYlM$bP)$%#U|IeT&mMM7?%p&z>(+ppn6yxq|Y;9DHr1d z2X*oGLOAqFX|-{#3<9xSY&wn|wZeYobXWradoQy2%qWe~_OGu)x219ZY7jrqD<`c| ztTULzOU*`Dyq34Cxoj+VAs&m8oT!dRqtJCv5fu3>*N;UL-Lu1HSn4OvVpqXl&llg^ zh;)wm_<17`>JD0++im5j5%c(O`C%{9&s8pmKL1Vac6`|@`umf>_IA+|E1!)H(~03} z{yfDnmg=S}o$~0Umq>gy3}NUN?-@c{wGt*nB>BtU?c%@9jfJb;?v;FY7*P7xh0)+h zngm1YZLJyish+do@TL_On@Yq};kU73c=-NNQN5tKWor;pA{eXC95dXcX3Q|12w;Xu zD(*dLg{q(#u$9mj#nvQVJ-)ihwok_G*u?^TS-P z+BhlI1HoiR_GAXB!bwc2drmG#Qd}yOQiZVJcXM(Qj2(tf6K5?fXr9xHz*g5PbGFB@6bgF)ymricN9t0AHy_sr0)99af zN;m3-Z*b(##v13>{#K&LhkWJQZAA!Vlcn48nR?v4Jvlw?h!y{Bd%Juk#T8GiSPT{W zD-yFRL%^0D(Xb>UQN!hBB~Xh+qgC~|9`#*xFNG6zkUK1R&jaFZ^7zOLX#94%e9>+O6OC!) zFx+jFjxPH?war%wQDG1X9&DeTv@QZ=zN=?AS}T1C%$s(*py^*;k)p4;|%cfDRZymxs+E+onIMt6<; zAI4oR{?EPk%uSNu{m*I){Mq-uO$hs)|Me9;_b#gBIk-excE~Gt-cCDgsSi)+`5pEU z?q(hqU^a7tT<#NDcxTR!n;U!j59x0<4->j@_AXpYX0b~gd`lSEcjQ#ExncD*c!F1n zHR1YG-Adp-%ABxqq;aQTiH3|74kaMr-^M?GB46cL`2>Vfg81|jrXAEL?8EVtynPU9 zjhGhw#?q1rS_uR%qBT*v1o2ae#M?wBl{n&fH;>Y%m3;1PK2Sty_7-WHCLhKdDydhtrRn0{M~mCI)Mr!A0^-d?P_aKBhipEuwR2i<(Y=MMVK zx~#oP(C==$;e&yQoqofu`QsA`$TVjy*;*}Y^;y&NMO*K#*!t6m^t5DnqNsS4{MB8< z_7!%W9n`NHj;`8`(U5O^S|?LDl1^f{)^rJa-lvc)X|XprpSC@0sm_+mSBgkgHkrP9 z%~eJs@;(K)Lpbl#a)e)~R<;`qYFSpCT5<~g=T>KLN6&i`=WGtv#5a_{-S6d-!C)+y z0=eIvB<7Cp8;KrDn$Z{bPy<~L8-$t0J>>k}{j<6ANnbjeNTq|pw4UH-=@7vG0b>^I z4N%N6!V2Ne-J3Y}=s~%f`b2g_R!CbwN_?!h2dpTqsTB zOA8;7ud+qhuw-*1Wa?Ak?9Ds{$rbU%y?gh94zOPIXh`@7cXR|8OyLSIl9QpN_P7_) zAh2s`((CaBUce_3!K`BA6#}BDgZcr5f)TEoTLb={fkhp~VxM^jimdMI-qyW4zoox_ z59I1k))AwI?Wr`_vkWQdpP}C5wpF4V_*p61IYe7o=7{8)mQl_TIga_X-k~5#9%b{W zXVmIb2r2xpZ}#AzE$k_(;6fA~KG*jsXJ?L8v_aT?^Zi=-%K z>J(Py4XJ6s|7$7NDLf@8Ho$f=lSAbZm*tpg*UaFOe9cLFbL>m=Rcr5M%ifDxI-@AQ zY~FvAE5$>tPEZVII=A&b_K3;dV!PA|M61bWuB)D>{lQT!H@wLlsMiNLgc)6@ZfZ5j zr`#s_+fh27J5C=Yk84-?YP%)LS2xXqcp9IaFk&8MZ-s-?1JzUAo=*IwQhRdR>xXYj z)n;0~jMQ6#_u6-Gk(FA(v+niw(RuRlX395|)7JRr<}90(yJ;yGNn{VMw@fjN#6 zC*kehRd-wqoJ`IeiPjBOYcx>F#rl0{2X*pSDjpO$b;QExh%Ff(a0O%)WccPZ#^7kR z-Qo~qWRrLMsPsZ?%PjVLLHeKD z=Dn0&aaAfJb{6NHX^3W)^dW35_O{;o1b>8&HtMdZU*KJV03UYMyuq5w?-%`1zrWrL zgc@(R=5`t^MGhn}sgg*mitZvsL%%_n6BJ^N>fA%sMd)A!g6w+8xhpM-4GODqQ99B@ zjCY~dy{5hF=Cbsy(Yd;6jCK<_c55^mb$2!8t(+<(H#ZhY8M84Q4t6~rcHHW=h-=T# z({POp@Get1^6XQ3kvAj=;VNJ`r8OS)nUDG!t{9&RqQ+y~I*=dX!1fAW0ecpn!nDq~ z3J0n69wjote+bHm`og?hAEw&`?zSmuc;7kyPcRT%vHzdlrS0tVwSTbo{J+4jpVgn= z`KQ0JK^^_Qf8e1mX0tfKz84v;*a<^k8}Wf=K%go9$n~~~fVNb>E^Fa#*CV)_G+YSD z?z1=GE4j-U+tkhX1Upm~QJiPIX+ z#o@?=I0?47L)<*X4T9?xcypafD4AN0%B{C$OB^R1|ba8&XI_2Xq|hBH?^F~ z@C&4SY6tx%zqF79f5UnIH^}Dc8OV15-5Ap!D2(y zOyS{}Et+yRod!vh#})UP)tjYnX!;&iT zm51y?8O}^;aA947wP6P$AMe81n;1)@S0V{U+*;C!nphV|k0$aIVOK(okmeqPw&6*` zNReWDP`E=S@}|$ELNgw;tlY>PWB;+pY9e8M)nLo?eO3+E5O#QG&3(MQq8MfH^^?Wb zn~##miONMW#j;Cr#cDj4O>kQ-kLS#v@OYAyBzI0m8klRm95XJ0rDclR_jpo;EpAI6 zJXJ%1``Fm;mX(&LQu0iwhdcq$Ke|}#g8nv^E_kE8`N-w^Olor3)Q|UB)HOHhb^9tn z?fH`?9Xg8*B+y2@r1$kKxb^jx z-ujZJ0n2Qi0kt)%2OAQdyy=0zfzIK;h5A|24K-*kBG9PNcg=^2ZZI<}n@lq@-H#kY zO|<5UkRv05sN?CQo+<-V__puV$hoT;W117<4Uu)o)()KzrqH0+Xfb@bT-NUE&9IIm z*a!R{5vg~Sq><7=*g&yxQrtx}7}BU~!JcMM8H%|fhG~wjuBvofkJFa>i0{#&x7a%Q z(h@Klq5xh!Ui%a7Gvs<`0Sdyo6-w%*{FtMlU#+i}=uaQQkGYSKDhVA$UJ=?*;Kvq# z?MKw_{3Bn|bIz?G40y2>3awq;iVxAg^N;AQQ0(b!JQT|8iu~B26$c1&L*bVX#i5a? zQzOyEZq7I2fYRk7;k{0+@OLAOqN(qXFw{eHz!*<)%d6*wcNp-B(fAyjLEQPfZN~4| z3}WB!XvXI-;1#0@I5y)zL>3YN|3@j6!2_ELIyOZS-K*z^avRJ)vSLIb$7UQT)-obC z8izrzXeR8~3?&7)V>AvuS<#H67;s{Y`O*9=Hsf=Y1SHk(LfV5QfzOdeeRs2{|3SRt zbL1V$AaTcN{0==?P69qhf_~bn=CBm_94XCd&EmlS$RT3QVB&uewfP;%fF-9`%{%@F zkpYtHJ1huJ$e-h#fCE7;pC7*?891pF^UXX+uK69wzzOkBY9`<~H%=ItZ^l9NEk|0w z5ov+Dc_;Xwxd}KDb-;m)=0+2A=*jZY1Rdul=)e?n&4eB_nvf&XLJnP>Z^oI@FrZmN z6t86{;udkwqU^pBN1$*vZDSN_SVlR8^mANwin$k`xUjP zZs|fn&v&FlotN0O?*dqR0wnuG=!IL~d2ntmvh3|!ma))P8zGSq(wgkHDGwC-bmxOj z8HpscLF*m+7+Ih2v)E>BXoIYMW$#l%cXedRee->08gUsLO}UX1%8TSbv)dT+sMn{-lgGl3Ih^_G-`UG8pp&6dzpH)M z;58%_Cjn?rqoq1dIRKY4GF`C~;JHO*A@`I#uykdhh+4XzS#PYDScj}wm_25_Y8z~D zk;h{EwkLF#jCZp~X4h=3_$Zs%BDoOu=LckdXKiTwH-m3}XLoUg zj?A!K%bZZ+`ZwA<&E2&1C*)n)AI9mLWx$PWIY`h&so-WkSbCrJh~Cgh!0~0r+jBiz zP{54&KqAoj(P|@|yJ3TeOl!+9KZGz^z@HA$;T9Vo-0>77sx+BF`)+FymAaO-2nG{p z@6tEhn<$yK>`i@$45>Pc%@|Q!|l+Zmb-(y+DTi!N`+82IXz%%_~%!U*SRfpgE} z&Dui;9P*I`HNc<(>zj6b&fl?(*&kh4iVcVc-e&JxXxBjCY}$;+twVQfUo>7ZUa__B zi!P^Y6Px$#f%(E>AHe^6I?o#YSey^r_?A!2+^C%9X*ne=8OyRMn`>)8wh+|FbQ z$GNRbImekJn_J8|F0Cc5bXv|i&Ury+89C8}b@j&A#Z}r=N*vZ{1ITPy28j6u*aYCP z1RDU%B^U3T3oJf6uAnABhc(myXqi6?t6?eu)Ot^>Pl!#-B^I|G*+zTQa?AFjMhJSp z!2kQZ94cDPuOvp_bn#^f2iwcLaHL#V8q60BsbjpV^-5C*bpbyI&0#4tpy{v8_}s*+ z!zyUJWQg7wChUDN?(68WFO>6rTNGVpGsZ|QvoY6@`GjLr^Ydk_e{)@MSmlf^SV!o9 zBZt+@c+#%=;aG&f3sW}k>Pp}VU180v z?g<$MkW;&_RcE+YkAhF=Q5S`EPzGXkpWreyMpcBNEHtU^6mkO-((Xc3=n37@TU^mC z%$v?~)}-TpIyKV|v26=A$pJ%#7+Rl2!!&JaV0!XgqN570GpShh3vCTF&|x`D_d6fE z=+GVSYJ^y${NJBy`Q9}on`)b`kQxGqDV228z$QZ*+g&;@(?iG5#5klFqT(sKf}LmS z>s^|H?OQaca8w{13GQ9GX6^6@y059ZWB+RK{N3Ok9+`nh+pBudI~{%7d++eb3_RLi z1^({NFFr8??hgi50q27bX8yUpFFrB@?+dQ%JMVQk4xR(Q_{a=AABe2#y^~en4t?^q>)vHz&Tpsi?@ z$>kl+&gSs5)DLF9{{biX3pT@r{@)vZ&IYfd3PQC#_=}PILSePn9_w1)S!<2?6ssau6d<{SP?d=qbJ_^Zj?C(Wf}2 z%)t*j;r}V#D0A>bPWX9}8;YhnYYG;rn{;SxRktkbv4g0#3DQMX3sY=?AGN{P6)p>W z2VKDqe?d^#;qzWw1BT9JX~9Y3vNy3X#?2jydUG4{OG6Z$?+ZtRr#IO>9jpDo1l83D}#^Y1i6>g9i4nCa8-1jAl{8o2dZS zCLEM7dlL)8oEt^R-h|HK)+QF{a!&d!F5+`dgzZh}ikP(t2l;6q#X=wEnuyq&(B+`D zi3QT1AH^L_`0VUwsAHBUoRl&9C>Hu)ABE4(8-_+^ZNgzsVryd2(Ri*2JJA|SnYD?# zh;~tDm}|mLQNBfra?;A!D8TM+-gead}{Q|_dk*^za}l>6;dZfIy0WI5?) z_9pI_a=(4b4JFOm#9dRqsGryo<+o3{p{H4!xNFJ-cL))`eaiidQ|_ds*${PC69M~N z7@C@86pMJ6<99{<+p5_d0HXmc8WYJ74 zdfD7rv0Fq&I>a$9_elO!3+Xct;Z&T$(b$Zv{lY+E$%8xyF#M5bXoXX74hGF#9RKg2 zF6R#ZN<(1}M{-h~F5nA3y?QA6KEj#dQA`fSM#%C5ngQ@xOSWdVD8`bbbQ$PNk(XdctY@WSGNs2nsT*1F)v}kXn{F4e_;aAApjN(%&UORymNMR zwO{wK)xPQOPvizf{nA~XF*5yWXTml*wG=pDEx^Aw@VP+(7Fp3(4U)=)8hv7)tso5Z zK3Vqx%j!e^rJJ5MJ-Ur=%z^t0omi0e#bV6Ur-sl5Sfinq-rMZ| zg&VL8!#U#@%NK@blCHb+`ju5~TGlwgqNw99jV&~je{Etu6RJ-mat3wYl4pfFr!*1- zKxd)PT${Mqgy^?GoR1Z>LbPdkfJOb(czH+vyTJXr^x$_0=`(3!e%BBLSP#IFh9Ejn zYx^{~rf|Y{f`z6YjU@)xufYW@L)^9XOOL`8AqueA8K8kMvRN3PEjQ*c$ojE>T)=|Q z0!dfP(qmRA)|W*9{V|L9f5U=tw7NZ5KX9FuG_oW)0F=eGl-Aw3SDB}$9=@}M)E<9b2~+&D2Bd;!gMV5f6EO1H!=)hh_-f2?@voh+L14LBCwj&mWuH3J3s;N z9$J}ohB47czJle=!}L|4c<&mdWngD^*3H-Q?HQpfxU%fJ;(q@GpJQ9x1_zpAg-X)-oNs zZ!arKKa;N<8JAu|TA{n5%h+yGtp01uglW@ZIlAR(2F%&gzRhNfhW-*4wFwJHcy`b` z%ksfYb6$?DHQv{lBWI+mE3KSJ&#-Y1 zv}4kSv>x9Z?K8#Yo|p%4>r%m4Fc1FEG9RQx?E7hc%pfoJO~*c=p4f#CkemR!gd<3H zM%-e`7#sFk1RkTYT}~_QKAR7~a-5Q{Y(6~13m(Q}$geU>CNm>*qo$j~Ok1eH&I+0; z3g1dCLdfE^ZJDEaVTG{^kb0UE81oN*Z6SY6avBq$eVdX0oUkI%!e)S?6q`ua#WJbv zBBPinbX;R7SFh>UlB?{g*~5T^=VxMgUJz@8)q^H1<69vS0n!%TXAz1&a9kZ&RD!(^ z`Z9PgS?M4Y(FF&&I9tZ_-GuYh%`ISrpqIY|ge#j+i`XVlxcf?0P5H2(*2+Rm zx*8lr$pVAX*XWQ{-ooxe@^&U{Nj-FueBl4X1y+Ml2E2Dbn4vIv=L42~A+>Ok&SSLB zn~RgiCWe0?tpWNK!ynLc39A*r!?785A`7*=LHZz5KVcS>02api7Vt7O2LrmkJ3+c$ zhAMz1uOM16DgC+96S^7#$0C*AG_M&GQH#_X^{Dm;)BiczR%o*TSjMWy)->A^r2C;+ zmLBV5YpcISO(JY+${L2W|He}H5eo?etN}FcnS<6k`Y{$v{GehXvKIoDnZQsl7HR%z z^F~T4`%-6{JiA6@r9c|H_I)J=(E~#4)`GgDH?Gr%Fq^xDaBgHK6%``(*R!-^yPO`p zjM!zKvW!tRcF`sEO-&$F3nNV)8nzmQ$oacplP5M`VoC_wHL`rV0L#q9=~B%4n1*7d zi9n(%Qwc;Gyn5%gG5=6g0YbLPKZ_C+rGOgrN=3?R%%_oWj<@wllspYjEbf7h3RoVV z4|pi~pEm^iSQ8~6x|DVAtrn)lvr?fz?~veaY0C+Hgm?mpfd7wB(})Hs`Z!0-g6vv`J-Ymvx>= z5Xu>>xiQ-W%hvwV^qgEVHDZR#GeG%P47uRCy$&%@h_pnOYY@I%2lSXnfn+|6#wT4qjWeKI><6=3TJC zT<6TiiAGK^KZR80a%r!J6D{%&8jm%xoh-hiUA*ng=X8gJ4Uw zL)RAs`+S6`RlEb1Rl+Zb_ohhy!ETAQk|A~%bgIG!8P!RsM_&A=v}(dj?+&ek_)Y)2 zeXpF9QRGpr^L`BvhMPCrIC2Y4(tt%ajR!6&vlV6P*#e0D`=(xHrGJf0j5+m&!H8Lz ztU}7l30~~IS>_j#tOg4}rrqo+@(6wCH;`#{$gwn?yDYOMQtj@laO6f96ym<>imb}b zA?!K5-J4!_`IrIAs42pdYtPnF+puTUu5EJsUd$8p0;R5!<@=Udv!XxPLT;IEm+3OJ zihYc^N$`H+6-Dl_XPQ{nZgzEcl9>sGWwnF_fZhPCkG-jI(wp8whU6w_g06aZs5NX1 zjW-DfQ6wm6)-xa!!Pd|U+rmJEogA=S^w2IEVC2O>4JJF}DZK+0-JyLjz-n6D4A4wd zVjM;DLa=ltfpLhguBB7-pN;kwPG7XUI!6*i0@iElt7X1j$_R?AI1%c@?v&kR{09Ed zD}tnoyWrZbWpoF43t4~Pc(g$=oavZipFutv?bx>0R*`D`{Eo!yOhTfsIC)sy5!kjE z7R2l_JE3Jl2oDB7;3b`_R`+@Cu6g`v+p98j(4)-{mPN?X^k5B!UsyU845Yvk0?(dk za{z6>dM)1o*3)ecKT(6jY%zYI3HI@31>5vE+6%B%1}3p~ng&_OC9=lOcCr-h25Djl zd?W{+Dem(|8psdiVIqCss!1@W*G+L`M*#m9EDJn3U>a*@#sg+A)Z`KO@e&j2f)ODN zSX!nBX0Nd9amXeB%kRuAf0v~z!mL}&(ijLcnzh^IAE__S_8aIboq3>FGdaS92)kyb z7HVoyAJlMfn+*0K+b``~lAXV)DR|13?p(W6o=)DEhn>6OJa1+a-VH)li?Z$u=_Gzg$TpalSn8z3I4}Qj2 zYJTcLvU9Y*oERRx#xnkJTwM*rzDWxX@xgfC#qGn%nSqYn-2HlZ^k-nthA>s0CG{D8ORpbx$BF z%_BUf1C@4u943z1ceVLZsavDhn^gtaE{TxIda zn5N9=yI>L^#bx;})wV^|>puZjw=*jN<+_5o*HRFKHsA9-pBGXLpL2riCoIa$nY3JYvj$SAFX?3PCM*BeQdYEqYF5Y-+5 zJ}us8GlLVeG+FN$#&U8rvqZG(r=RR9V9n}4m{xb|(>)_r+}3Aw zqO1;amXF1~u{0DT{$3?wS+BXpUPrWEqeN`Ef})a-?mLMQ6q@QSbl+aH6-AV~ z$emyvj3%H^^EMif@@a286p6&#sYEatVWUgHL7ITG0W9r5YKoy`DM~#_b;={vx7LQW zBUA_1=tw=S2Qd7Yn2lr}@^>p5C%q0-@4nA&l~ z&5DTmd_1cn_Hv|BMXO4#e8ana*t8GZTwNMa9jjC#nMt`y<(Mm$D*J*V*Fgf+K{GLb zB*4AIcMtw?z^_WRT|azTkYO&MnyGLDQ-)HLW{iqvp)i|@3U+2?Q@u;z|56_$LuH;` zI`Bf7$|YOKf%NDQXoOcfo zn0#LjjN;wFP3!ioQI7*eJDx9u_}jkRybQ%|Z?gvtX;^5ev1#<(F_CUe zuRE=AE8VPiyZ+8-eCr>yDwV5vAunEf<&Jz?FKt)+Q>oI)S8BB|KnVHMQuR{KO{%d; zAnWr+E@J&{Z|3Y&JPgE6PYx5U^KmVsoT<^%Q9C2a1K;Vm6Aj1PrR>3Ie2MzIA9ine zU&0&BHv>bZm&o^v12Nc6w~9(RABlI)GtFvP@pU8TzUZJ39CnK2NHLS>SZaxZ=Y+W~ zPIk1)fmA}1menYURD6gB_?SPI@`m}~Yi&7o&@o73=)6}_U zueD;Y%lz4U%Ljr1e<&0Qh5g}3gzP|DM)}|+Du5|?&Ru2VXY^;6Yn}Rz<%%#&S3=e8 z%ISI5D;<}bmwuVw&iK=jo*c^Eh5{$4ez_aNp|Q~>Qz%7OQtV2Q9Be^F#ou@;<4F&T z4f*s`IoZQRoV)Un=Ts{=Z|IsSQ`bb%NL^~O4^5#iqTU_StSedp;Z<6yR+hPfB9>`E z9a1A4)eGg`=tZ1ust45bS;=Lm2P`$A&qN$a_E?=v1F}}#nQXs(`24Ic?W_$`WJNvS zkk;$PFP{&EJHu`ZU46Phl<4Dv{_o%QMch7A)O6JU=k8Kj-Dl4RYxw{D&rbjYzyIZT zsDSMsj@Q<{{q1blUch!a=`vb$s#t5?M(KJ*z%I5d>QaEIOLteS<>xBa`uq{!LdDu; zii){WgS^n{%qH?xj(y~OZtSLTeIWE)ojytcBdr~I@9lC^5p{vLOm^oqe&=Eh7`7py z^mkAUR)I|&m37TaNX!oZl~^wKwo>^cui{LjG?~d+pRMzxuK20|1eW6%ML+zKqht2Eb|=@LOHECUhBq za_sGCs#MM%7f|ol@AjF!Qc+;Gj-FF3WJCx)0zShNQl~F8$DLPnTqP7MlO}x#gPVrExX9#Cvzy-LxrnZ zU0_sn?ug=m!#0nV3WhpKACPhoTRM_aRkpvuA(uFjR8}dP0pf%P+)Px!HPXf=hgvZ9MX!ZpC zO;U8VO4MdYg?gQ4N_BG;dJQmG;o;E$sg#i$lGQ{mi$fZes;LI(35UYSI@V}a7;t1i z)hcph1SqkrdETmGmz-?qg_zz9?)JqlL)rlb2IMOoI0Y56f%*QAxC0D&_z{;z*b&^t z=uLQ@43$xxHs+X0^v{TG9;2SFbu*19wC=#857?stRNZ8HLKji`8SYV-z|nx=1r5JI z-ELJz^`kaiidj?43uHiElQb4%spD#5TOtzDi#Kb?AV945gEmf2cC@NKz=49IxNNJZ z+&SwrTA=win?rypy2uG*5Iz=Ntct(^D2;RhMzIZ1?EvX!n?&C|8OBJUUCTWU)}cr@ z4k5He+o-+13~4)X{$?nKTc*YnGCB@iVR*EymK3q!=8l^f)ksk>&N1q|cflWoE7ZRo z4X9cz!(KKY(<;W7^OYiCAXu`2Ovj)aAVsqP^w9)xt!?2|s-RBZpbU+A$K6ma(Q&HX zzPZ81p!Y(1iquln^+g{rX0lXJ*vAfsGFBX%0cld%ltl%q8$OcNh-85qd%5`y%jQNAW7Z#vM*VDnH2NJ5 z^~u>cLIH@pT+Vr5_0S(p^}*O^@FjH@9L<$0u~G&1p<`F|f$5K~kaI`^PO|P zbIx~`DOph7ye^}~4(fbS`E!mGgp=H17UOqNB-Rkv!gwA~ro5Psrqdy~dxDCdA=NRl z(?NyPNCKWyQz#P*VVje+z_}XuxrT9ULs3rGI?e>}F!F5A8C}y@E6p_xleGyFz$rk_ zlkn)-3qh`RQQQdfdGwU3fOO&YvFFt6xtv-u&FjEhnNYbS80y?R30AJ zCg3_fc(zjAB)HQ~*k{IbP)uGK5up=U=P+J?QpwClUsc*RCS-x*RpSRUVy3jyz`PDD zU^GBU$u*j&3aVIQs2mO>G-%#$H=6=XcD;zC7$i&RiH;&$I(a39EE| zWMBa3|H9(3<|hLei0{L{QJHAOm7EyAgJ*JtWSa_YIc&Z?>@GB+F>x%?4dpuLqh=U}wbQ0#UVP9O1%ksI?;jd*l$v~K5Jtz)*1B11RB^%g(kKy2FE#ofS4pUBBhI$ z!0acqrMDJ?5t;NDl2c;TZ4{Or60{m-wh#i#4!P z3Inu6ONRZ9tqQs&5@DpuU_kxS(h(3yXi@+t^lA_e-UAIVNYru9m>|>02?RI-vw-4d zxp?z>hy5gta6|Jzl6$mUmXiXGt|;-o3{~5@T7Rs`g$|_Ii5V?^2en2LPGKfgtMdEKrqY* z?|1+OyX;|eU|jGgs&>1Wz6T9p&t}505mX!YV2RZ|$>mHy11>{MN2~KE zs2W@#x3k2DH4PIwefj7CjRQxnGP9llz*p0SHR>m;7uKt~l0o$KLLCF(#diBb6@#tW zb`8S>*6ipJiUu4qSWttRIO*}q#oiHUAKRH-Dk9$r!nzHeLx(0X?1`F(IUTHP&62m+ zKCo2-v15VBfL@?pKm;V=Xj8j*5PW+_+oU@Jm6sAiuzR5nB50uhtCy-FT$6g57NRe+ zT^E65;7~*KN46;==6WL?1_CUm9p#8r68=-;TFMRu#dwnbXUYmJ`8)I!^KnCAfrVMX$ht!5eHVN-1p9@bcdZOV#Z*HaLpL0|!-03zB_!Rx#=mj}^vyNa=Q^c1y6 zc%s%r$nQ6o8l>%^&De?9E~PwGfw6E>*J5(d%;C_MH> zN?3}IeOY$uJbKqywZ~{auy?&w@!|64OZ6V6czLNkCOl?G8`_U???DZS>xwS_KX$^0 zWB@zP|E#<0{O`9udxgJ0{QQdyfxo!?m*`;fpZoa1WS#U6{A~)%zrY7$=p1JnEQ}w* zj6j+Ps*VfWKpoCt$yF`i8Yg9+f|Go1*nL2vaBb`OTR z%=F2E`i*(4%mJtS2fJTz`t9!W4~Cy#!SdYuP|G{XKUN2L1nRn~Sh> z>2|mWo7nOaQV<{G9V}UHz;E?aWXVb0Hg^}sQa&TMd6;Nn!*o9ApOli*PG12ao5o_HusP zj0fwP)ht%5#tYfj>f1%IfYuuxMcQYWMa1wG>bE!uQp);CViNv*;?R z(`Ywya#b01JE8c+`szjvDb3ScF_+yhMBB^$akvty+}?E8u_c=5oXK!EE#|Ms*=(@3 zz6;InGSV=8TkTy%8`t?@BrSI41NkI8ZFKK%X0cLmqEym}@Ist-dhH%HQMpu`*MV~T zI5a8TRc}MLH~pSW*`dA*F2bPySGrgSzwWJW!cTS)-o9jj2wydWg9e8Ea-~=dlr%M- z&6Rt%t^Im#c3c{)n-_zWbR9|dbB*J&6h4p1$+JeRP5!;j*?DQ5s44yQMRIU@wwmTj z{lYjG))G=EmX7Y1ZO>2WEcq>HK6s2oYfZ*zC~Qn)$n-=1}@;G8a>4c#unCKY=#mMwn5@lYn6MY>c` zPHyoNwn*8K3!Zz<<#D1FL#T*NiY5<#ft_dRBJ%cgDQj3HNL#+ZmD|(UvbsDEt>RaS z;(4*r%`^(li>x?JE=%!zA)l#tg7awiv~z=KF5MS6hDEfUof$49-*GqJ{Fr=s=xo|N zEHF+parVrEZE!^VDO`VnAbuOFeC>^FTm66KsJKi2H}7))-@khGqrW0f@Wan<5(Iwv z+iysv|0jRvH`CYHu75n$*O+lI@ISWw|O$32d}R}R9!jc+*p|LLnA{poKK z=6}!U@(2BYnF#lfikxuMN>epGK**xQjC0)6874aBvWz1pYfmzqpy4|YeWcu??#upFQDW2?wxT~g4#K7m?wB{J9!W2BMSU zr!>qb=WdsLp5W1M$`&m8!B{~QvQb({y$C;N%dHg;-dOq|^R544XlkT{OAP`sWz`F6*CjL{GBKnOON4 z>)(3N&BRxC)hG3^!@h?s1Uv4w4zuOtzP2!v5qy7>1AzWp@I(a9b|(mMF5PzPpAB8U zJz_9zeJsv{B`Yv5ZA%XDJo8^P9*w~Kmyn-n*LRS{7Sg6LPP^q#+Ds1cSj(TIyXTse z?4KqJnX35;~}&B8FYV>%Uq3XrRFM@pODS%A~>w} zvZq?TlIh*{2F=jLuo}`%Bcb)`K2o}#-Veu_X{Mg2XA*@*uURdL>wuK0XYR_2s~dTV zFp$qi!_jqU*{CgUS~7|~UWF&($qjt(!)hs+J|CP7BFU@4_&TM86RUPhUbf`C7MisZ zxBJJ@L8F2Ap3i!6cfBr$hVz`N$oaXLk6f=Olr;FeVENnI${gOx?FF-#Tja$2JwFIVQxK;Ztedbwyva|G1e(5l=H zbqeJ`GJm6X`>AfYvzW|8X|C;O;wQCXtC>2!PHK@(;%5JgR+la$9 z+2ad{Mi8IjVbl6;*kW+ur>Iy}{lelb~&Go# zwc)i!aj6s-Pu3@c%`Op|i>Mw7X@lVknlc&7^hj7iTf?FP7#QKp<4k4ms0Ai8cXoLmXYL*$UGG7Y?B&JT_EZP>oyxrJl#p<%8 zTP**fFDUtNvM+L4fKLMaplBvk&awhWyb9k-5G7eu5C+&efVTtLh;YV7!Fb$Ub|8G^ z{1Jm0n{Ts2>-=Svx(Rzc1Sm=`Gyj{6A<`SgL%?tc{1%hP)B`nZ_PuXEQ| zr}O5bn1Y~PpQ6mVkQkHQ?m%eN(!$>rDZhqkrSx6{SP5Zxt15`2MX@TwN{4_@-Ezqd zfmoD;Zl5$x*;ugcfk07q7O=R>I8tpRi%1g_r#}gylFkCscT83&i&#I2mEgm>-p$ zMUM4BXJNB!@_?IX2&6$Q=Ql4)?u2_v?RVa`JLj^Q%BejsEKBSe%!2-VCcb=o)Sw&d>^{Bb zrET^*o~`Ps?t7r>B(NW4ppOl z4X<>(+K-ivwah%#I=P%oF2?ezG8ePSNrTz&4BAg77JYTDUG&4XRtt-#D5X-nxF759 zCzR&>{!RQma2gDa>(y0p9XjpaDRSU4alR0{1_J?zS#&|cW;xom$TE#$>J_Ms~<`inH8?%jeIH9U*<~LN+*1SCKh4oHc<%= zZ=*{8Byt(6R`Yj@6;jg_vYp{cE-;w{E5pdJU%fglRgu^2wo_f)ho|x)uSv`E6G#f5 ziR)@$8INWY^V@8=TB%0o>u5W9QCk#ml7MuG44*Mkl)_R(l*M>hj)-wlj)u%M zpAQV%mejI@EH_yw6C@u(7Dg6_HdAsmWK`ifR44_D(8g&!(r~GCz^~%K9zruoxFm~{ zkgXyjUV*CMuyNraT^}T}VX#s9y!v>tUGiIZK< z$eFQAx^o&e$K}v|cNwU))q1nki{|3<`F(p?s3env{OV*C>?oOfV?8X?)~C#qQJ=zA zqh>5S%D1Qes+u69=C_oibesIY)Bi7K+~xMaqgOvVB2Mtb&(k3AXMx`)^TR*=W{2_3 zzA-c8{q?2s42i*wG@jeofvJ-{)wS-^r}31K#*-)Q6tdM!rkKbVC|&1qCjGIRsT8j2 zWh532#>2#xyi$EWn95W5_2=wF9)b8o=*nbBfl-7UzYNaLv-+5+uy}H^Dbm-F8HrCdWJ7(A0wX_zY(Z=z9RT1wN1SF&Vl>%S9@6$CR+-U&!f46jLCNG}NasGsTriQDKN(0GXw zp!grD)!6WPl27;v!h$y-^C3)xJkuz{#tY@IfA1e+Xe=QpOApo5O|p!J02PQ$ViWJ%I2wqYAvAxmaNHmM|Vr=0HlFh@2jDr*$`&!j#?aGt)U^`Q~ijyf*s8 z9)&c(}Y^Ulk#jbRHv+j+?0(BoP8Wxpbg3<$KYch z+B=|P_y7Li$k!JMeFr_H2c3f|raEO==?Im{%@mB7&~q@Kg9thFISCLMV0VS(ek!!s zih#{wDwffRfJK>TiAzt%?pAV_=tNd9Z2o z?Mx?FJK_;ckYIk*9ORh9lA@&mdX!8DGhu1%SW-+{Ed!S{Hm*ERF)n;Vdk-c7JhFhj zeLoTmP|CkbyZc_tItF#_~P6^{R3M&?MXU5h?gT>6v`kv%W zBxIi-GNsD;y2rj__Q;3F$}ZCXNLvl%{KI^J?c<0xXjAI2i#i;exfEb9ubpPE4*}jH znr3A<(FYZL6n@U+G#punXRCndm&3wdj+i_Z*n@nqQ1;#l2?~lq{!@?vq-_Mlycd1} zwupC)%9ixD5M-B;yVrGHrc$l+#%N%>WxdhFRimbRs3pcFB&WsGpYu|cxY+_h_EbWt z1L5;fC>)L|@3EzKScxzp$i~k8A;eQ5j{1;yd$2(6%aGpl=X%j9YRuyaM+e<#H{Qh* z%?{Y)LNs3b#Ks1ne6V^Zi_%8*)>dOV6r&o+!q3SBVC&Y=8fyc$%^8|^g97zqbR#f0 z@Op$8`bMEsoMsa1!aO51Fnd{50r`F%(=y;@K)OqFBpmfMN04X^AdI7__Y56;tQ4yt z$%DF$StDf%6_?L=5zso!P7BPO!i8rS49rpns~I z0QIAXHZ6c~a0patmEn<5dX>%C=uKqbAOjNGj*J2FH0HZ&xow>;zJjfMk1LVz98on)Y->WrONVW}BeGC{2?n z$@aP!+gJwH0}V(L_63U&QFmo_Ah5^jrfhE+z}sw@A(~@>&6W|Dwb8PHmwnaNGLg4{ z!ctR%`cM|fEczY}{*HwcxU5s30X$k&l#D^Llg}ecG!zpVNajq4H%S#pG-}Q?>OaQg`*WEuwH{% zgS8?#DQDlTm@Y2B@dl|}4)^%HslK&&iIer-5i%vx69z%)g`u}+H^v*R{IS}`OY=GE zF$4k|DUgxe9i$K#j6tGW05aRuKb!Q-TEKgu5U>8bXY4L$l@cLi!_kjCjQY{$cy$)a z=Fk^E>DGki$oxO0w6KmCQzQQ%u3u1z1Ty`wbY;x=fek@Hrgv0%80b2iUWcbeu1`%; zHr9~;me5V$Tm^Z-rq$%mtYofeT(^$sRZLg~1u|9f0t;>%9x!W)A_Al?G7W;spqnhQ!(v1fIvNmB@+9M*Lx(U7fx3Ya z;AUX3%VT?t9^8n+R%R11h|)d;(2Kh-!=7SDbh<$RLm*oUGo_g3lD#8y^-@o<7jNyH z>8{gmQ&(KM#<_R2)r7FMA=2{(6Movdcf_ACG-c_6W!|yHb}-?a)YUE*YuxgdEPsEgzeiTMgkGD_=D75WT_Sioyaz;Gaq6T z&oERYCjL{{sdbCd4l}j6`W8QazO`C-xv`qk+UvE{T()Z6PHh|TbIjG>D#no^oG{5^ z>LyM7YPgIykvYxSc9H}A-!fTdJJ~(Y;=FhCSYx?vE26FLAzQh1{^kTSawxypv~5}= zpKsYV?X1r=Y}-&9JkqSh^%DCwGf&WI=B9bu)|(wA7&>?vxA%@*ws7Ynn3(?bEzQE( zPh$e7a1XN9dsx72lLm+_L-;d|;AGMML^g2S$P82X#Ue0uwj43j{1DZqJTBmF?UM z*iST_qiEPCv7AF!BP0Kh&1Gbjkl6XJ!+!qA9rFPIZ*GmVBg3F>8$sP|S2ypqRFd2c z)gHDhNLhwzfApbsl)`}(3_RLTg|h{DtY|;|5NJW-#~b>KN5@p*luiN{ZB+j<@rDjl zkqU^UUha74@Fv_l+H!+X#BAOi9UsbMLUmo4QfVj*8>2D%H$>k#p2wv8*sjfEobK3e zc|b+R1L&YVW>z?=^XJ((G`KCGY$!_19qz%ibQ{NKTK4p+y10b5daA7U&d8T^@2EirQ~sRi zD2>qu44W7$SjRN5N1ak3(NzjI6A+Yd=fO-Krvo@P;V6mbjhDO*Y)~#bg0w2;BP{tS zGnNHlCL`r=aGoqzm%-EH-ZX)qQmxU?GRT9-Oc5y*^6AY-jc3^9VxrMK1(LAkK=>u zTJpGgc{1uq@pV%!_Rhqj6e+CNH}q2Snu62L`$1#_9$j5lXDunzJ3k)Hug?;zVlB2R zrtq6hM&kvI9v?R{t48&VUQMad-sbO{*Be*+sbzGXf&+9a8f&Bzv0)$^L%>2z44lS~ z(}j30eHOnAoX4l>^LQxG+&@V-_fONk{bpeF=JmcBxZRJX@f)}cl+w#UDG&@OX)&M# zBDfL=geu3%ieQ&G?PO2Njb-92pH7|)LI3m9bRt+u-QOi{2|~$4aFluT`t1BHv<{uu zCFwMsPpq?pLBFp>Ym3|Xd2`(j1i~lPLN$F?P|maK%*`NNY_xi3qpQ=i?0KzSX_klK z`;%1ayiyL|BAbeqDTf!$i)rXg6o=i!X<(YUt=yH>WF*i}6fQ3t?R>vBYj?ZFR88&N zU&nJ&v)q$P7iZ!1c|Xxe59MApyj<106dDpys>OaLoe?MEUF0~~%B|DWzPV z(a<7P+Mo8clBivU<)W8+0_*0z@MZfpv0p0g zXXf!UWQ-ZX$0<{)rOwV9cWpH%m#%{Sdhl+2J-CbY#d-%B{<&77H!vmr*$u>qqVatNq*{6_WDR*71tg zB#}I+Ho_Toc+%_jh`t|rl>8(jHCXp=7LkhW*5yr!2>+=>nze|O!cjJV-@M+qniKSv7cFjTGiHsT;5DhqK4W;r&h_6ewoS&%oR=+I96R3A#$GZVQFX zaD2S1mPVn|K>4_F(wJp0bJ0N|A%>3I%KcUIBsj=wolA8VN)!_7W#Y6KmDTfk74*Nj z$`^BsEM_~oy^Y*gm&Kb|^0KI&^{R>CY?Z8bJ5l9!y5A_|6NM|KtPG~9#&WTU+>X_z zI=v`N67kgN;-niLOi~w#oA7unCEvV`4a;KkvJ;AoX1(G4zIxh81mt^hKF&=F_3ODL z-?i?S*W-#jS%voZD}hAqVsVm9&CiRM83wV;C|e)a7H8vB>f0d*L$>&VAo#A8G6^Cr zUXSlLf8V^`xEdVYOfODC>g2Yxn4Jz*;fQwM>Lw*Ea1oy0^`c5wjRorBK6Z?NI619o zSG2Zl4?;==V;6D}!(GNAhV`lmVpwI8!LweZ0h|F@HT#OvTO}Lio4b7fY}rp-%@FN3 z$c!2%?aN-UIL`%Zrzi5X-cd8x_wCrCF+VRWx9ZKbcTtPnEcP>N`F3zu8BYTp@nSey z7aGg-GL9VoCzau>xr|{~5Vh3hNnzaRoK@SQaC$fmoff%B8Qdb1ZiT_l>LQm&Vs1&S-AoSL>~wcGVb9LI{O;AKQkwpR1w zM5P#rUbeO4Nv>6oDU(WWdUMs4yOmX7F&_>BwMjKMA2M2~HO|Y`({So4pZ~PG1L*%1 z$#_hRGvwbN2!azt)g*}0Ds>v){C)F!^J=(UE+?DSMQPmFKVMyyJMq?He4^|x&Ijd4 zf7DIQr-9S3Rzb44ZtH9vn6#zJ-89^lA!5crC4uE)JVg-_gG{y%Niq*H%@EQW*9#EI zQms$2nf7IIaCR3BAdom3%r%O+&g61fz0=Xz35acpWwKK5HIg@1{b?efx=DsoC!OM8l*ptCx2N^9;xdsK=ljv*ZFvY`x^}%x z=F^FJt=7A*UM}a1fY(NU**dOeY}zF05Co3mK>c>m(8cXeExPh;Z6 zHEKjG@-v9<(Tmege5DT5VQ1RWiWlo*xL$_CefJ`%os>G^abLPAD1}b>^7JOPu7&TF z%YdR(7yZEXezY9B3?s>Ybk>!lfy+Rm+Ez}a+hTB$pD>EmcP|LiPujnDMZZUif1w@q z|6=e_2?h^>K{~wm1tUHr$qmL2WL_Q6yvyVN{I7oGCr|NEtp zZwUJRr*?_7X~;V@Z!ZmdFoFB0Tf?4Hxq230W;sD__lYchGUvxPuMhPflHVL2CUoKA zQ@EB`u}cDcOIWj?$f@MbYrChx3%p9~3D^BEbSr`TD09MwdHzAa5)O0)ftNrCB3curOAtSmN_|Y_GO1HR z^o#Gj0yD{3vzj_jf6SLYrc1Sp!pBk>X{cDRXArN{D%p>fM6G6qe>&2x(c7C<7akYO zDFs9RXxJ}?B!5^&yk!n2BZps%S7F)T_rnK+c~bkiKR$tgOmlb)L4a^0S|5JVsC~hr zO|$BBoP+b3q8@(1AE97*y2`(sK+}DZ;T-$xi1egE=^36VDqbUhb>F;y1Fdt2`c?DA zO~134i=D6AWC};pNzBEbE^4b^y)gJL=yPJ}Z+_lFZeJurMD(Swy;!T{IP zCSj&=k9fcL_-yWCTFS;#nQSI|CdH11n5Jv1^sz#);*lz90bv3 z^O>IznmzVt#OLo}bkoD#48g)%A2<~#8=VqU5`?!lIwD_X8?#~J!UCE4RK#d9PeF1; z{NmB0`+^Rzz8KS#@Ua8kjly@{jumn;9B5gfaPQHB-_zfJ1aS2y+lWzvdODaL+NKniEzlqE z`X{?r3h>#<+66>g+17~o8qX-_2^`OSTAz@SJSRK2cG!knbjL{y{X-)GaTe%ufTSp9 z>J(Py4XJ5B|F1H>b9hRiZrOq3WTt@1BR<C7EmV_{-Y~30bHn@gA$!E6Zlz!Ch2o8Lw=mK!vvPRRD$MV4$J*^N4q+CznY&hN zAgT9h@qUpl7RuSCIiYq?{*LXAGGa`TOSa`LPyg?5|gHwc1~uk0;T) zYNMOgu4C<9Gk7Z*c>UnQ@cXyFbPe<87A(qM?-|nAb(*tpw zrp}`K$(GgiPIuJ_8D#+k1aKaF1 zx7`yEW8~m>N2v5dX=?V8@R~lJ&ZkQ}vo&7>Ofu%$f%`dHf^3cWFC&I4Y;Cj9_rmm7 z*cXG8UU63{B6b$%ooR?x)`~2XnsQ8xciO(V+!TGGkQnu~g5j1=mK8ZJ%k6F`()qaO z(lkg8A|ZrC$rOunM2dzR^y{|q^X4H>lo6@ zK_O4yI>Vcr&f*|dV7C^F#pr+wZ?#M*{pR%sDq}U^!w(mJcHHW>vFpG*(D5yd=6$Ac z6gVP*?*lPHxC+=#X^lrE^HHheONf~;YCI0`EUQITI7n@eDUk{O zASfd)MMb|Zr1{Op{SF}w?{CTf6E`aSc(!sX`mKMAf4-|fzyJUI?SF#? z`uF}k!1ZpNU_S~CcWA=k*G7Dx84zf)KXScA5>{Vk(w_2gx7!8WO*%e=WKUR}48fhK zpS*h++0%vkw1b*d$b-TnomFHC<~w|oa}2nfK}JyL6L8!9V8*>ePL*{F=i+b`K%4|q z?hrQ*c7x!06<)iH#UMe2x}MF!p(#$+;4(iJe)2gIW=T;5?y*ndSPU!{$KEXR9#I1n zA`a^WcQ1knPQeHKZ5DZe0pNlz3SewheI9J=4j~NGi8@WH*sn>?p7`~vBoHpKYJ zN36Y2kw1gtijji@91&Gy7Rr}?p$esl0-Pxpft-&8wAE4OpdJbdMnFH5O9Ze1cp|lj zxDKBEL+_rzieMUKTp{g?8dfGifS>=Pqko+eF8zg`@42cVRCwa6H0R{b!>0Ba)rK)s zCIG>Cfg~vk>6k6%id}&*FKx?D8D(Mu<&)M%DE=wS9cdnn3U5kZ~U!^y)Z%3)F7NW-*MKsSVQFiL9ISEEED(+M-!+xzu}f>KEKiJ=HFkGL52i9RI3 zl;mhM6b*^7csL>l!JwH&%G0O=kW2jjE<-M0Dm?#jQm;|$60@YFQ1)bq&4#dnt{qnl@skgYT{2*7tjQ z?_0V?TW0U9QAekGup!~ehcW0I;GEBVsGl{sqYlkY1QHd-&V8up1~bF5#WW++W#ky@ zqBmcR92r@QI-Gty)~0|Iz8!gWa_MU%nB;_bLu4It^g}0v8FZ+0dJJAZpMCgxH*DYt z_5uC>LZQ)7l154g;b@9Q{o*d7!H`DX$G(~)Wgr&jn5H?qwx-c-Jx+T%L3|I7-eUXY zTU)?rM6%+wzY@Mdu7@64S^1VKCG}E&CQ#5XAFD0;Q$oP8@MEM(;#HVQp}>zL09!`X zultc-&~x6sAPji37YeQ2-HU{1U-u(=FXW(V&4)s|eX*Z;^y1NiYbxUQsd#h}_v$3R zIZSsa9w6O56EW!3i}*0WD4P2507E@A4~U5rx4e5@M2`vYn2qGwjr6Izkv+RX?E3?= zkvt~6V>ThrZak34h614f|A0~%Jh7{=XIB)_y?cEqw}JbS9W#n}cH@Duwi&V6cuaan zH&M@SC@H`Lv+)?oj&3~JfEQu7XY*a`M)KqY#MK`H+LJhev|n%ABsfd2nO*9Q6|Y?D3FfF-BcO*`_F&;ZHxJvIa{ z;Jauie8Q>4T97Hqxo8{S5ZjZzvnYzsJ>!(Vp{)HrDoZ zrhp+)IV`$g+YNNaM? zr#w&?(}NFor${8hC#?_cb7XzO&t{*kxua$NmAy|LJv5Ll_s#d2X~btdMd4mZ#$wIG z`?hWL(^(VgOx8dRKN%JYNCQRi(tPyOz&qhRVa1`)- z+4Ht*WE%pQu^vbS>Ygp1sq2QLJ!D$j2KhOd(FXkV7CmmU`N17eMWRZR82EQbkEqnO ztw#`;V!cb>9B-mz+O{|KDKenyBxVt!PI|^OtB)3#4b*V-BgMD(V@!N^Pv%ozePMwck_PD!_O9~{1U4R;H6I_Dct?;rOfz2Tc~bcD9dz5=4VH=~q%Xe zR)AO54u~>*z7$^z;~Om$<`U$Sjij%o#gCE$7MP-)02)nHfrJn1`jc$KAVJTijE+9@?4^$il8^D*yRhp2O8k89j43P z$1Zy7o_9GSwkZGie@qIy(W)u7=>n<2aF|d@M-6N-_}m_nd6^u#gd`^T@g}ray_aH> zERH@dU`v|@6@dzb(-XR6?ePeP&&AxOyt{ev;pROav6@HUyGAd19eq1T@9~J$Jo?_% z{KJFae8Os64u^I%E`~kK{I0QYK4LX5g?El!40;>~yPDs8#A;p)#deL}%c}2eee(&k zai&T&QBh>TEV>L)*|l~P#Wo-~aO7f_s4k9=iZD1YFIor%TX1q<&vl#Q!3f4nt6qbX zqMe<$Sf(#$lc=Gp4wk6Q0^_*z&I~#6sRD#YcgG66um(-kiC_;nR8(h|NB}u?Hmgos zp#MLF@3}|A^(JzoQ@dvUiCy!d5#m9QcJ}J3(n7v9?0Cf*%33_4J<9CnvcDtSKJ5Zy z#u&hO-ZKSwn;2x{jS-upk!8B)ZaKV77?|Otr57F8up`mV<2bw6>XqYaJ?_!Ehj)s* zb#IXmqN8~}!Nx8IJ(~AoM(*}KCI~dyY~RcM>Lm_1?(us$UQxvdCz5SK-vfl6<3|++ zE;iT}SM)eLbHQh;AB=naA+Po~Y=#@-e{}m{ul7ASs(bt%Q}l#Sgtrg+|4$qQKs7xc zSOYJ&2+uyEUUTr+blh|B7=mZ}F|YPL5d4|#qwee`4ta}O4i4evXyG+}$%_VF;@aZE zQOOHOy%x^}A1}8G&++RxOz;%!Rk@zEft?x@*S6t(lz1G(YxUTVg zd1!dSU-nwxCpxQu{{ILS`mmiE1{vo*BYUm!OFUIvFba9CffoSWF!Ep{UN8!I!RRHv zDem!m;pj`8Qe5qaz2N^6ZxmPi5ij_>$PI;y&RoDEagz?M?c$bgKlWhN4o12uYGH|O z?Z+MM>jIapeGgf|+5QHjaJDZ79UT}lm#qgciObo=#vEN;h|Vr_vCH0tha;o2i;Zcz zx{#b*=v;|+0s8;@a8ZX1!bA3YS~s$@8(jdi&%{GEadfdM9Jyu^a(1EPuDuHn3G8WI zP!)L#XHmnOi2(L4JcKZ37aP-b%_8FLLZ@(h7aMr#;(nW(xT}k(vkP4ivv=XaKb^DK z7=x>en6nF=58AufKz;Wt9_T`HlAj@t*}Cu&#+bl9M(JiOk-`Lv*p}Xzc33 ziL{1LX7A!5q}`MmTwORZO4`IIFR6?H3iSV<8s+>pMF&?m&Lx*Nm)uJ@bF9avops)0a?ej4`9Y+yvpw@F*sSVk2bv3EiM^t0zY{ zdlX|&t+FXvo`!oA%Fb2bJ7khun33!DQgR_;dY(WPFhI z4~Ab3mzB{OKRWvO(P|E5Iz1ouev%;?0iFQ7KG1E$o~B$rCgvsFGRmXr>?L~Og=ey`ur%egvlvR2Wa;S7aVsYR;)s~>_I4R3!PUgg|xm_%>1 zrGNE6e~jiAk2d@N+>f>lg9f7)+ZP6A;;sj7`O3~WZF?Nrn!{G|3qyrw{I5^MXSOPF zBxkL)v(P6hHd~Z!ExKa`?JYVaJhVmQzAGdD*_bSVrPH`z{qA{ ze74FRTF+)4@k#Y*~E2 z(l>na1TY7!O?=gC@2E&{>%_`aan?6nyFmJ!b6--?4p(3~(B+CDi0fA0&=!Xr* zgsjdJfhX=d*`^Z(^L>49czFba7z_t1Z_g5w-eTdu{N1|z}`Z#9@K}Y735XR zZD}xUa6p3;CZjJcJsre@xPrn@l%b4YL6a(eDW#5%75b1E1olw0ll8@OaiEyFGs^~F zaz0|`g6N*Kw&l=$ds$I>i@!4C(tB_#3|H|P>L$hNzqc(IUk=;Rtw<|it``4htrkuF zEiUo}i^fED&}(J+V5YgCK-QYzdn^+3b#Ap<@9o>GKr-aJI@;o=-PS(sSa?j{6n1q$ z==GI$E@WWdc?8(8Xmi?+9}RX(aCs!;(YSr95NwbK{r?xt2Wb=fGOdpposo@Y3$qdY~=OCHdCYBS5qeU^IsODpoQX3(bw1 zZXPRb!vZHMa8VS#l{`Yo=Cy5GBR8|c+?(Kf+zE{NhrhR>Kl9r%7l41W(BH+Zh_$d4 zpeV&Il6A36DhJ3YrZk7XC4{T@^v{;7>?!MEw1wv_Htc4^e6o7bgl&F1I3k*KMEBX4 z;_rCw4lH8QIRC%`W~m@8XQ8`~yNxW{r< zE_~QPYbPPbT@4DNWP#!Mdko0VZ((;KerqvX5)ZvNAL#!dY>*n5GTQqDh#85JcRt#3 zZlpFU(rJwLd2@5(WMcRS(jK5+G5i5-m#|x*c{n!1PGrND4@e(m$|uZ%62ivl?g1~8 zE0Ny1uALxVF9Q{tC9fdbF)3tN9XCfDfJ=qSZ<^PPg=ocEopzi*!sLH})C#@|Xv^63 z*q-Ltf^cjOhd5ZERe9u6atY3uRVNktUtt5G%r*NL`_jGk zr9S&-4CYgifopK)=0v7Z{Q7*bz9T6r|2zWXAgQ0py4^bzz)oc3wNIFoMam^*T+}lO z+Bm2#rm4-&U;+C7uaM%Oc2ARG4JHg?jBlYQ8@N!!!2qAJ3$~`Cj!V>_XDmg44%66q zwTQOp41#Ud9z)+??CuCrJ9|f4b`HP6-n$C@f!z{&B?By_2hrezjOrvbq9FdLy;k#W z@Ez@f_?Q0rr7yjNQRGo=i!z6U;pWX0M`6QB8f}qH0d*M zF_+#j7%?l8ok`g#Aqd@@Wqu*fYLEbU+TDRVU0@884l=FI1(v4sfMvEss@($(j@$@? zLfltdoodtW9QK^?!QFVYea>jhs42pdYg>D%W7-zBb0tR(Vx1rtsBMib-}lU#75|+* zLv$^I#^L4rcd|A-M~j zpsT@m)Eg8-<4tOVFcKJKjSK)qur;K@esiY4PL8&G^w0r1VCcn~nyucEr}R6J=sVJb z(JYtZtcEN}iBS}-3$>*S35-H?aV?vnKMvg6IDO%Fb&4c{L|gA^thV*`sS60Q%BtC( zkJjuaqc_n1f2}qLnsNZD-P?wDXm2Cy?}q@N6oZ)#DfSuY!*R!#n$n4QqJAOr`49`KS*RlCRRx@#VPI&@WL4tlov!L|u`njY-I@Ecpl zhJh4NLg3j8Z4SWgFW*n^(ALXs4!=@^gS8ny(FFT^vw~xJJnaQgl>tfYou)w+e2J{F zmU>6gZr~;c!$)#pOK^7^X}~{_hl%8UyClJgUN^;&9Rc+JC$4wP`G+iBF=pLjmc~$&;jGgx|4ez&(l;@}} zQFLNxGmGfY<3C~{N6*Qq4p6C2%BcSMiEKXUzG4VAn{3s?{>HLNIEe{^chZyiWP0K$ z0c;G`O;IK_p)=jD)+rx9hnmpVd;8gnjhZ;|*rXY(bKQSe(zWz&y5!A0YI>g)>Okkt#RIxCfRO^-d@fE^rKR=#m?(q zfK=Qn4~#-vp#N9)MIs7dDB-p}&?XL{+O|jTu*N@rE9D!JU5a5eD15NlD;S5hC-B^5 z@x`3h%;>ve5+K24yO!!uQT6&yXlpdIN?yJ#sgbV(@e^u3s@LGgs2C z^jOCb?5u))^-z0srGwAv!(iOASf0U-4ut-ogF1xoXH%3`VF^~*5!(B^fi`FjMl6FU ze+2Nfd81hdCuV7~-!Y8kE70KV`6GV1^xb#Plekmo~y{C}V}Q(cfF zjKlcz*a}xv0o<#?R|sCn+v|T@cR~LD?bf1mM%6aGbSxEtYb$WRBV#U{OOy%zRp0>S zRV!9>)`|_fYsH4P)QT-4J*p5Cg{;y7w?VB~F?tZf-#7{(g{g>Lm6xtGDmRT+qO0aB zj$4|eJ_`B@--uF_$}sswtD>X(=?WeMzPc;Kiiu<}k%eF+KB`148?~@G>KGq2O2pRc zC@T5vu@f6Xp{dQuI%~F~h|&nT6YP`W3<@=Gst{=Q$D1-5LB%x43)K6ytQ*^)Wjkb z0=-ZfW&hytYp=_#!(6<0e3h=2pVWf~uG%`RMA(7D zY#J)qnUziTA%Xt?y9rQ+$~?XGpoJQhOSYlQW6GRvBZlZsVgL?M3ATuhFPN9Mm0*jQ zbnOX!kNL3ah?!VI0x6&}!Fj@IX?RWJ~5 z?l+o|)BE1>YI>8%*6xRy_@7bJ1R?7mf@EYWlo0z0BOp zCxMHT)OBFBiUcyzyUc0ic0X{ROKJtNbUnIk%uY_x#JUp|Cxt>+?yls?xG=g}Wo}1T znBw!6iG#+yz0!i2Yhu)<^js~x@3~-GSBvC-R+SVhMlWw&G+Hn4w+EICP4HDSxMwAm z?W}Kyy=5=kZHz|paIw6XXT5s;CRr*f*TLyuF)t zNUTD6DHyv-O!kAhi*w~9lsG>-N%bz5t(b@G}*7_j~B~p)ZhJV zbSFxwV7%B3&DC+LIH}B(a6j9tsI_7&IlRnu8&Opn#V)1ztQ4LPE96LFadd38M8Ga) zuCtSOymBCw(4=iOPAnB4k|8l6Co;jP7}lxSV?TAVz+DA-jdji#R0$>z@_r2W&N=FBBX}j#-^dqh(&*nQXrk`26fH?aZgCu%ezHNa_vZ zA4x*t&ZyspS6^=+CC0cR|A!BxxL*uyssHP`OJ#L`^rP7;{QcqQR{()u{^D;^0o(rz zg;)Q^-^k~k1#GvYF2hByinVq%N|!4_PPSzUn<&&nyz1eKwW6zHtt7{!jf%C5U3ds8 zrlJOZ!RySXiVcB%6eK^iDO{g5$G+hNC4iCEj=c8{gtdaYz>1S)I#v6!jV62}>O zJ4jhK1fl*ho2M6Z`P#>9xmx^K%ayBjdH_un5~bq?p0Alyk5B)6Q@g6%8q-7>eLXr;D!K?1q71KA?}M7KkGtAzcd)H{wVP9 zPx1SYP!$FdOQtmiO&}ram??U!elr!-MU7l4w~HeUs(_9xep3M@d#G?Vs|$>Z&O=3+ z2~hJ`sbGkc^Z_Xcq0$kLYB2o$5nTVFhknH zW(%c7nss%PUcv z18uh3v{IU1sMBj`gB2cL%#cbMxgl9i`T`@0^8F@|8T#T(wXelV73>4CfH*;W6Lu~Z3K2A^G@v1(6fofFSRv%n-3m0hp z&F&CjiXjTkB@iE*E_Ox00GLLy0K?dhq74CbvroeB@f5~LfSu=_2I)|wn}85n!fn)E z-v+dIaQhFd15i75sSSYdjkRu5FA;}^Qe0MP0n8tE#$Mitb_8Kp86m&VaP2aGPyv$ofpZ()Rz~zaE3DVv1c<+6E0NUueRT zs?p7);HadJu!v#ePwR)mj$9?Qu4SizzqdWLqMW40)sX9f= zUARdpYG6jCX_K+pu%Y+0ne!^lnI8g&B{mFAj;$=U=7;1r z7!qR{s2DU`L|}+txPW0$M=bIz21f?p#cl%RH~Hf(Zm3s?1}7sx5kGwMdN0WbD*QyK zK!8Rv1?)G?wQx6R3T+S^*i;*oL(Bxzt29S`Di4oq6L6g#JX@)565MGg>@(vzC?>Cr zh|meFa~Ll`sbprOuPSXD6SBbZs_}yvF;m)UU|t6nFdCqw1yw9DR1SxcatC&e zVY1y2NXuD3XUY(GplJ&*9VDH=p_L|R{pn)VVc^ABX+W$!eT%Vgt5nUNnEPBEG(qU7 zi4EK%FpQf>W3R#VgWH&meR=v~D|O}XXm44M?dox3Mq1y%%f z2nHBILAn5cZh>q-c1*AU(on8*K5B+xSUYV>23GJT5GJiV+ulL!m<4W(pwye7rxP9M zj{PQ;>$5fnW1T^7OrU|SRA^%RVsM-T2#85yBT~9}3Cw;%TY76T7?DYjLB0!}7gFIn zIvE|{1ACk;2iwuHUkR{P6OI8N!PFTHb%~FNk5A)FoM27+Nmc*g@M*kuo!1LJ}}QMKE}^gU<*do~k} zjldG5)9N(WB%~^e9xMX-|Fhl^tyzyK9K1|jgO;#zO}q+@1My)<)JiL@tY&bcCh-OW zBQYN=Ar}p&V@^Hu4E8>y->#hc(L8SP{m+twq3(8fi*jNgrWh53>MU2CQf?1azP+Pu z(j9@yO9>&^y-)`cG|>P5@KQB|Yf?|sLiA;}>mra09BPRE$TnrfTyLbqK!C-xqa2Y+ z!hdR9OWC2I7*EpwOj&^?e}|r8K5i&1uuyH+Rg9-iodvkR)huH?Y^p87!y1dQO<57_ zdI~}`2rPgUKtx+Ac%8TA@*sL{S26aEo}%^$PtCFcwbg zT8wTS0Gnc5M3>STTYZ9N11p53+6X@xb!c}s^_%7x$DpL+{6u9pnE3QGO$T!MvlSiC z|0_$`dAX_s`TIqhj_?dc$BrIb6&^Q|kY@8ptlvA@q5>J4*voYsCMWC4;d4|R!x;PU zj9g00-qC|f5t}6!^sUX=L*p@JeOki}g~z@~2}|*@FUwAyNADV|_883v_O6#IK3x8M zsoujBFE6#ngvab?L;ErAJ*WY3UD4(L$4~fB`^nS)&vlob|NYiyukiPWpMQ}d@E4c= zCLK)vb00sLtdstMzfFPp7x-Wdo#RY{h4Dj}VcITPCvZ%}H24Qm_2tUC6pe?TtE|sP zub^(V4m0b>Uv4fzr)j?Qx9*Do)?I7%adW#cb zFfGQjsbDYxoP4Gep*85O{=x3SY#Tc15X?W={eshPcb9)K{7jRnR3aFbV*ZR6&15jS zq_o35*c$#W_h9FMp5(ks_DXibgyV3V*RI!PxCurpY9XNQKdA?gqbplSJMOj)yYaS` z4OG+LbN67-|Np$rMcBD?JKTd!Y!6OmNvbEB`0;;++7$;`HaW{CR*4qoez3> zJCk>CDwc?6;2j)=kIC~1d-rVMWq|YQNk_8tKryo6B5a@T11`d#M!nDH`kvb9>bvD4 z+{(+r(R#ULeJN0R+q)D1Tl|DAQa0p*=bm$UoM^=mDq@qO$-`e@=UKXly!~9t8WsuCmM?JS z_B6JvF3&@&_*J5KUTkzTjY9JxD^8QkQaoSCXX>5cJlZ|&+#s4u_XUn&5p8E@h6~Ae z+|4&XCSM*po6Z{k!^UYQ&YpR&4UULEh3hX6#BW2Duf36NtN$$P4+o9>sR=U_piB9QQc>UO5N@ zH~vRX|9|@GN8}0o!_RjE0)No|w}^26sK^O7tu$5B1B5I(%s9tAonfM5F3UJ#vi2my z2^zlh@Hb*!j5wF+!8r$x9n3Mwbd=^!$J>u5nQ;&P5!}y%xtqcxjr)X>iyFC7F}GIQ z2jLY=4olp@n0!dsKv)xX9QBhMbhE|Y>arL0y%Psy{N3nM_XTtu-@P-=N>Doo4)X-hZ71&meMEtuE)LuA zb}Izwg~5MnjLv>8_>_kEcSpU|8ZYI9Et3IiR9ritBA=q)Zb(k$D_qBzYjNtp5902tHzrqs{ zJlmZhz`1nWt$#Lj`SyqbI6%n(?DjD8(zfIP&olo;XY;|cv&5atw8WXi7&##j@msQGRvPq_cyuB z)wot_u44HK+1xII!)h;ks?{r*-feHt3|$PXA?-90TCeUSrR(YaaGaTD>WO+LQE2p< z)snalNSS)(uDrOqk(USq`D`>CU5A#9+Tx}qquAqBcp{$M!1q3^mXhi7!Py{^yc&$J zQ%X3oYPaNNOU`SdSu1h7e;ge&8i?=ttS5KZ>vCu~)>pNsj(^?E`{gTD)wKhEDs z;H8WXpU(2PeaQe3zG?>dJnjuDa<|hgg>&t@gqFXK!m&J9Sp*xyRPipSHTv;#W!?+~ z?k}sCi*__eK)nsE%I#36P!1&XH)^+^>V`Xu$xM{y+I}W}QX96KspIRU7U?8z_OD7y zsTI0gou3q=rA9e)+J`6lXD8K*TUCxN%8m0{cC?%+%k^pbxU%kCr4s$qL2DU5j-L!; z7)JS`eO(GJ-Il+NIBb(WZZZ0WLt-+9b~EM_LjN;*8%jeap%jx(d*wgdo5K72M^ zeyt;eS3mko;sihZeD5Ie%Xj}aGx9Zl1C4x}{O7xpP{3~f`(E1q$ma*R{^yMT zU;VRJ^!I!C^M^-&o3Q_n*M`>`#ideUJXxO%HoHV&K5WaH$h(R&-=5+~i;+STxny=OKrHoLkAm;_lixdaP{e*>l`ao#+ zKZ@z)X+3y-TukQIx$CRbd2>-rLC~&GQD$98jLB|yAT(-e;qM~19i?g1t(4wt04pI3 zZ&d|xv?x|(Sm_WDs#`9(ArOm_(Cw4PDH{v6JrF3$&H@&98Ak$lWD#j%HRw-5sHC%i z^xc$VmC-VbS+gbv=SGCq!?rj<2r&6M-eH5LfDWPAg@uiWD^iLGyh`Z8AH;&w7h_bu zBlBYoVc-kI_aZnMVa%8xm7PV7^+9K0vuyH!Not~qtUx_QKnqC<{KtX@vM8^Pc+?6! z=n;gEAW^KT)gC;W_2+m68q-?3#r|jR@Lt}*l6e=z^M6nTqtk=Wt`OSuV(Eq{r}7$n zbZ*ZJ1(^tt(i7RTd2Y|3d?3OhnZg}W6u_$|*U|bm6BzUI*~kRC#e65l;?WdkzYLKl z0a>g+Y_)P8y+=sn&QU{LPyf1edq&byCX)?@<6_7k%*JDBnw=cctzi#LV3+;QnF8%e z)-^j0*_`UUF1HSootZYDsnI9(u){+4ko~S(o@jx|-9Yc}`)j}Rw%s|G%~Ve9d0|;% z&tMkx|9_hJ^6gQBZmhHW^qQBp+3$F^s;9c|fvS_hew2lJ$!sE=BW3FptD;;Z@d8&1CIhkCHRh|%hik1C7Ee)1rFL;Y*56Mk&HMeE_<7(o7#i2BtKvFz+Pzccz-8imA$G?X`-xaP zJIEF*?d52ZF4U*7>9ma|8u3OW)I6;h7SqOjofgyC^!0tS5xDN&CT}lir0_T*n*v zQmVhqm9mvi_y$cZ!qRP`5+2@0mHtWOGFGkT?-nbhrYU4Q!;@TKG6`0Okzv1jby}(- zuiI^>y0{NdJT|jn3E6cJiXODBdX58!e=yyW?Et zK6z5P9ZMB;L1tI$gbiH9AYzwJ%y^G< z&wFx)&zLAmVJRZYVmvHI#JDI&LuQ)K2Zn7+YFR>-o2=pIrc98e16|v7Anc) zAip|U1v^Tn-dGO{we>0UWYnjy)upp!NPYG!} zdD2cHTg_yOiF|?5bslHZAFG*4;i_InV$oncOl-+3)#rn$JcVC>9#7_djZ~oXt}lk% zpp=u*_(tsIxhX!k>b2A;F!z7Q??0xjPsr_tEQe#5$q*|@?Av1eE3&VUJp%EG(3Q!M z0;32yei@vfXZ0~tVe#Z-Q>3pUGZqUAX9*kElsAnfU{pra8JWv8mNrY_NuU}iFF|>Q zcs4nf%MG;)+I_TR zk-d5Sl{L|nv5(cTk0C#n@YlchC#*4~mm5>IRz$#-Rn)&FmpXbgJwQ5cW?RC<=9DN2 zP?@vbo?nrDBW&OP^bB9zItA0)QwC|J!Fuo$wT~Pa973=reWSu03acljaALWbS_34! zLVLcU3i$ae3((nGxNWF36y_!H7aDL33$>HZFbC zYv2te2m?85&^BVu`#%#I^*EM{c!KK>MOTG*ffPVf-0v)f|1r;7#TPOiM70 zcqK``w*EWeQo&*7$vXky#E6>2h4_+~kLoEe0o+FSz{YE&0LA}+R%6ZQAfJd6ga>ax z?nAf;InpTO#%tw|fA7zsX$%mQ!9z7Qmn@SZPz7ewm_*7pCWA?Pv}Cw#V6;_}CtU%d z0BMFK*7$(PPM}wEcVb8BHkTtpw44Bujs6Av;sfeL%PDCmb5LueMV?7P9iU%y<^u=1 zA^-oNmYbbehax=*L=j+yDs_aY22^Umf1!ya@>GgzWJ6=L%$E+VKmg4Q+TF; zJ}N5?5A2E$IVoU^G67PBZyDww14%R0hOPuP^Ovyyz-3*4^e$Cm@SHLN1=R_00svis zJ)&9>PqD)B$NUoNdoOByY;HwWO+klxk)f*&{jyw4Q=I~>IIA6FiKJoE&=9Z zfQ?(ictFU31f2*>54zJ@4W@8qxBJX>&Q!k9+qZA@Ix$B92AD#Ak!57SJm=hbPnnmf z-*BoEf-2h@Akie8O~&ekO2}Q=K;Z0S(*k7>njE8#wQ29birqi_3l!@MKwo4$0f`|M zQ%$l=I6`G|H-#c5_8cr05JFCUE&?P5m|fwyp9(EDB2Y7&4#eNIdBfE`u{K+=kglTD z3#;ebYc>bbH8!;77@?TO^0cLht%p)8-0NiOy&F#FBNKxI&+$vdd9vkP5uo*E$S=Vn ztbyUS!h8hn!<7!;lu3P1n;3kfeFOk^$ss6Jw#h+H=>*p)$J7z#%mK;&KLq`Rbs9r7 zB4QB|EotfLc)Xlw37?QDbloSlfl*!eksSlGH0_bF$+t6|VC+aoFhhdHRdbS~6Kgsx z1=1ro4U`FkwPT=|G+H_@%~`wB9AaGfmgXLG0(@i=`|x}ud68ZZ6R~^RbvGdaY$A5^ ziI`>|Mo0$9$p;-Mbj0yQ8#Ifk14h;?m_FU5JH$q!GnD7>04Hz+*#pONDN5w2r&X8 zDB+Pz?a*ZJx~UvZ{?kBgBuh-_QAHSWI?aYL#y%xTfqf?iMy_pgvGqGgd1Tfbc20Ha zfy|H^u~QOENVuYqe42MP2?VG=4Kzs26CLT>u;fDM(lF5TfQ|(4Dv()+zzxC-2C)LI zNM`54DSp^Y2312Fb^wf}>uD3z6X;iFXDF!c zw7Y2;PD&y|7xz(6oLn(Sp->!rLXYKrL1)RFL24ds!vauzdw#Nuoa)ew0P_ez{{LYD zB0aPWG`!NPEWgqEP|soS)iDpIF-vlMhwTY*Cu1 zzP998_J=7)vhW?b04&woN@HcU)hcA%!98CP|Xzbt$|z46FvU7B9O5QxK7N zWp*a8$LXfeS~3xGuVlt(wgEOuM%Jdil6Ai9tCo^Uyagndn(ESrv^eh(oy3wu2RuJE zc>C=fvql%O<06Y0IsD%7sBn?Ev7tDw^L={XlYR=}E8}v`_4aLLjwd9-+Q-xFp z*3F1`G78zN(IVg^a%da9MQ>OKN)rlJAZ}Sl<E5J#)`ew;=aRGrh zfO0uM;r*uk*2X0+)+ZN0N~9-rfzlE~A4WIE8!Y{?+Qv(Z1@bY7;vNJtz}+DV;$RFB zor|#9rl+$>&&UOQ;`j6HzbE?af>H?x8554S<)P<~HrlJHP}YaO^hu{C6i4p=e}rIR zZD&k%{6Sp5rW6U}`eATo%=v){K>^Y`vOEm5HJg@)r$wPpO;=oJ zCRH-QTv52@9nrFwunY?1s^SS2c71r@tSOQR&~lNjAearh@6&q+E*-K3DBKhTbk8yo z5gFkMBU_(2B?YorSNX2nUqa152Iv`e|-j zh|8Zr z>@x;Q+6NJ}?AE3GQ*?_?a|oab^c78gO#(j>$Ka{7Os57d2(^k zg>bMSTIUTWe7E)Ff~_8n-B;^NBBwSc6A_ev{Qo6mC@`;4>Y($J?}hq(FX{vBN$#}; zHbll4a$Tt&Ugl@rS6r%2O>6JCSczQ$D07Uf`(4nfOoPr`9}1 zN8HrL=$rcZ<=$%H_0DR7wbyc~?f9xSKeeU7FL76YuN+5?aN;CWsT(r&tI;yfLS}bk zTSX4?|Bo_T=CIhk%;S7=@l;ZK=)55{w<3oZBZCJHBxHAQ+tf{3*@k+E3#Gr{f;vt#|N% zTRIIAn~v}oI>E`KO&idkzz1$=neGbT8tWMn+U8nnmU3Kv!W zJZr}sVGBqbiWGC_Cvdc#TnN7_z(s4;Lv4Nl9hm6TBB-r zcP#E!J~ArL((9e{YI(0^HXFq>x~pvolCFp z#PsP9WqU?tm+WDeZG?43GDey?keK80Ew&N2~hGFluC>d``Z z?a5sBrZRr=m0CYh>)}(3zRJCroWp<7#4N2|vMauPtWqzfWAa_gUlZv`Io61#uNyfT z`|0w$f(zM1FIDk1LSed_N|lq>sdBaDz4dgw$?a_^8oj-Y3~npY%Vy(h)bU2PO<%58 zl5*Z)cDuc!m!h|HIIZ6I01bF_(Cc`+e5SGMDAVQFZs9rde!UA#MAONIT-Z&^H61ZAE`ID zT~Q2Nm9ypeDy!6!+r-@cv9f9%w4_L?N(jUQhic zNg8(Lj5tl)7gq%}8Wj6-w$Z4y)BVb<-R!OLhXwT(}sH>+v3KMuaz52{*QP>a|r{P5JUS zo_^J{+TIRU;okT@U+GoL-MhNazw9*b;eO5C_ogP7l&F1Ic>G$wNNs$2^s)ZtaH*CY9U;#Tsrgj84E zzga>mwOft5JPH1Dg*0gjDIG`I|9$&r?`l*M)#|deOy2j`ekCMbwc_4pBj*cM>w#Rc z6I>L>Ay;-8DW-fb61#8?C#%^A<1CR)@a#e-FNdQ97ddbBdeef)^^&URwj9hZX5Mb3 z;tC`$F9WVl)-UD~^%9g@Vt!k`i9)XA&3!hT7>+O3<^0H>5et{KtJ*BtNQDMjS@K`D zm4}<=m1mHgcN*%$Qs`aqG9T$C*jo;-eigM+8d6kSU>bXXO zQ7kb^R)>{kX&j4vKLVlK7QYn;K5NAcfrv}Dnp!Hxz8_WnZYIy zoIkX>QSV&54lGu^kkVDdVpVcs#t@}RrZ~T$v1M}*AQALgXh#@U2~!xhnrM6{WX{)(Us4bicIj%QrKe_8TNdwX1fc=gBQno=WD*H?4Nm#O*^nysRzi z1?677oA$0N!Mmj^vGv^#R>kpD>`2$c(KcIK$JY^H{9hG^v*tRCSwWh|8duqIty3zu z{ek#!DyC+Md`VW@V(E4{hz|1QL_Xy6u1ckVd=;$6u6vl!#QN2$+U$8UOSw79r+Uem znA-qyYTBAr?zbap9L~i&ji4{u+AJ>RVonS-+Vjgvs#OgulVWOmchmKCiyLvd7!JhB zq?}p|nJiRl^+Gulh~1>qpLTYD{C`F^9y8-~`}em3!3Cmh2tAJ<&<%}t>bX)VWBifdUP6oUOxH@29HnZUdVu(@ulv=t|9Z*esZbbT-}!;lhj zy&O-GghUsU140r_!%WkSwA$?wLb6t?lVqaZ$PG%XkcflCAy2B7OLZoVVR<#b@y;^7 zwA^Xj`+Bjl6!cbF57nkW9WOsruIGi(L#bXLDplV@-&MIAL{(8P7K6pfdRq^xVei{F zSA(@Y&P<1?+N3*1ev3?KbeodPp7bD{mtnTa>u9mst3~f_`cpX_yNmi`SDoBoBqw6o z`%JZzTg&n|-48|Y3qu&wmD_bR9hVoCO7EfESf{JuSt(eHUaw}J%UCuL?cXdHWd%SD zx3^V)OO8Zp^Flk~y&2s$OY>RwD(`9dm4|pC<3VYXD_ObecLTA`HhdKtb;_CBiFe+Y zs`*HB77kpxu1md}%iLlbmacD+BVw7J!F&&0XF8FMI#7q5X=k3h-sS?;0s`*4*U|Y^ zz7rVty?0q9+bJ|Mcd>0H@SrqAMJX@);;k!G2sZ-05F$sqzL3}u<#JoO^4{k>%k+dv ztTubWkbc(u#WVUn(fsot&j082=_mYX^WN^ED430=7U6rm;N*(Cw7C9K#_6jbu|jWyEX6;UNt!1dEthLs?Elm%fw zC9Z@M<48aWIQj3#?>`aC)r1NmtKk?Wm=6a&DKo+l`3WZoJf&!#GdQ-48wXP+@DvCh zoYq9@5}cok#XiPTiC9LEPNjEF2Qw*HvmC3(Kc@2^maF&qL!7}56pt6JqNcG@(lPRA((pDF73SNstIQ=+TEeKC^ zDt=D!>h4AV4y(=~@>eaEcm2+CA$5MC(#!xT!M!C6tsaBdjh+#?IBBcHfzep z>QVvjlV7+(@IFmP_y^g_&H@3Bvf@;e)6sw7IC?u;KHfN^Iaq;jD94-6BgF#&IgkLm zKMnx(Nb_}|2NPXFC)e{jVdQWOI=}aHGZ4$b#CF71CKMEoyo{zl-@DN|&t#YL;Z~mL6(IDteNIu!!q+qQ|2KLe zMCfy)1nv3xtb0DgJ_t^mEoP_0(Co1%Gd_P0t(%@7R|qEGx}m8c*{GD1k|2C-kt2## zwkR8BE-Zo6r{atzixi|##IK&*y02&h>#H$!2_HLHyV3ETug3}n84l)S;j6*Se`cxD zBYOQ`Atn-sS(VN=I1oh*r1MQUAfuv($HrDK|6He{Mq*{cq5~C%8h3v3~XK z+LLXZQN!|dFgv$&Dd00nf5!8l93Lsr=csBIaN5c;Mx@s`qMRdf9PzY1p(1$+I{5s& zjj-sBT^Rc3IskDJ>~aQB6azYirFjE14dnl`gu9MN3FIw1v+c}ek$J>z*=9O35M1K# zg~9ob_|kZ_`seA2=cbiRsj4@f^JTK-Xs|a7D52Ewp?%ICF{E4RmwWz5E#A$J=8dE; zuxw=)tJLNE{t}xo%lpKt)f#x!hq&~xOy;tMhf81$i=o8PKM9rQ&5#m@nBe zIuc7J2SOgGf{fk*7Yu=B+YCojE9kE7s?#0|q~=`Q zom7+#9jkE@9BE9J+#TI|Im@Q$TW5H8*IAy$vh3D!xg4Ew=>=B2>V#ORGQXH6HbL+_IDiAjf0W+L51O!}bnQ z0p}K(!myfBg^kqqn1D?12Zu5u-jH;v8EL0R;ionu4euYo|A`#M|2FQ@a`tyV`}3du z&Od;EeyBfx=CA!te=9ZgZ~yZmUw2~%`$=qgWF-uJt)CAx4g?zNk3w$&!s<>;+Eaeq z?S6@Hla3olvM21E4AGsWpQ3x2*wcmTw1b>fz(HXromC(O^BKO$H3q`XU?ZrD3AAm0 zFyql78D-nTzBob!a880P?r?4%>IR4FRYdJF6$5|@bvs)?L(@53UCVqS{NzgjW_d$6 zxW_&P!eOXbYB#%03S@w!#qxswbnM z8N6{{<4umKjJQC$Hy@(?6e-Z@Sp*6`7Eo44 zSwMQ|NH7lcV{wTCY#^RQ<>6chNB_{OC$i!&4G>po^+g3M6EMI({r6t{Ed;pq7d5=+ zvVx<+6L+z>pl}}6wWlm?m@;Js5P}!TlAEIh|2TgCfH4<16_Nkg zsh`u?CFV(qA??W!lMQh*9S_H7(U7y@)LEJ$u7q!`(JT|)(ibU=PMr0Kn}0z@r{#wY z)jJ_QX|tMwkpNQDPbN*FvS1e~2xdw`3flvC8?f${@h*bBNw7G-5=$`V=AaWTrQIYu zn%Gl;UkQhPcz6uzMkEbmMLJi61PXIvZ~Dvzo+7j?GdGZ9>^~M=O)RX<8f=<=%BEWJ89;aHQ#|cKKsh>_TJ#OYv~c|jYFr8R15W56j#T5Q2Al+Aw>Lk; zYP^n`fVBE!M0=Jd@H&d9_i+*RJxg}Hjch9NEChq}Z+IS$fUq z$Ob+K!oR8-zvI|AQDnCo2i13&Xnsec`5%{^z_Z52@5t1C2R7R2O~9cghkFxn9Gk%7 zV-tKC>!BIF>1{(m9u@ZjD1=nVz4<=Zl4;K7^Z#Qn({3FE; zwBV*2P9pe7r|SF{@V9{sUF`H7|!&?HKSrIos?v1#3fV%t*{izRGw zAeI4Ji8k$oj-F~O58;%bEc^J8Ar`t@O8^-G*5s^DI8bQQqYrkc0FvOH)*ALDP@nL# zUuSD!E17>~?^8vOHRQm3<9+5DaqCafaW8zGg;fd81c(dXrOLW4J5H-8~u&sY1-UhaXMjEve{{+*ZC8O*U0fciKuwXw^A zas?gfYJb=SVGnz1Ir`|Lv0-|dj`oz=nR&n(~g&hCQmfebscj0u42Kl|tTIPKiD zwI}E~*d8Y7+%s?rv>X6*5fq%+f`j*2i|7r`0*)^`-ri~1o&lzh2Y^6#d&_&ebHi31 zkk$@SegS2)M?bAZhg+fcj=q$O{7da^rqHDMpRA2 zOhME{Pk(0f(Im6Z8n$}8k;C=q9Y0=^@sv|t=;0182&cPrv$oKIhEjOX4A8m2`lg+p zyLW7T_C@xlV!cG2Zsz+I-qi_c4x9eCwd<$+3zrrB6+yikSMiwE*yA>>Ai?2E!m?V0Fiy)GOBOBFC&|AQ)2~17+lsC_KMryDZOI{Gep3 zQK`H{x1~okc$Na(v{}Qov(`BS)RS~0SxuU8!b^On=hoCIf^ce`$_2sb+0JAx2uTiH z=`@`mn$Q^{C#tZ{-uSvdOB+Us!#u4QVou8_Vs`>I3UHW$^#Y737w;PrEIvEVphkfX zbEsaRW&A9nhM`2E)_Y2OLtWgOrr>u8jVHguN~LeT_W! ziE_7X`>M;RM(?R*GI?kvz*=pYY!c0^DFGsehaHZ$rsNb{5y7!N<84ZBpe^~;#W>@uVkt9Qmag&CcZ4s*hRE@+e3 z(W)FrZEp@^RDpg(GU^hg-P-hOp47An%F>#J%qLhy^3^+H8alJVp^~jn;yGiF z@wD3-4ThTijH%n41{waunXptxEcp#KxCcJl*1iVns3cR94 zBP7;A8i?aSrHb~W|#6#0UMZz_86gpuV|K5RbC2cjZ!Qg#+%;sfw=o*%oaEdpvdev&lHOb@X<_5NR-=QKch+;Jh^RHP|KEUU`clec6jd4pnuqMrIZ` z$L)7!z{IBv5DwK{Du}`wG?6EQJz!g++N%TrkqXM4?2`{QdN}}@YXQnz;?@@G{|GMv#`4bYRbqL^F?V@8V!o&Q9qOhteIQQ+7*tHu+#W8v7Hh?UKi#cqe7F zTfRdF!6y6VJB44pCIDx*{Z4^bl<~pNWc$>2AfaRXk%fVq4VJ-`9QMw3=yNa*X1D!* zr}FoFhI{RQa`^$L@*OnlZu=d&=t!R+FCX&%ui6BFa(X;622NoSj&+2b`rt6>?DoN- z36AB5oyvD$@E4YkytDf}zPBQSC(3Tw@^*ZsW)8OsU$0@ABvHcnk z+4$!*!7RJ&cN$+O;_kNpiTvYp8sFE1uI#km=QO@f#NBDXQ-p>S{e4d3`^;b!$p62F z41JhRb%%_7p7A-2@oOSgc4*{x8UrT+?9#|VjX2TB??j{5#HQ@F--$$r_?Nw-6iB|#n|JM*h``g7zvj3Aa|U$Co?WZ+{ZssUl>rLNmmyRXFHRdoT9duv10YUWKLytyS!? z{%$WGslsa)KiwR&RN-Wd*?Y0q2752OcG=J^GHVr&$;E!4@lF+XrqzuyYZZ?%?Y_;h zQ-z(Qy!#yGWR)>OLH_>@J)Pga>9A9ceaOB0L+)gp*~Vkva`B*-_&rvI-#!+) zrDmyOpANfK>}eT0q5O72>BgG1ipL1GA4I%Ug`H{PRNPhQPS%=@qmDQdnPC&KkA-fs zS$g4Qu-U3O;>KlF1nq>T+6S-x!KZ!D<8rvIjLP`d z(Z`QwaY)nY_^|VnjM0ejMBwe2<{Nf&NI_I5c z_G>Y={F@g3L}5VGE-lmm;QK)!W2ATZ_=$ z`{dn6S=JcxU!Ky_Mnt#%jRkOjr!fm)Uo6f%ZD@#WC~LWJ_bLAW-_rLgzvJ6_ES)Va zIFnsU&?hN2O7vMu+&wF3EzxG-p)6`Am*scb|9jG}X%Fcg5#7`##&=F2l=Tc6 zaRSkXTHnjdHIx&+Q(36$#Z+SD`gpl0OILS&?b3^AMTA0GY!6T;82K#p&z2iI6lCq# zBQDB9%mQgw%hY3JDAuM$6#5Hx;{O9XjKlNxVE({$meR-qa)46y=Te$?cV1roya4;sVC%F8p&q^~jX-~8^^>BKhxePC|^tOxnwX#{!JatjQGg$^i? zj>%|4OHX_7Ag&PT%k3IQQ!ngJ6T;Eiv!8b?I;_3$a#zH1ENLJ zT82Z5?PW>nP5sK8OYfnr&|Jl>FE{C|{(H-S@!_y+-AXh9#%S?x=4esZzs5yAV4<+Y zHhN8z59XTluq^hT_ZTD=>)dL!-dm?vfo#ZkRg}edyUlspHt^`YVeD#z(DExC9Y}G} zc>>unXbYN;pG zl`|lUDb1mK4dd!P{d3?dd&+zmW#M^K4eu&q-dXL?gr$E+G$M+$o$j+J#UDG)4(!CF zy$#wl_#|EFpcK&r2fb)cW7=xMdFXZ~FrtuobOQR)LAkO4wWw{1gnO!H<;I7Fv{n&f z+Etez0t*a8-lIWQeG9(}>049Tl6mN)`5^!Q&Yq}2DWkkkz?i`hMdza|`$TGyB2C6< zo;M~ZE+%?>Ak6{V6+Ir%atX5)ibr5G{6rRR`2hGJ!#-gi6h9`$-5T&Rg%W9{YpWB$ z^)gbSSc(dw8Iz6-tK;T~6L76i`c31SJ`k;NtJ99~N0|LD(6WM$0?N{7JvOJ=rXVd2 z%`){^2b<6S7Bh*c$&EF1>EZ)rcr?4p3g!js_l!>ShJJ|&6F93`-!`3IQe$6X}Trfyhq z6-ZoV7=eJnYY*S+;}0_xMNDn_&te2c=s*oxH3#H13Ip0Y`Tuv3#`M7CXB%r83j27! zNKje-j0!)@%#qcKbCyaEq+7?={K=PLuM~PuxP*P?7I`ty<}S)6$oWK|O-f69sPRmL z5N0qBW40-llZ%7!oFC>3bvo%?IKXtGtfNSs45pJDovZ-iC|)PZdKzTs6Yx4wmW~+e z1legKOKPx^Jee=!q$678g`8#@EOs`vPK30@Y|+v9c5?BPNoziPZ?*cYpV634AqG2* zGbSf;jpEnshW8zysQmK?i~~?VLv`=YP#`;rk<&b3NER(F32{+F60~Sg%}i69oxu|F z|L+0fpJq=(VGSmXVoYzLCJVWc#6c&YJ`1*{qt1@0LC=^A0oqJs`_&T4qCE&URXa3& zPqFWw5Vfjzlx5ZMd+NQb&>#3MF;_Ce5RhIf!U&I(guRDyGm&vN6 ztdiit+M6N2kY?3I06J}TrcRe=!=wYG)rG)dI?ougC7^cC<_P4*F({n-YOB-vw7Y;m zXMDCAj}G@4W$8IZ7`V1Mm)g2*aywt-_&gXV*ad2PPM+@v=FN)y_ypK8qY<-ZXcqet zV-u5nBrEE4$e!t9T3elI;xuJ06ozUE4*1PIAp@C04;yrbH%Fx20q zGKeF=L1xQ9P#m^~RoHLN6!^(emYW_rLj{bzSW&ahJBpNk2N8Wo>tGbitvItF(^6s* zMdLzcX+{E*5Y1dmCg_ii_x5&Q_+3pRNgz?ydupp?yxr;&hODw_wilxzL!q@WTGp1soN z0N(!Q{d9%0Uhi}Gl{y^E$@rNr*q6H%Y{TQ|FThe6l*HU=IAo!h$Qx@e@94A}w27|p z0S;^$?z>JJ=nvp9k-cxVB$&`^t~l}|K>q&&%LI=G7|vSLdBFUITptNv9&n-VIT4a5 ziz7WSe}!d@Lp}j`eoeRhW1g-s^KLOuqd&xW*6x>op}lCXH_%kt^FXUiJ;Ia-yJqDU zs%qaJ_54CMK^5r)vAO4jiE`{g$a$f%ainMdE$@&76rR2QHC_3 zGA*vwZXdryno!nz>)wh*nz-;-q>1-iciiv$jZs#}%N~N~9`gU!C4BMx_y;CZyF(9_ z-QoSi%<$qlmdS_X?5aEVZED=K=eXK1W?7KfGD5GLWlN$QF;{P8VOgg7$ZDVO3u{}&_?-T^zkuw*;CCUW4^rzVfi!>3&g{$cQJPI6NoEB_Ji$5 zPBEk`MXIMkn_(zhbHmyRsukB64#qqQ>(>&trrdzL5Y%~77%E#Md28of&xu7W1bLw} z%BO?#pF3S{ou?wX%bR$)@T?jfc-2zTk-M$)rr5g8M*S77zKxsl6=fpjd0uzN5~*bs>T2-@^m2~Qb?Y`&>?ZJ%lSS4cdRKi`Z z$Zk1N@dkqK%UI4`Ny)yjU-%l|Mf?h&@0@bGe*SfS3U>kJOhp)&x)^k6#3*SN60^;b z!OqBRsx=An|3?$B45fMc+Cdg7lrGuAE>B5wdPo>TM}+}AKq=UQ7QJ9xI+TJfXvnoA z^&R@dx+5jRvKOL&%#e>~e|}kS5Tj++fJkWr+A}FOx`kYV8s&3|qB`Adzj@CE(Lw)d z#6zynOd8{l*uW-o-(i8dI+xxH&2vZ>xux;BeU8HjP2jJK+W(Xn)+$r)g+6AQ&$ZN4 zJXsfIJ4BknDa&?f+@YkPKxJ3XC|BAU7Z>cJBX8&DXXC+Gach9qXR0zAZQ&3<({fy% z;Ujiln)4BrYWetHpxyY-Q52Fhzo(3`p}BlJljUJvOeMXw?ld{PX^hjwyJ^;Qjt306 zulSeI(QMUwxbC#0D54+D<%810WZJzA$`23e%g$ht@66?GWc6@W9&PKfjYo_$UA1N~ z^U%B8Oz-4mWwmaKYPOs7kC8j}^GUqiQ{=|Hobe=6p`O1N2o48oyxy5M5)ZyB@%k!u zD{eMHF%eoNGQoS7_>hXuvr_(c)Tqs_GALr(2}zS|w(IL|e3Nl@bhAm^k8aS#mkkpq z&F=I{17?hgo}1EfwD7*;fE^qyvj16DQj8eAd~nccyrAD688Q^XM@`qB#aObly&v}0 zy=1pG8u^CH^@DHLt5)x#`J8g=nGUB9?Xs)t+YYM3T(#8-p@?R#Uas9vvzwZ{@u$6> z@Qplic~aMP<;pMDOINX8W8F%r*Yiky*-s6oGjDx8jD({7a{6+)zD54s&qpiC8}meR zUH?KI$8wY6ObPUpy`oylg`>kps#^=G-ch*Wjm+|a#jr?$6sAVUR7(Q9qs%pR@{VT? zq!gMotwuiXG$m~(21Tt_<#vuD4vG9Lh_9EoC&?Uw=k_X7s+a{=Bm~kY0scg z?%w*Qk}Ks)hR4%j_95smB_@@Tj7?*`PG(Rl?zl3d@)oR?a!}(0>A&uKSc>_|H_&tGPWnM zJ91Im>p$2`y^UTI_ zHGzE;yr)=A;rgsOb`K{=0Ss6>irzaDwhHnBpBUmUaCsNx4jkLiP})0421_OnmUPA? z0J9^0Mb2hFR?D?YH4zuw(mNq4-y|v@arn(GokrdXl-BfPE>Wq-mkD~?L#P{qQ2m%p z(~GHeZXjh3d1=U*9%(67)CtTxZTT26-e9K&*Yp>FlBq?=M~4Rrn$M)E6G0jVgu z!7mxbYniLjak^9{blerPQ^V*&7y=>|aFA>Pb6+y`Sr=>chOT(EN5O~xA%6c8WQDHT6(A|dm<`HLekNR^(~B>JB#s>+T#VjCr(qdr24Sye0w zWJXbw)E9lK6Iu$AD1(KxR-Q)!tf1^4aFhvuE z<{FHTNf)ysXaG(lTYzzFN0|>nbfZq<@9`AQNRXXlPeXL*q?>>vw8Y!Uy?%&j?-2aW zNQ|&dE)!EU9JE6BXjwf_mCmV9=%QCkb&h_nk>`B``5;;$|Lt-{*=iZ}(m9!DjIZRX zMU;WVk{uu&gKIz(?g7B64yKW|gr8Feb&3XMWYijdiX|c%PPyCPzF~dPTEU+pvlMxK z(FXLHJQZ~8kU-dPL6^wO(saetknCq>}o_8jlj}{u>GIj%kK5GIr9jC_l-BY+nVDL}LgYx!G0VQ^j=Zn&xbEY7kp2 zzfB@BhTs;a^T0BNvkyBGFCK$suCtHDQHR$s>+Hr_Q zxmf(8$ocm`%X^hJHMz4Ydr>&}2j?(uZ%~oJ4s;g`WsT7|;My zzt(XiVA*o&=ge?476PktQVR!8v=3%p4O9Yy&)|Q;uYZDJg`3 zs$g=V!YRt8WYQQc0RY;{ie}UYCupV$`A`u5h}u<5)PyZ>RnW8$JUV;gRd9uVqZha_!lK?FJ?-dV?bvTp zx;}Ga(8n3_#ta(xN`)q-FGj}&h=4OmtVK!}FG1N)Xi9G_2P1OnG0Jzb=7m-Gj&?=| z_`n`#!@*{B%vS<@)x=}qM<{hhL(Sr&YE9L^QYj2j5)B#VJ2on)mPCY!Dx(3_OG8IQ zAfrhEolvVTICu{-z$j70J-vf$MNTBZ7MMvCC(p&(H%I&@aU3@^HY9mOyXiT}V(W^4 z_kCDun^&t(b-B=nG~2Nwo0=3jtzf7vXJ=W;6XiSgctgV1N$+Xc`(BW}r=G}rA?iCt z47pIE|1gKdp;hpb^R|oObHNVb>>@P~bT`5~et?2qcJMhcE%+04e!rZ)hYVoOX2!98 zumo^go#vK|R7KGai$MPWxOYKg)*}grE>pLVC5&7XuR`O%d>E3n(nu?t8G@+Eyn(?; z$_GQpO~WaLQp3~-mzdR)UCsJG{a$h&8)-!a@T98SGGl0i5*o#l^`5 zWFOO+)heRc3F5lFHHSvc=dV1B?qFOiroF}HfrBd$D;F3F=oQNgn1Ey)?XNB#CEv-# zA?uDv<+KPP+P!iOBFrKG|M_c|A$%qEyj6%c%+|FCECbszM0@1WV#FA4z+vEk#k8Yb z(2|7zRQM|8$O1)wlKy9l6?pQGtW%7~^#uz&REO6p`qRdm1-yUIEu%kdEL()fS1iJz z#fo5Er{IVNkp+|jiD*iNu5(_wJj$Mjml!7(&sp{e&s_Bo^Pd_+4bgU3%~*xlYNb4P z0b}x{wu;f515i^;i)dClQ>)Kd*}w~7E^UOLj5;(s8|$0q7~7z<#`%fLZZPxdc`F^* zj~77dx+C>cIYf)k;Tr!9vH%9tRgZyG%kWnp0e_R zzw5ONA8voXcHP4kUQWv%gC4Vsz15F?@6i4}!{6Wj z{QsE(f3YFZ#^itL<{Oh$T7TegQ(*B0Za701*wbLg_+iYj)h>A_uua5P@Qlf?Z zP{jYz#rkXn|K6`)wQgOib06AXTmW06)0ULJI{)cUwRJfT*}=ddw{TaH$2X|vz^6eLGf6CT=)#}&W8eoefACU zua5@^?>>DXh|m+4BFUJ?BZDVj7({3``e1zUZetELoz@U6K6v*9d))4^_+Z4DMq@GA z6Yz#l6H+LVK~BjLej^N)oG+Z*((;9c@pvK=ODhg+g{oh~CxFjQ2t;@tJD8XUH+ zEEOHO+dA(?+ViAXj{lm62SfhNaL8oYEiVlv2+yjC431@}UPt=)YjL>SVj z)%nucQ>$G4utJ1eX`knEF>9|E_sxi>n%K<3xpE|%Y&C9ph_EL6S$Q}LG?dNiL0r}_ zdXNjHUukCUrBu?D4Yk+(%Rtdz zyua&i!)p}LDftj?noHk~lSxlyyYkOg3GXm|U+&$6YPV@mFfMf#1K(9(TI)XC&BA%l zL@CDQz*1UtdhH$+QEaHqTd~l-^iQ&@^1c85uHW+!I@Av#L>Th_nP%1@uKQq^@UugN z4-XkA!beTlpyIG!DCTlvey&E6sY36*<*KG;m-)fAc|C}EZ-dc(s&-lM2I^s7v{VbX zDZaNktLL|JMd`<{qk~LoGfn0D*>Nf`m%aXQJmf0eE2%=awzykd)#g1xwFk#shlsCGPyP&d=L!)Y!p7C4MS zwCtVfAtXQWFyHi;VtHt9+Sn}6cQdi~%n#dOi}+K7{(?bHEv)i$XJ$JX{~HPTj;#O9 zyFC8)k3Rd+ACV^b?ayx#1%Cf;_=1-7fB8@SX2u#@*B{S~HD(%&qL5!2QEZJ47E}bz z@rdJJs{mo}#(%-(|IdE*qd)tb#QDGGefc~3CnUIkSB?vATxqIC1PFO_m~)OtI>SZB zLYA?`WX?%?5Hw=v5pTqz7_l$YLvRiPJ6K?n5h%@rjt_57GUp!RBY2z#3pYhX8jlI3 z5H$*=VqvW`44*y^V+=w^$()nzYgb0;>)D9DYXj#!{IT#g76 zLTQ68g1oiBU4iZ41D0?(Zc+3xMRc+ti@jE0|6Q+Aiv_f8-@ViKN{~AT0rN!9Lp$%H z)*|s|%ky@m-SWeFVf3Hst+Vb6KBr*bx!rOp<_Qt~hHb%4Kjr(^PleglfwX42nVEKJboI1W79*u z3q=ok|7`6)=KZs`=vlrwgDaon{o4+@iO6PEepU@f{ChwlIC8gjo-9ON^Q8es@aqc> z0QvuK5s3($y$cXvAG*WdKMT8jf5HGBAaDSyKg_sv2pr&L?!QnZ6omUv_Px-rZxf9J zqD{v*t)4&cJ~)82r^h^hw(4H$QnHHL56klx9V-6nBtG>t%0sCodajkoGBE6@-OC~K z{ONLkmrC4>EBWRooSu-+?b^|0)pXM+Ci<{_BBojwf5iD{xL zR}*r!)@zpY(pK~)s)2qv7b*zpho5cP$^1J>CQ+($yVe?*nQ+8m|vZ zgJAS#Fusi`0eRDI`PMC8dhVaKs`%0iDIL7sP6#yk zhv50+`i(4QLUj0ap1;FG28!@e)3xViZ&37gJI#C`)n3W-^lb=%<(}fwQya!|tJJ*K zj}(fFrYJr%%8g|^lp>)-&T=x7yozP?v|;boy#uOvt7nX=wy3YW!g=O!liGlSMT zav8ZAhS7||b^A8&S?_xO_Re7&>hXZnFW{G=VSodKk+y&becQkRY*MT%jX!y(pRHMT zr(f=NTaC{Asm!%Eak&inlj^K~Rc%#64c|KGjc2^R+h{H^i5BC5spj+x`+O1R^g9Fq zpriw(>H60S0HAvO#BF8C9p^%VwxEoY4_LXd6IpDbG=n=f#{(;N&j0%)uM|18u+Go- z`0lX(FQ`ZOI7J6VjJtgO{|BG_=nqH}{PyS9jsm~`?n~z6tNjK#`S$hC4+T)b>i+w+ z^!-uH4|x5*X7c~pKlK^?{WbjgJ1_n;asS`F)uYzvTq+%mC-0N4X4g2Hi&H&xqz#TM z!~>ACOgjlH%(w8UjG4nIoht(VLC#l&e4&~+6JnNwrdnAzjDUlw0?zu8PYy>!$pOSp za-$G-$aZeJ;fjl`kOeTcjJ;^f`Q^jTHuJ151Kncyhi;G8jh%go%K~B&5C=s)p>&o- z1mYF>T!K@QIR(c60gK3eMA+k_!+6|Xb|(DP{v&y$@Y^?eoD)YkQ{{A}n$9PLN)ebB za4*t1I36b~q~m8oyZ=#&uQOH8?PV^S-llGEGK=Oi7lWZ)og&S;Adks!cP7*-ap6w` z+m2uwH7}*}7{E&i$6Hy!Ia(yE(!F#z5UP1Dxg!vRB5VFheV2_LwmlQ*l$}W|9x@I9 zcc6$gF&gwo7K-sCD7~7}S!I;W&a9aoLvSOG*2A|r!4Y7Jb6jD9rUM;9vkMO!Kdwj+ z5qOo@g+GV^r!RV|+!w~j3XXvNt;DLrng~ zi}Kk+0zKfq^M)g#7@=SKDUyIZ)*lXfIWOMhNaN8~!(5O5Nn?8klP++ak3g{ z_5J$#@0@*i_F*#?r&e582JGocLjM0pgI~TsY0#W?)|g)7(josHN2_{n_#UJ>3hqZ* zsMqu+$TiXd*;_MSB=T! zdhFX27g91hsWBg(F8fh=*;g0y>wcipYGLpcC7*BST;aY;R+LpPd6V1v zGu@Tq6B}}UDRsx!E;-yz4wAWIdp%mlv(;&MI&GtfTBKItdnEICKxp!T^f~D}ev@MJ4NGK^U?vsIXu^d`#L+$8w zWtqEE%6D_W67P;v#fRus@qX+rs!MXan&t(xiY~-Y3EuMnuz_KGhT{Hs6zGf*xbYql zWLAW45!%?T2Mm`=8~iFZ>|r#Mg-fN)OrpN8|s3 z`d#k-yZG!!7o-V(`|~^s{C(n&$^GzGzdho7vrf#6dH>{KJpGbq561J5IxyB`&ke2n z^e~=Yz<6@dP9a%NByw^(OK_c+iTKBIqL{s@762^ji3CV3IYE6s3gs#M$>Vsk=xacM zF1qe8ut5ovQU6Bj<)sjx2iLV!C^+}O$M63{s87Il1IpnTZZez|B=v17{tfw8$RC08 ziC8O>BLz+o3jESFKS%X3ps+YN*%a_K~ zy#(P3aWpxG<%Zk^?LNxUaS*k7Y2O4f*aW$y5Ng{Tz*6w}F(=+ng*>J$4FE=W$nXYg z2%(mV?CqPMnjMYl^H>G*81S)#KmNVHhZP2RxiO)&;sk6#MgCg~siQaJ1Hf@J-x4~u zAfO~rWx=pLe@gz1fOY!Q5Wcu~3a+>3G}45@dh`>O4-5=WA$XI%RpAbW*OMTe7#34& z0KiK=&U^$2CQKjfUwo~H!XYJ3&yF2R`$$MLFM1;)h7rKM9PtzpwB+)dyD{&(8HL?e zQ$d8~M06yULe)J200Vk0lE+}o$VuBI0|6iyL`N1)rx$!GC*i%wmbWL&UZOjz?~_Q$YKoJ~UHv$YJ~$Y}Fin#o$fm1WZdXjd&$VzPA25;Znh2 z=E*w&;KYcU#D(~hn2+iyF9F;}_rS($qyWYLfL3G8=OCYm6NCqEK<-1h2szRy{ zkALsap=k^dl)*zaHJ2=tAy5To)0jlcHYS5fd$eS@ZD6!jlP6sPp#W)yB-Z$V$WEYF za(7}!={A=mLbRL!k&XTZ{Ne-ZM9V2@Cv#A1qeY%cK^>r9bmjvGx*`AnvzD8kScf7# z2}BWKhAMT0s0LJOz<;5MB=ST!454+1-@f^lP(!~D-eEM>7d zi!uRHg>M<=AOlG=)rPJFHS?FS|G;Hkfb=d^V(^?Y0tM9xaRLBcf#gfFZ|W3)iRwz_ zDI}BvB1r=6C%g5k-DG(~W4TE%XV6wi$_;Jhp6UgQ-!Mv25H11cVt|cX!gxT)fdrih zOb@!#S`DUfWw-mxb3{J{0kK83qW6FJOPOz6;n;JOgKViayNw{ zCiWaG77#*CeJ%nd2AEypxt|IxHX=|noDRg_w0XnTJ+U@hv5>B!)eEcV+iNxl(KR-- z<`|)v#qzYJh^>cGE8Od3>b)CI=OYt?1JChG#CfviToIu4X2>tWBCLVow!(Y_?ZcH0 z;FL*yP@5QhqkRMbcgZ0rRkq1NPw52LDaX_i=F9=f|NjF0gmoH2G$LXV5-n-z>3F=H zXbGQ?DRkW@wSiGx_mLd~vo!6Iu*tVGonY)pM=(Qz#Z_~XqZ4a7Ed|meHw}~tgSBIz zm^4~CFU?uI(i~!3_?G4#bOL;268rFcBzciu4->I_+I2S}0c;|6^NE;dA4W(9$;k&D zD0IZ}L>n}Vr~^jUESNs*AP|=3BxYgoQdsMD=X3i&(&$B-?I95|j(1s%I%+1=1*!t( z3UHzgF@p95*eIro)>K2xWDr7_Nx_4B@9A4sq1;*FRY(j-@(3{kA}HaJOzqHQ@4BfR zP5#qBY$Que=ut%&aXQV0F~&Y6NP&GP21c%Ja4D6U8nIInOh~w* zkbIhVH3sSg5dZ8B;7!a9GEL=aoEiR)<-)D!4eW@jj3_ghV{!grF~Ib3 zK@+rLby!Uu&duBk(3RIwx7P=emx!WS8cx(f2_J>;n4N|#>%eRyN~gYnaFQY=PYL#5 zA1sxVcY;iZVt{`NF@Uv=!!YlKUqUV7T|KiUy)AgyWnlO2T=!9`R(fMJu-r1=XyU4# z)7{^aB3^*g;^{AWs!H5!K_Gi7vDBIHrQaV2gp~J~(mO0gnBZY;Xa5l6sW3;~z}+4! zfqm)MYW`A7T1AC9JYlP#8|p^7=%P^p>ztq3OP^TV5R(s8&umeesJ^!3SoVi0N3!r8 zxd1HH+Dc<(;I`32!3SJ+T%2EigwTgTmKFVMAp?4{J1e(fkD-}z8?r&GxP|HjN7q;~zzBtD4ANJ6Pr86WAXS6Pqs+9dW3HXyO> zNbeErU2D34Q`o46syGF?ck5tXC9YdefV@t8k%j;5jZx~n&Xf0lL38o+-@5<~Dn>>Nbn)KtM;R|NO8=?v(8Z~Eg%nkad_lDx(^;&Yyhp9p;1M6l)JQ;=T)o2lL z5;?Su-l8|G1EmQCD-gG=qjKt1I9m~b^%lw+sukd*Tz#`-y10PA8$h|7pYVQDerw|r z7weM?ASKchxrlC}^!)qbW>P${Rr|BPyR>UsDfYtiqnAWG~95jmR_x zfzOR)v=$X{hZl3Pk{?AMe58DV!D}Z7%g(~rQG^4^>Vc)sd;K&wEyU&3URuJj-gDAg z*y8oRT1#OcmLrnT(SU@KgN%QPHH7X6)EtZ;HzR}99y?+3u*)cHWcC?@B<+KUT6XKw z{VBRdr#S@B1oEXYSBl{-Ik_-KFZLXN@xjX3+I8A(Y71As;yk%H=t4MH5UukD6TaJe za=})Q#_p^2C6QAblZgmQK>q&^V<<4MQR<-clkbK4eJ|<*?Md#n1vW&+7;;^y9$x7` z)AM|C@dV$uA&f*K=JAK9y~hy=^{de`&O&B)V_QWI^8bG_TjsFX zz0Bi$a`9AWx#laPsqQgfxw-#lCo(XUU+vm9ypb>WY#V;ompZmBtPLG$Wa4^_f19}{ zXg71;y=|$@$`XtnoSfSy7dyUi`yd#c{`@J;s zW&X1dVha{O-q2pWxTFlHv=g{&Bm0-ZH?)}wC?K+WdElWfn(*Y}zzt#%qj;lryeX3j z<#lC?N`1kw*BYaK-Sl1Hd35R&)3vdU(;Ztq56H-PhBau9xfL#|{CU=nIl>l@HWVr5 z&QIWIJGl^kSAdHISE=**x=OE!JQwg=4r%}I|As@CRY_I?c4>{O-QBUcTlvVSJWH>4 z(vug~PPtlJSmCmobrL0WyY+RA`jXlUFQ{)eTFy_A&8#<4SamMFz7x}@LzL|qm0hxj zUA7U{8Oa!F=0IYO%aaR!YKv{7v#m51w)&aLa@K*+Eko8eRvwnT18gkU z!o_awvNTU-ZpzCBqSa#swO688(vyoCIZXL;j;+*NE9yQmmavYkz@D^A1)!@Kd?pYm z_tA}+G>`sHYGTaq`qN_dlt;Wrr#MY3~xd|XRxYUL8W8dF2P{ol85 z_OAA0>(Dla0O(jKT#L)$p_mNgz(QCOGm*=9Hj;{$A`P(~na1l8zu0tL#hb27yyt3) zqqlEds(9}T$MGw!#C&`$=0%UF#3fM?gSaA!{^F&wA==5APV%ZyTg#<%JX#t;{{Np$ zT=o=W4=eeeC=``Fqr}@crFzM~_1CLjZzi6Wx5>ev-=Bvn%lk;Zx$TN#;HsQ0$5&aU zp4=wx27z3y)hmr|GNoj_(k?a&!@$E;ELJZT0{1{unI{T?W%GLKFGyX6c4*8euFLkiKJe7Y9+G@&|zwz{|p4IktunPCa z_xVb%TJGM}eg0*qaTl*$sopEsxLwW9aOOx{dKQtXE9+C+p~6EvFl^5Ytwv}b_AjfA zkS`qW2Up_8l^VqS-gLQjxuG$UqgUlxAfXPgdc7XW_Y=30pCzQa>i*3VQmNf)+~rB| zpDUzEQ%LDJ%Kq=$H+xs3lBiafrDgKIzxFF3>8cg?HXAu#s9F!?ik;x1I1ah8%SbWh zbCKAEb2wSeMi^&_Y=UPOI(a!9CAi3WtJj+rM6Q=qJ-6jxb}{pIBNbO5d3hOdb+Udj zm#CMZ+!FKK@=X+SC2#Jt*~D;sxi05N{)||-tXyw?x(I= zHZ5mwl!7vt#%k;3GI&2$o9gsBJCP%?(e+g~G?>J$<-5Rm?2W#C6CM_%Xrto~j%K~# zgGV=f98WyXP3p0i&O)|y#4UH|Eg8+%3&q6!Guu7Q1u-!Bv44XvMQ|bk4;2DTjv#%(< zO|(|HTc!J@bzi=j;k4f%F{)j)8$C~Mk@8eBSH5Yrqb6=2+TmqwQ7>fN+=T?yVT zU5Tyley}Qzr(#FC9*(xz+B&|D0OS9vIGi=tVay8BJl43%j%%G#x$O_chf^^%OXN$k z+7?T<%RzLIFDLRLpLbO%1>~z>J$Bv0geKOnR@G+DlUd5mQ9jj6&cxgXkW43m%lx?Js&8%z zok(jrzEWJv`k)Z(kGiqNRLlhCMS#t9TcxcyX?u&SX`t(ai5Z5Ji0kEeiXdw2 zXc}gkZlu+2mk^S*TAd^l?M7};T7^U$Bo28}wOpz*X$;G&`Hgp$@ulTXjPhj2=q$`cSF*9{R4z-5{!pa@Ow}gc zG4fkvLZjQ1T=t{~>AVcHODXP=AG_-01|vBU%id?IrQBMU$LW43 zdS4jAn6BKeqv^Q3s8o6n<;FT)4bMu!TJ(A~^IXQVfoT6`xhN|DYPh|v`de}&Qkxgr z5%10Dwpp6bvR8Re!>>HV3mFeelU&KlO}`t6b++NF(5O?++)ljnzEsUenzL}=(sfE=sMGhY}A1|>`XiJ-1RmWs1^`#-@T5`ukxM1xbMBoD%nn< zk-3X)|G)OGtT~P&Ig?V@D{C#^_S)+sT41b5F3~g7-E&|D09O2HRI3ElYSeYlpSu}dC& zOBn2Xaw_@cW8Kr>5?&>G!u8T6-Adp-%1+pe_>~A^6gZTClmGk5m8&0YJbdQuQf4)T zQG)pp@JW>sj!>U)2**?M_G!Vel|0BTnIK9aR1mF+(j|zW+Su6JxVN=&-wR5?vMVr? zoHd_pe7(81^Js7L(e5`7_8xtSG*leyS-~f}kGJ<8-`U+w!at3)YxH%t>cV2NoMEL} zYSc%38IW z$g<+X;aZRf5f0?M`hA>^?sM)STw4L=S0p2}Q2+bs|aQ zUUz@DcsBQ7Gu&=&Y;D(S+cLpXZWl2BKUAY&BA}RKgcX8gkI$SfI#;fyA+a5?RhkM4 zAuof`=a~nc_4^NIlcP6xh6({(f4l4Cwd~zE;eT%`AVlc(1cH2ja(L-xx8P>?ugDh}Rcx z-E}&^x_(Su!mmB--6(wL#&JwehW*L0cfEg7ZD}Yg<*<4kK9LA!je9o{5Je57_$CC% zjM2lQu~qQb6%{3k;~{$o#+)^7eQDvwFUjv0Ag+9~ix@R*Px~k9T9;~3O8SnZ{9HrS6@T8{`%nc>FGB+o1?*Y z|G~<}&fQnHzQUr1$#H+Q`sadm9I*RIO2w(z`PpLJjDWztZEokG;u^ z!{}&Q@;vY!A*&$M+Y=MRqt$lFLyVCj?{1^g3#F-9`T}0lcSqCFYrK=2uK^~RWb7dQ z3`o$e5&xxPxN7M#TWPjx^q+Sts8D*vqEtkD7U#}1L?fFFunf2P3uiE>oFO$y%{2|~CEXs5L5)i8NA?D=4ZBAiW(^-Xs)b24QGQl4NWi-P^P?AboDFI$Glr(&wlmDl&Qnde1p3-*q!z+J( z<%3^c`6d2&uRnKheqX@*=%bw-();{M-AAn+V#`VkWu}7&tXY9BSSfz`i~soH4Ql8= z{jx0f-8jKshz$#N!jRW0J}?OgOvE3#-XaOBzcuWRM7Z0l8Qe{JK0>mG94A9`r`b>5 zy*%yd#?w&`HK~vXg(IEE$P_GA__mN3a5saFFrE$}Z4dh=!aL-C{HB9*akvT~PJ%ag zh?|GFL2&&TUb`$}kf37x=41*9O>w$P%6#g5b`1%$!Uh8O__x~13Pdc9y*ct8Q3Dhr z4(o()FM6 zx;q;qe+I=BBL@d0BC5#D;+rM!2{uI(;7qXy!8FLYLc1>l91o$Y{?5PNzD^03{!Qk4ei9>8c<4WFPsyE!ySB*IhJ`6B zKyY3lO^QM~sJ@K8fk|%V<)>`0nn3xaCj%7!jG|sRtu(wF8*j!)pAeOhvdHhC%b|bJ zpZdzl!w;fb5M|OZ=Ij2HcC}FBsFuB*VV5!0n>CcC9}e5z?v!ur!kK9s zv#{HTwE_F?JNOpP-ZZfUzYH zZ*g`h@A0!Q9_(y*OMbchO|(%iZ$91hp5J@=@Bz#<6_1UJn9`#m(pQ(uTaT7J4tt}h zH_!p@W3QvqXnat9`lNhDEer^N{MF~!1^G3WE_kD@ed_z)_V>M=trxdB>e@Ry9tG5O)O5y6Z3(0f;*&WuL5O0XALk1o?A>0DOxOc$x@_oJe5)C&T!9JM( z-;1erl%$c;K^UMooD@$H4Td!8K3^s+WfFT+bTjE)cQT>JGERFmM0}5k-r{xgh88ed z4-vmAD-I-07m{YSnKlv! zN%MOWR9sks`2?eA>ct6$dT0)c1r)bjJT8Hw!v(zwooEnu{(jL!PBe&ppMxfJbhw~5 zRVNw;6KO1f`Tr56GB_uyniCa8bQg~gYNKR%I@ z4O}XPjpkf(EpoC!d<>v6DGa?VlgOxSFj`(1dde^QgEj?@!l=?eLqal!@MAG?4JExl#16eb6)${y z=0oQn#6GeGw|y9iU?UxkCof=c{g$#p{DatD43^A0ljF&f6bfh(mBW%yZHZ0$E`aTh z#@H<3kORI9I7&366FORwI}fiUBFpynIAfvTnIVx8(wejmDGwCdH21~*5fVv=PRoW} zL)It!WN~(;24r!`-xH#F4as?)e9xv4U%f@)UP#80_QU;HYyIe?jdUh&z#1hoEE19W zu~(v2zM~;mIiV^@{*yjNpJ&HI8obf#_6rYZzVezsJ^(qHjPj|t7Vvug2~GllPw-L~ zryPLmsZ3YdVe`C9?y1p<)|8WREYkg?t|^UXO@X90 z7!zzyXf7G=(pS=Z29}VrnPu69sBd>Dxn>+<{-yAfYkms8M`qZ8HYP}1fA#xfoUDgN zd+L?k_V7#>mVpwoO#A$sn!`jHuL=hY=GjPQRrq<4crG%llGxk%=m&SUM5q z)Kl-A?WJT^tYP2@oB4QD$BQu~Z@F-xhZZk*C7W(~3mr5JRx)aU;sX6klb-eyqt8(@ zvlJ_cif-xm8t*Csro*OQ*SlU47hFBuW=3OE&TJ1FUD1XV!vwI0#nX#jjW ztP#>#b}mWkNwSf&CaIhVi08E2rZUCzO8Qdnc}dT##oY6@MdC`=a-O%Kb4E^}&{uC< zW>@J%DRHdR3P`#v1BAT*CjcBvumX@Qx%fU=VDYdLj2C1d1R%@X_xX%=Zg^6~<0 zu&HYFD_O{_*JTVjt2M{wRY8uo7J5z{aM~EY!3i zV992o@(Fg4V)qW8hTcg(53>Fy-b>CHOZ%Nc|48c3Sh~|?kn>NpyxDlb#cy!H-H(i; zeVNoX?vzDk?Q#&o=PUL*Xy0HKvrCX4sidz0&*$M8rD(Zns|??8QO=b*GRUn20uyQ1weR zOJXc&7RF6ZIX&rkM5ku*5ZhR&Ne(C(qO?9u!%S31OnUNMqN9q6n^koELac#`9NJ-e zJV2HlW(|K7BeGN^_E8$D*UL}@? zZL&D}LctER1{IGAgwq>Rvv$0K<_k6VbyN&Km=E6ZN(vshi&`JJj=s&-J6=h_19uVl z`Hj!Mkphou)gs_Q&0*%Z+MazS1s~Q5+a6RL$H5l(>?Mw`ex zoaxI1i5jZo{%cfbfpOe?a)O-rQ~|<)?rsb(tbQAHBKQSki)x}o0?3gGc0B4}{{Int z&mDlDwvij1;3oAaiuyw>#ET9%OHapf2l>{p;yv6&S&IeWQD!&G{*G+>vLj(`aRGn}+!(SeK|iB=xt>>}4I$AaE;KzDej*wEc7A5_QW ze1h8^R2<-4!pMg2=pfi63*Y5_br}a7yZtW5E2{WlGFg`T4iY-sk17m8ZO{fcaGaf4 z^vU&uvD;sD!Oz$XGwoj(e$55nAyMu2JG$tkPhG&r{QqNP0#HqlIb+~*i*Vv-xccB& zbnHGjn&5=L;)3sB@C(96-PtS;xm7JgLbx0)Tte7aI9Mn-{Sl{l&92B8B{ggVUp>Qc< zW-l^rFnbZ2b3}&xV4{f3Ay*ojMl6?pW}=ugiMnzkoyRR|Yb1bhNjZf*bblf@AZnL%bwW1GhQ}SY$;n z>Lis3Rc+$mbr8zDPu6|F(tXIkE79ADN4L6S2kv*0S&;TcGv>*mA+iD1Z0aA9|Nmd8 zv&whkSdYuu(xyX}i0yW}tMZ8~=W0G=sDP)Op#YU2QvF|f2&w?*e-5v5;WtdGH(<$2 z2m7Nn-z{wRe^&x5WzgXHqFpeVNxHUd{Ysac+8PH~Q`kzrQ(I`3e}^>r5~!hIa)xSK z@^q+{(nt^hokO3!Lz=}2#7Ki^#|r8Y4Gj;lsGVXiFUx;3vtLS&V3~-XN|WST5D2i& zK_fvRI#4?d1>8h9;hJCp)rG0Va3cX1u#~tvl(#NKD4g4~V_ zaRCcH3nX2&r6;LSESE(9eZdj`YaEOt>h@s$z#p@FdwR<7oh2ks zL2YDOkb^_rYqq?xb#~K#F4MODIgovHwd+@W0A1fG!$%67Nq9lQ{AvPXw+>tyP4F-vJ7E z9|KF;7)F;x@bP1{?Coo+aD@b-mvn7!+2K6}t zk9n-+iPG-rcmS4jNdC{pqfB$5%ySI+RnlZKDVwXBZjO=GxWJqWLKTH?rHByHytcJ5 zvV|4;-iFi@PGIaGzN)c5`)wryAg)sOw}cgm79IhLQrt(fF3zOVLPoK;edND}a;DWzdAyzXFK}kVbT$CKP|I)(I8#k!tO%y zHWjv{9=aqS=Ko)1rW%AY;JpXNtT)IzAF#}YRGUR|j?p@wEKXvYDE~lO1LPg$52)S3 zY6b9cY=)gkUB`DnN`3`KKvC zO$w+%t0qW!4aGp^n+uE<#r<-xeCLZYbX3492SK$=$^R;K*vIx!0%AZ}_x41XQoyA` zLEb6BTN3Lz{R;5}7Qy`gAb>eW8iU{Ywz8ounfx!MSH;1+!b_*Zl0- zN*FX&DKFt6n?)|1Xw%NJ334u&Xp_p)<~5!r2;~eGZp?;YS-PE@o^x$79gl_ww;gFZ z0jn@mC#UHoLnj093iEXW)?z6;7s%HMSSn+vD`jVzSyHBz6vcd*lM1BD1vxEMSWGt6 z7eYE>)?_rUmTrGG>`YEpb*)eT9gTS()4*z6vN(}x6n}jm)^{XD6~7O_I7sT3h;Dm@ z0@-OAxz>qfX3^%7GA_zYf|>-CYTCHn>(4O%{~9U&Y4uD5R)5G8W4Q%P8o5x!L6J|b zf}PQzXQ>+WPP!4G!!$mx4gibJAb6>EG(98O?Fdm_yaSdl;WOfWKc+vhTVkzb!cuyW z34D-IorGFc!JqL#r+uTcOsgRNPydy|n=WM(c~rYWBW)W~(f$z}q)hN2h>wd7Hzp)!>jW#^*G_Ufir;43E=ZfUPnp ziM2B^$U-iWH8$PeQM4PRi4yoo4xB3Pwvh(%19_N8-`6z>p7hcbM|K3v|G(81cr+j} z)~3b-wigO{EI-fEwZhgdwlr27%(JFl{(|~q zy5B%k>C6MIO63R(5x%E$3!%#DgM#+PVDO8)U%IU&J74RJ@s^S9T-@5V6Q0`=MZz<+ zW^vQKE8}exofv9n5&1s;ArW%>A{o^|s`0rpsxRJ=?3?x-rPyS}s>A-q*(A)#gvOim zDLD6h5-9;R0c+2wWHter^s6=X>FY{)#rrRNLE>pSR0|st+Iec@{$(X zORutkMJEBeKy&r>{aN-6Ourucy)i=0g%;xhLhwSHY{TELaX!^1c{K&L%ZmVcR0>#p zUUvzm5>|O|6kws4$`yT(Xa+D#*tQ2+#GzDcYh-~ne(|MLEJV5%!*kGE)9e*2Vf6%_ zMHXN5=?xowGbRC2T=I9R#uio9e*&z*QCb4Ze+hH1RuF_X-*(GwA;t8x1lO-wlqDmn zZ+g;W2tKReOUJfHZaPG-*80Z>smN2}sO25@d#FRWdNM+36;80qM`)|3{Yk&mA8-ay z@e1gvd84HUC$=={Pn5Bo9L=+ zf9gI67$pY*iSeR5Djl$Io@QJRz-W9-Ll4doJikMMB+Y&S>cS*Cbh1$ZDGQ(N$%z7G^GiXrSsqR%8t)SKl8*|VEjj#Ys z`ZO+O(1d;^@WQASge{Z-F78bV$mGBp_a=QB`#EUBR&}LS0Xr5FEvPq&dXqejm-Epy xTGhF|snm7Bb diff --git a/sync/imap/connection.js b/sync/imap/connection.js new file mode 100644 index 000000000..85b4ac6a0 --- /dev/null +++ b/sync/imap/connection.js @@ -0,0 +1,179 @@ +const Imap = require('imap'); +const EventEmitter = require('events'); + +const Capabilities = { + Gmail: 'X-GM-EXT-1', + Quota: 'QUOTA', + UIDPlus: 'UIDPLUS', + Condstore: 'CONDSTORE', + Search: 'ESEARCH', + Sort: 'SORT', +} + +class IMAPConnection extends EventEmitter { + constructor(db, settings) { + super(); + + this._db = db; + this._queue = []; + this._current = null; + this._capabilities = []; + this._imap = Promise.promisifyAll(new Imap(settings)); + + this._imap.once('ready', () => { + for (const key of Object.keys(Capabilities)) { + const val = Capabilities[key]; + if (this._imap.serverSupports(val)) { + this._capabilities.push(val); + } + } + this.emit('ready'); + }); + + this._imap.once('error', (err) => { + console.log(err); + }); + + this._imap.once('end', () => { + console.log('Connection ended'); + }); + + this._imap.on('alert', (msg) => { + console.log(`IMAP SERVER SAYS: ${msg}`) + }) + + // Emitted when new mail arrives in the currently open mailbox. + // Fix https://github.com/mscdex/node-imap/issues/445 + let lastMailEventBox = null; + this._imap.on('mail', () => { + if (lastMailEventBox === this._imap._box.name) { + this.emit('mail'); + } + lastMailEventBox = this._imap._box.name + }); + + // Emitted if the UID validity value for the currently open mailbox + // changes during the current session. + this._imap.on('uidvalidity', () => this.emit('uidvalidity')) + + // Emitted when message metadata (e.g. flags) changes externally. + this._imap.on('update', () => this.emit('update')) + + this._imap.connect(); + } + + openBox(box) { + return this._imap.openBoxAsync(box, true); + } + + getBoxes() { + return this._imap.getBoxesAsync(); + } + + fetch(range, messageReadyCallback) { + return new Promise((resolve, reject) => { + const f = this._imap.fetch(range, { + bodies: ['HEADER', 'TEXT'], + }); + f.on('message', (msg, uid) => + this._receiveMessage(msg, uid, messageReadyCallback)); + f.once('error', reject); + f.once('end', resolve); + }); + } + + fetchMessages(uids, messageReadyCallback) { + if (uids.length === 0) { + return Promise.resolve(); + } + return this.fetch(uids, messageReadyCallback); + } + + fetchUIDAttributes(range) { + return new Promise((resolve, reject) => { + const latestUIDAttributes = {}; + const f = this._imap.fetch(range, {}); + f.on('message', (msg, uid) => { + msg.on('attributes', (attrs) => { + latestUIDAttributes[uid] = attrs; + }) + }); + f.once('error', reject); + f.once('end', () => { + resolve(latestUIDAttributes); + }); + }); + } + + _receiveMessage(msg, uid, callback) { + let attributes = null; + let body = null; + let headers = null; + + msg.on('attributes', (attrs) => { + attributes = attrs; + }); + msg.on('body', (stream, info) => { + const chunks = []; + + stream.on('data', (chunk) => { + chunks.push(chunk); + }); + stream.once('end', () => { + const full = Buffer.concat(chunks).toString('utf8'); + if (info.which === 'HEADER') { + headers = full; + } + if (info.which === 'TEXT') { + body = full; + } + }); + }); + msg.once('end', () => { + callback(attributes, headers, body, uid); + }); + } + + runOperation(operation) { + return new Promise((resolve, reject) => { + this._queue.push({operation, resolve, reject}); + if (this._imap.state === 'authenticated' && !this._current) { + this.processNextOperation(); + } + }); + } + + processNextOperation() { + if (this._current) { return; } + + this._current = this._queue.shift(); + + if (!this._current) { + this.emit('queue-empty'); + return; + } + + const {operation, resolve, reject} = this._current; + + console.log(`Starting task ${operation.description()}`) + const result = operation.run(this._db, this); + if (result instanceof Promise === false) { + throw new Error(`Expected ${operation.constructor.name} to return promise.`); + } + result.catch((err) => { + this._current = null; + console.error(err); + reject(); + }) + .then(() => { + this._current = null; + console.log(`Finished task ${operation.description()}`) + resolve(); + }) + .finally(() => { + this.processNextOperation(); + }); + } +} + +module.exports = IMAPConnection diff --git a/sync/imap/discover-messages-operation.js b/sync/imap/discover-messages-operation.js deleted file mode 100644 index 008aa1345..000000000 --- a/sync/imap/discover-messages-operation.js +++ /dev/null @@ -1,131 +0,0 @@ -class SyncMailboxOperation { - constructor(category) { - this._category = category; - if (!this._category) { - throw new Error("SyncMailboxOperation requires a category") - } - } - - description() { - return `SyncMailboxOperation (${this._category.name})`; - } - - _fetch(imap, range) { - return new Promise((resolve, reject) => { - const f = imap.fetch(range, { - bodies: ['HEADER', 'TEXT'], - }); - f.on('message', (msg, uid) => this._receiveMessage(msg, uid)); - f.once('error', reject); - f.once('end', resolve); - }); - } - - _unlinkAllMessages() { - const {MessageUID} = this._db; - return MessageUID.destroy({ - where: { - CategoryId: this._category.id, - }, - }) - } - - _receiveMessage(msg, uid) { - let attributes = null; - let body = null; - let headers = null; - - msg.on('attributes', (attrs) => { - attributes = attrs; - }); - msg.on('body', (stream, info) => { - const chunks = []; - - stream.on('data', (chunk) => { - chunks.push(chunk); - }); - stream.once('end', () => { - const full = Buffer.concat(chunks).toString('utf8'); - if (info.which === 'HEADER') { - headers = full; - } - if (info.which === 'TEXT') { - body = full; - } - }); - }); - msg.once('end', () => { - this._processMessage(attributes, headers, body, uid); - }); - } - - _processMessage(attributes, headers, body) { - console.log(attributes); - const {Message, MessageUID} = this._db; - - return Message.create({ - unread: attributes.flags.includes('\\Unseen'), - starred: attributes.flags.includes('\\Flagged'), - date: attributes.date, - headers: headers, - body: body, - }).then((model) => { - return MessageUID.create({ - MessageId: model.id, - CategoryId: this._category.id, - flags: attributes.flags, - uid: attributes.uid, - }); - }); - } - - // _flushProcessedMessages() { - // return sequelize.transaction((transaction) => { - // return Promise.props({ - // msgs: Message.bulkCreate(this._processedMessages, {transaction}) - // uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction}) - // }) - // }).then(() => { - // this._processedMessages = []; - // this._processedMessageUIDs = []; - // }); - // } - - run(db, imap) { - this._db = db; - - return imap.openBoxAsync(this._category.name, true).then((box) => { - this._box = box; - - if (box.persistentUIDs === false) { - throw new Error("Mailbox does not support persistentUIDs.") - } - if (box.uidvalidity !== this._category.syncState.uidvalidity) { - return this._unlinkAllMessages(); - } - return Promise.resolve(); - }) - .then(() => { - const savedSyncState = this._category.syncState; - const currentSyncState = { - uidnext: this._box.uidnext, - uidvalidity: this._box.uidvalidity, - } - - let fetchRange = `1:*`; - if (savedSyncState.uidnext) { - if (savedSyncState.uidnext === currentSyncState.uidnext) { - return Promise.resolve(); - } - fetchRange = `${savedSyncState.uidnext}:*` - } - - return this._fetch(imap, fetchRange).then(() => { - this._category.syncState = currentSyncState; - return this._category.save(); - }); - }) - } -} - -module.exports = SyncMailboxOperation; diff --git a/sync/imap/refresh-mailboxes-operation.js b/sync/imap/refresh-mailboxes-operation.js index de531bbab..1efaf9f8a 100644 --- a/sync/imap/refresh-mailboxes-operation.js +++ b/sync/imap/refresh-mailboxes-operation.js @@ -65,9 +65,8 @@ class RefreshMailboxesOperation { run(db, imap) { this._db = db; - this._imap = imap; - return imap.getBoxesAsync().then((boxes) => { + return imap.getBoxes().then((boxes) => { const {Category, sequelize} = this._db; return sequelize.transaction((transaction) => { diff --git a/sync/imap/scan-uids-operation.js b/sync/imap/scan-uids-operation.js deleted file mode 100644 index 615a940db..000000000 --- a/sync/imap/scan-uids-operation.js +++ /dev/null @@ -1,98 +0,0 @@ -class ScanUIDsOperation { - constructor(category) { - this._category = category; - } - - description() { - return `ScanUIDsOperation (${this._category.name})`; - } - - _fetchUIDAttributes(imap, range) { - return new Promise((resolve, reject) => { - const latestUIDAttributes = {}; - const f = imap.fetch(range, {}); - f.on('message', (msg, uid) => { - msg.on('attributes', (attrs) => { - latestUIDAttributes[uid] = attrs; - }) - }); - f.once('error', reject); - f.once('end', () => { - resolve(latestUIDAttributes); - }); - }); - } - - _fetchMessages(uids) { - if (uids.length === 0) { - return Promise.resolve(); - } - console.log(`TODO! NEED TO FETCH UIDS ${uids.join(', ')}`) - return Promise.resolve(); - } - - _removeDeletedMessageUIDs(removedUIDs) { - const {MessageUID} = this._db; - - if (removedUIDs.length === 0) { - return Promise.resolve(); - } - return this._db.sequelize.transaction((transaction) => - MessageUID.destroy({where: {uid: removedUIDs}}, {transaction}) - ); - } - - _deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs) { - const removedUIDs = []; - const neededUIDs = []; - - for (const known of knownUIDs) { - if (!latestUIDAttributes[known.uid]) { - removedUIDs.push(known.uid); - continue; - } - if (latestUIDAttributes[known.uid].flags !== known.flags) { - known.flags = latestUIDAttributes[known.uid].flags; - neededUIDs.push(known.uid); - } - delete latestUIDAttributes[known.uid]; - } - - return { - neededUIDs: neededUIDs.concat(Object.keys(latestUIDAttributes)), - removedUIDs: removedUIDs, - }; - } - - // _flushProcessedMessages() { - // return sequelize.transaction((transaction) => { - // return Promise.props({ - // msgs: Message.bulkCreate(this._processedMessages, {transaction}) - // uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction}) - // }) - // }).then(() => { - // this._processedMessages = []; - // this._processedMessageUIDs = []; - // }); - // } - - run(db, imap) { - this._db = db; - const {MessageUID} = db; - - return imap.openBoxAsync(this._category.name, true).then(() => { - return this._fetchUIDAttributes(imap, `1:*`).then((latestUIDAttributes) => { - return MessageUID.findAll({CategoryId: this._category.id}).then((knownUIDs) => { - const {removedUIDs, neededUIDs} = this._deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs); - - return Promise.props({ - deletes: this._removeDeletedMessageUIDs(removedUIDs), - changes: this._fetchMessages(neededUIDs), - }); - }); - }); - }); - } -} - -module.exports = ScanUIDsOperation; diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js new file mode 100644 index 000000000..3ab16b52d --- /dev/null +++ b/sync/imap/sync-mailbox-operation.js @@ -0,0 +1,152 @@ +const _ = require('underscore'); + +class SyncMailboxOperation { + constructor(category) { + this._category = category; + if (!this._category) { + throw new Error("SyncMailboxOperation requires a category") + } + } + + description() { + return `SyncMailboxOperation (${this._category.name} - ${this._category.id})`; + } + + _unlinkAllMessages() { + const {MessageUID} = this._db; + return MessageUID.destroy({ + where: { + CategoryId: this._category.id, + }, + }) + } + + _removeDeletedMessageUIDs(removedUIDs) { + const {MessageUID} = this._db; + + if (removedUIDs.length === 0) { + return Promise.resolve(); + } + return this._db.sequelize.transaction((transaction) => + MessageUID.destroy({where: {uid: removedUIDs}}, {transaction}) + ); + } + + _deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs) { + const removedUIDs = []; + const neededUIDs = []; + + for (const known of knownUIDs) { + if (!latestUIDAttributes[known.uid]) { + removedUIDs.push(known.uid); + continue; + } + if (!_.isEqual(latestUIDAttributes[known.uid].flags, known.flags)) { + known.flags = latestUIDAttributes[known.uid].flags; + neededUIDs.push(known.uid); + } + + // delete entries from the attributes hash as we go. At the end, + // remaining keys will be the ones that we don't have locally. + delete latestUIDAttributes[known.uid]; + } + + return { + neededUIDs: neededUIDs.concat(Object.keys(latestUIDAttributes)), + removedUIDs: removedUIDs, + }; + } + + _processMessage(attributes, headers, body) { + const {Message, MessageUID} = this._db; + + const hash = Message.hashForHeaders(headers); + + MessageUID.create({ + messageHash: hash, + CategoryId: this._category.id, + flags: attributes.flags, + uid: attributes.uid, + }); + + return Message.create({ + unread: attributes.flags.includes('\\Unseen'), + starred: attributes.flags.includes('\\Flagged'), + date: attributes.date, + headers: headers, + body: body, + }); + } + + _openMailboxAndCheckValidity() { + return this._imap.openBox(this._category.name, true).then((box) => { + this._box = box; + + if (box.persistentUIDs === false) { + throw new Error("Mailbox does not support persistentUIDs.") + } + if (box.uidvalidity !== this._category.syncState.uidvalidity) { + return this._unlinkAllMessages(); + } + return Promise.resolve(); + }) + } + + _fetchUnseenMessages() { + const savedSyncState = this._category.syncState; + const currentSyncState = { + uidnext: this._box.uidnext, + uidvalidity: this._box.uidvalidity, + } + + console.log(" - fetching unseen messages") + + let fetchRange = `1:*`; + if (savedSyncState.uidnext) { + if (savedSyncState.uidnext === currentSyncState.uidnext) { + console.log(" --- nothing more to fetch") + return Promise.resolve(); + } + fetchRange = `${savedSyncState.uidnext}:*` + } + + return this._imap.fetch(fetchRange, this._processMessage.bind(this)).then(() => { + this._category.syncState = currentSyncState; + return this._category.save(); + }); + } + + _fetchChangesToMessages() { + const {MessageUID} = this._db; + + console.log(" - fetching changes to messages") + + return this._imap.fetchUIDAttributes(`1:*`).then((latestUIDAttributes) => { + return MessageUID.findAll({where: {CategoryId: this._category.id}}).then((knownUIDs) => { + const {removedUIDs, neededUIDs} = this._deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs); + + console.log(` - found changed / new UIDs: ${neededUIDs.join(', ')}`) + console.log(` - found removed UIDs: ${removedUIDs.join(', ')}`) + + return Promise.props({ + deletes: this._removeDeletedMessageUIDs(removedUIDs), + changes: this._imap.fetchMessages(neededUIDs, this._processMessage.bind(this)), + }); + }); + }); + } + + run(db, imap) { + this._db = db; + this._imap = imap; + + return this._openMailboxAndCheckValidity() + .then(() => + this._fetchUnseenMessages() + ).then(() => + this._fetchChangesToMessages() + ) + } +} + +module.exports = SyncMailboxOperation; diff --git a/sync/package.json b/sync/package.json index 07c308f66..0dcb6228b 100644 --- a/sync/package.json +++ b/sync/package.json @@ -5,7 +5,8 @@ "main": "app.js", "dependencies": { "bluebird": "^3.4.1", - "imap": "^0.8.17" + "imap": "^0.8.17", + "underscore": "^1.8.3" }, "devDependencies": {}, "scripts": { diff --git a/sync/sync-worker.js b/sync/sync-worker.js index 4850f28cd..ad148afcc 100644 --- a/sync/sync-worker.js +++ b/sync/sync-worker.js @@ -1,120 +1,10 @@ -const Imap = require('imap'); -const EventEmitter = require('events'); - +const IMAPConnection = require('./imap/connection'); const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') -const DiscoverMessagesOperation = require('./imap/discover-messages-operation') -const ScanUIDsOperation = require('./imap/scan-uids-operation') - -const Capabilities = { - Gmail: 'X-GM-EXT-1', - Quota: 'QUOTA', - UIDPlus: 'UIDPLUS', - Condstore: 'CONDSTORE', - Search: 'ESEARCH', - Sort: 'SORT', -} - -class IMAPConnectionStateMachine extends EventEmitter { - constructor(db, settings) { - super(); - - this._db = db; - this._queue = []; - this._current = null; - this._capabilities = []; - this._imap = Promise.promisifyAll(new Imap(settings)); - - this._imap.once('ready', () => { - for (const key of Object.keys(Capabilities)) { - const val = Capabilities[key]; - if (this._imap.serverSupports(val)) { - this._capabilities.push(val); - } - } - this.emit('ready'); - }); - - this._imap.once('error', (err) => { - console.log(err); - }); - - this._imap.once('end', () => { - console.log('Connection ended'); - }); - - this._imap.on('alert', (msg) => { - console.log(`IMAP SERVER SAYS: ${msg}`) - }) - - // Emitted when new mail arrives in the currently open mailbox. - // Fix https://github.com/mscdex/node-imap/issues/445 - let lastMailEventBox = null; - this._imap.on('mail', () => { - if (lastMailEventBox === this._imap._box.name) { - this.emit('mail'); - } - lastMailEventBox = this._imap._box.name - }); - - // Emitted if the UID validity value for the currently open mailbox - // changes during the current session. - this._imap.on('uidvalidity', () => this.emit('uidvalidity')) - - // Emitted when message metadata (e.g. flags) changes externally. - this._imap.on('update', () => this.emit('update')) - - this._imap.connect(); - } - - getIMAP() { - return this._imap; - } - - runOperation(operation) { - return new Promise((resolve, reject) => { - this._queue.push({operation, resolve, reject}); - if (this._imap.state === 'authenticated' && !this._current) { - this.processNextOperation(); - } - }); - } - - processNextOperation() { - if (this._current) { return; } - - this._current = this._queue.shift(); - - if (!this._current) { - this.emit('queue-empty'); - return; - } - - const {operation, resolve, reject} = this._current; - - console.log(`Starting task ${operation.description()}`) - const result = operation.run(this._db, this._imap); - if (result instanceof Promise === false) { - throw new Error(`Expected ${operation.constructor.name} to return promise.`); - } - result.catch((err) => { - this._current = null; - console.error(err); - reject(); - }) - .then(() => { - this._current = null; - console.log(`Finished task ${operation.description()}`) - resolve(); - }) - .finally(() => { - this.processNextOperation(); - }); - } -} +const SyncMailboxOperation = require('./imap/sync-mailbox-operation') class SyncWorker { constructor(account, db) { - const main = new IMAPConnectionStateMachine(db, { + const main = new IMAPConnection(db, { user: 'inboxapptest1@fastmail.fm', password: 'trar2e', host: 'mail.messagingengine.com', @@ -134,13 +24,13 @@ class SyncWorker { throw new Error("Unable to find an inbox category.") } main.on('mail', () => { - main.runOperation(new DiscoverMessagesOperation(inboxCategory)); + main.runOperation(new SyncMailboxOperation(inboxCategory)); }) main.on('update', () => { - main.runOperation(new ScanUIDsOperation(inboxCategory)); + main.runOperation(new SyncMailboxOperation(inboxCategory)); }) main.on('queue-empty', () => { - main.getIMAP().openBoxAsync(inboxCategory.name, true).then(() => { + main.openBox(inboxCategory.name, true).then(() => { console.log("Idling on inbox category"); }); }); @@ -162,8 +52,7 @@ class SyncWorker { return priority.indexOf(b.role) - priority.indexOf(a.role); }) for (const cat of sorted) { - this._main.runOperation(new DiscoverMessagesOperation(cat)); - this._main.runOperation(new ScanUIDsOperation(cat)); + this._main.runOperation(new SyncMailboxOperation(cat)); } }); } From 83dfb664e1d7ffadf376abc59661f4a67251b547 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 20 Jun 2016 14:57:54 -0700 Subject: [PATCH 006/800] Add initial version of message-processor package --- core/database-connection-factory.js | 4 +++ core/models/account/message.js | 1 + message-processor/index.js | 32 +++++++++++++++++++++ message-processor/package.json | 11 +++++++ message-processor/processors/index.js | 10 +++++++ message-processor/processors/quoted-text.js | 3 ++ message-processor/processors/threading.js | 3 ++ sync/imap/sync-mailbox-operation.js | 11 ++----- 8 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 message-processor/index.js create mode 100644 message-processor/package.json create mode 100644 message-processor/processors/index.js create mode 100644 message-processor/processors/quoted-text.js create mode 100644 message-processor/processors/threading.js diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index 59d021487..47d293c91 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -30,6 +30,9 @@ class DatabaseConnectionFactory { } _sequelizeForAccount(accountId) { + if (!accountId) { + throw new Error(`You need to pass an accountId to init the database!`) + } const sequelize = new Sequelize(accountId, '', '', { storage: path.join(STORAGE_DIR, `a-${accountId}.sqlite`), dialect: "sqlite", @@ -41,6 +44,7 @@ class DatabaseConnectionFactory { db.sequelize = sequelize; db.Sequelize = Sequelize; + db.accountId = accountId; return sequelize.authenticate().then(() => sequelize.sync() diff --git a/core/models/account/message.js b/core/models/account/message.js index 9418aaba4..ec78ec946 100644 --- a/core/models/account/message.js +++ b/core/models/account/message.js @@ -5,6 +5,7 @@ module.exports = (sequelize, Sequelize) => { subject: Sequelize.STRING, snippet: Sequelize.STRING, body: Sequelize.STRING, + hash: Sequelize.STRING, headers: Sequelize.STRING, date: Sequelize.DATE, unread: Sequelize.BOOLEAN, diff --git a/message-processor/index.js b/message-processor/index.js new file mode 100644 index 000000000..719f877f7 --- /dev/null +++ b/message-processor/index.js @@ -0,0 +1,32 @@ +const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +const processors = require('./processors') + +function createMessage({headers, body, attributes, hash, db}) { + const {Message} = db + return Message.create({ + hash: hash, + unread: attributes.flags.includes('\\Unseen'), + starred: attributes.flags.includes('\\Flagged'), + date: attributes.date, + headers: headers, + body: body, + }) +} + +function runPipeline(message) { + return processors.reduce((prevPromise, processor) => { + return prevPromise.then((msg) => processor(msg)) + }, Promise.resolve(message)) +} + +function processMessage({headers, body, attributes, hash, accountId}) { + return DatabaseConnectionFactory.forAccount(accountId) + .then((db) => createMessage({headers, body, attributes, hash, db})) + .then((message) => runPipeline(message)) + .then((processedMessage) => processedMessage) + .catch((err) => console.log('oh no')) +} + +module.exports = { + processMessage, +} diff --git a/message-processor/package.json b/message-processor/package.json new file mode 100644 index 000000000..900e7509a --- /dev/null +++ b/message-processor/package.json @@ -0,0 +1,11 @@ +{ + "name": "message-processor", + "version": "1.0.0", + "description": "Message processing pipeline", + "main": "y", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Juan Tejada ", + "license": "ISC" +} diff --git a/message-processor/processors/index.js b/message-processor/processors/index.js new file mode 100644 index 000000000..a13b95158 --- /dev/null +++ b/message-processor/processors/index.js @@ -0,0 +1,10 @@ +const fs = require('fs') +const path = require('path') + +const processors = fs.readdirSync(__dirname) +.filter((file) => file !== 'index.js') +.map((file) => { + return require(`./${file}`).processMessage +}) + +module.exports = {processors} diff --git a/message-processor/processors/quoted-text.js b/message-processor/processors/quoted-text.js new file mode 100644 index 000000000..39ffd71c6 --- /dev/null +++ b/message-processor/processors/quoted-text.js @@ -0,0 +1,3 @@ +module.exports = { + processMessage: (message) => message, +} diff --git a/message-processor/processors/threading.js b/message-processor/processors/threading.js new file mode 100644 index 000000000..39ffd71c6 --- /dev/null +++ b/message-processor/processors/threading.js @@ -0,0 +1,3 @@ +module.exports = { + processMessage: (message) => message, +} diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js index 3ab16b52d..cb69df2f8 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/sync/imap/sync-mailbox-operation.js @@ -1,4 +1,6 @@ const _ = require('underscore'); +const { processMessage } = require(`${__base}/message-processor`) + class SyncMailboxOperation { constructor(category) { @@ -68,14 +70,7 @@ class SyncMailboxOperation { flags: attributes.flags, uid: attributes.uid, }); - - return Message.create({ - unread: attributes.flags.includes('\\Unseen'), - starred: attributes.flags.includes('\\Flagged'), - date: attributes.date, - headers: headers, - body: body, - }); + return processMessage({accountId, attributes, headers, body, hash}) } _openMailboxAndCheckValidity() { From 8ed94442c445110bce6245227478e375486ddcfd Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 20 Jun 2016 16:03:14 -0700 Subject: [PATCH 007/800] Add transaction logging --- core/database-connection-factory.js | 4 ++++ core/models/account/transaction.js | 18 ++++++++++++++++ core/transaction-log.js | 33 +++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 core/models/account/transaction.js create mode 100644 core/transaction-log.js diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index 47d293c91..c054fd32a 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -1,6 +1,7 @@ const Sequelize = require('sequelize'); const fs = require('fs'); const path = require('path'); +const TransactionLog = require('./transaction-log') const STORAGE_DIR = path.join(__base, 'storage'); if (!fs.existsSync(STORAGE_DIR)) { @@ -69,6 +70,9 @@ class DatabaseConnectionFactory { db.sequelize = sequelize; db.Sequelize = Sequelize; + const transactionLog = new TransactionLog(db); + transactionLog.setupSQLHooks(sequelize) + return sequelize.authenticate().then(() => sequelize.sync() ).thenReturn(db); diff --git a/core/models/account/transaction.js b/core/models/account/transaction.js new file mode 100644 index 000000000..6726e72c2 --- /dev/null +++ b/core/models/account/transaction.js @@ -0,0 +1,18 @@ +module.exports = (sequelize, Sequelize) => { + const Transaction = sequelize.define('Transaction', { + type: Sequelize.STRING, + objectId: Sequelize.STRING, + modelName: Sequelize.STRING, + changedFields: { + type: Sequelize.STRING, + get: function get() { + return JSON.parse(this.getDataValue('changedFields')) + }, + set: function set(val) { + this.setDataValue('changedFields', JSON.stringify(val)); + }, + }, + }); + + return Transaction; +}; diff --git a/core/transaction-log.js b/core/transaction-log.js new file mode 100644 index 000000000..f636ac6f5 --- /dev/null +++ b/core/transaction-log.js @@ -0,0 +1,33 @@ +class TransactionLog { + constructor(db) { + this.db = db; + } + + parseHookData({dataValues, _changed, $modelOptions}) { + return { + objectId: dataValues.id, + modelName: $modelOptions.name.singular, + changedFields: _changed, + } + } + + isTransaction({$modelOptions}) { + return $modelOptions.name.singular === "Transaction" + } + + transactionLogger(type) { + return (sequelizeHookData) => { + if (this.isTransaction(sequelizeHookData)) return; + this.db.Transaction.create(Object.assign({type: type}, + this.parseHookData(sequelizeHookData) + )); + } + } + + setupSQLHooks(sequelize) { + sequelize.addHook("afterCreate", this.transactionLogger("create")) + sequelize.addHook("afterUpdate", this.transactionLogger("update")) + sequelize.addHook("afterDelete", this.transactionLogger("delete")) + } +} +module.exports = TransactionLog From cebce1081f4b5486c1e40e15f2f7536b6b3f47d0 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 20 Jun 2016 16:23:09 -0700 Subject: [PATCH 008/800] Fix error & and turn on transaction logging --- core/database-connection-factory.js | 3 +++ sync/imap/sync-mailbox-operation.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index c054fd32a..1d9b64781 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -47,6 +47,9 @@ class DatabaseConnectionFactory { db.Sequelize = Sequelize; db.accountId = accountId; + const transactionLog = new TransactionLog(db); + transactionLog.setupSQLHooks(sequelize) + return sequelize.authenticate().then(() => sequelize.sync() ).thenReturn(db); diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js index cb69df2f8..632f9fccd 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/sync/imap/sync-mailbox-operation.js @@ -60,7 +60,7 @@ class SyncMailboxOperation { } _processMessage(attributes, headers, body) { - const {Message, MessageUID} = this._db; + const {Message, MessageUID, accountId} = this._db; const hash = Message.hashForHeaders(headers); From f995f6fda150a8bba5cf0f23a3db81fe2c042cb1 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 20 Jun 2016 17:28:26 -0700 Subject: [PATCH 009/800] Add delta stream queue on sync side --- core/database-connection-factory.js | 5 +++++ core/delta-stream-queue.js | 29 +++++++++++++++++++++++++++++ core/package.json | 2 ++ core/transaction-log.js | 8 ++++++-- sync/app.js | 24 +++++++++++++++++++----- sync/package.json | 3 +++ 6 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 core/delta-stream-queue.js diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index 1d9b64781..76cebb316 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -2,6 +2,7 @@ const Sequelize = require('sequelize'); const fs = require('fs'); const path = require('path'); const TransactionLog = require('./transaction-log') +const DeltaStreamQueue = require('./delta-stream-queue.js') const STORAGE_DIR = path.join(__base, 'storage'); if (!fs.existsSync(STORAGE_DIR)) { @@ -13,6 +14,10 @@ class DatabaseConnectionFactory { this._pools = {}; } + setup() { + DeltaStreamQueue.setup() + } + _readModelsInDirectory(sequelize, dirname) { const db = {}; for (const filename of fs.readdirSync(dirname)) { diff --git a/core/delta-stream-queue.js b/core/delta-stream-queue.js new file mode 100644 index 000000000..7da47b556 --- /dev/null +++ b/core/delta-stream-queue.js @@ -0,0 +1,29 @@ +const bluebird = require('bluebird') +const redis = require("redis"); +bluebird.promisifyAll(redis.RedisClient.prototype); +bluebird.promisifyAll(redis.Multi.prototype); + +class DeltaStreamQueue { + setup() { + this.client = redis.createClient(); + this.client.on("error", console.error); + this.client.on("ready", () => console.log("Redis ready")); + } + + key(accountId) { + return `delta-${accountId}` + } + + hasSubscribers(accountId) { + return this.client.existsAsync(this.key(accountId)) + } + + notify(accountId, data) { + return this.hasSubscribers(accountId).then((hasSubscribers) => { + if (!hasSubscribers) return Promise.resolve() + return this.client.rpushAsync(this.key(accountId), JSON.stringify(data)) + }) + } +} + +module.exports = new DeltaStreamQueue() diff --git a/core/package.json b/core/package.json index 184b16d50..cd7564760 100644 --- a/core/package.json +++ b/core/package.json @@ -4,7 +4,9 @@ "description": "", "main": "database-connection-factory.js", "dependencies": { + "bluebird": "3.x.x", "mysql": "^2.10.2", + "redis": "2.x.x", "sequelize": "^3.23.3", "sqlite3": "^3.1.4" }, diff --git a/core/transaction-log.js b/core/transaction-log.js index f636ac6f5..2462a2926 100644 --- a/core/transaction-log.js +++ b/core/transaction-log.js @@ -1,3 +1,5 @@ +const DeltaStreamQueue = require('./delta-stream-queue') + class TransactionLog { constructor(db) { this.db = db; @@ -18,9 +20,11 @@ class TransactionLog { transactionLogger(type) { return (sequelizeHookData) => { if (this.isTransaction(sequelizeHookData)) return; - this.db.Transaction.create(Object.assign({type: type}, + const transactionData = Object.assign({type: type}, this.parseHookData(sequelizeHookData) - )); + ); + this.db.Transaction.create(transactionData); + DeltaStreamQueue.notify(this.db.accountId, transactionData) } } diff --git a/sync/app.js b/sync/app.js index 5481a9411..f3f1e31e9 100644 --- a/sync/app.js +++ b/sync/app.js @@ -8,13 +8,27 @@ const DatabaseConnectionFactory = require(`${__base}/core/database-connection-fa const SyncWorkerPool = require('./sync-worker-pool'); const workerPool = new SyncWorkerPool(); -DatabaseConnectionFactory.forShared().then((db) => { - const {Account} = db - Account.findAll().then((accounts) => { - accounts.forEach((account) => { - workerPool.addWorkerForAccount(account); +const RedisServer = require('redis-server'); +const redisServerInstance = new RedisServer(6379); + +const start = () => { + DatabaseConnectionFactory.setup() + DatabaseConnectionFactory.forShared().then((db) => { + const {Account} = db + Account.findAll().then((accounts) => { + accounts.forEach((account) => { + workerPool.addWorkerForAccount(account); + }); }); }); +} + +redisServerInstance.open((error) => { + if (error) { + console.error(error) + process.exit(1); + } + start() }); global.workerPool = workerPool; diff --git a/sync/package.json b/sync/package.json index 0dcb6228b..2524ddb1c 100644 --- a/sync/package.json +++ b/sync/package.json @@ -6,10 +6,13 @@ "dependencies": { "bluebird": "^3.4.1", "imap": "^0.8.17", + "redis": "2.x.x", + "redis-server": "0.x.x", "underscore": "^1.8.3" }, "devDependencies": {}, "scripts": { + "postinstall": "brew install redis", "start": "node app.js", "test": "echo \"Error: no test specified\" && exit 1" }, From 29a6448922c7ba64c7ebbcf378435750e7627a84 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 20 Jun 2016 17:33:23 -0700 Subject: [PATCH 010/800] Initial concept of sync policy --- core/models/shared/account.js | 9 ++ storage/shared.sqlite | Bin 4096 -> 4096 bytes sync/imap/connection.js | 38 +++-- sync/imap/sync-mailbox-operation.js | 60 ++++--- sync/sync-worker-pool.js | 15 ++ sync/sync-worker.js | 235 ++++++++++++++++++++++------ 6 files changed, 278 insertions(+), 79 deletions(-) diff --git a/core/models/shared/account.js b/core/models/shared/account.js index 9195662d9..0f5a56940 100644 --- a/core/models/shared/account.js +++ b/core/models/shared/account.js @@ -1,6 +1,15 @@ module.exports = (sequelize, Sequelize) => { const Account = sequelize.define('Account', { emailAddress: Sequelize.STRING, + syncPolicy: { + type: Sequelize.STRING, + get: function get() { + return JSON.parse(this.getDataValue('syncPolicy')) + }, + set: function set(val) { + this.setDataValue('syncPolicy', JSON.stringify(val)); + }, + }, }, { classMethods: { associate: ({AccountToken}) => { diff --git a/storage/shared.sqlite b/storage/shared.sqlite index 60b0d053d6888e58dfeaa081eb714239b65f4a39..677d35ba3b4e2a681f44ff8cbc09eb8d9345d43a 100644 GIT binary patch delta 146 zcmZorXi%6S&B#1a##xw|LHCgyF9QPuGvhV}W-X>0jN3Lgx-f3O$XLqA8O+EoE-TB} zR5{t7xs@1%dm_Kh8I8CW2CCx`(6!n-9R delta 177 zcmZorXi%6S&B!uQ##xw!LHCgyF9QPuGvh@DW>==qj2AZxvcxlP-o{wU$Z5gIE-ov} z*yKFfgt>IGEtB=+|IF+xPZ^k>0u|k6R+nUCWiaI9OG?d4&o9X@cSz4o%*@eC&d)V8 z1ewXi{E&h9GxI~B;;YP>0!+;6oQ}!K`K5U!A^F* { - for (const key of Object.keys(Capabilities)) { - const val = Capabilities[key]; - if (this._imap.serverSupports(val)) { - this._capabilities.push(val); - } - } - this.emit('ready'); - }); - this._imap.once('error', (err) => { console.log(err); }); @@ -58,8 +48,34 @@ class IMAPConnection extends EventEmitter { // Emitted when message metadata (e.g. flags) changes externally. this._imap.on('update', () => this.emit('update')) + } - this._imap.connect(); + populateCapabilities() { + this._capabilities = []; + for (const key of Object.keys(Capabilities)) { + const val = Capabilities[key]; + if (this._imap.serverSupports(val)) { + this._capabilities.push(val); + } + } + } + + connect() { + if (!this._connectPromise) { + this._connectPromise = new Promise((resolve) => { + this._imap.once('ready', () => { + this.populateCapabilities(); + resolve(); + }); + this._imap.connect(); + }); + } + return this._connectPromise; + } + + end() { + this._queue = []; + this._imap.end(); } openBox(box) { diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js index 632f9fccd..ca7c4f962 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/sync/imap/sync-mailbox-operation.js @@ -3,24 +3,36 @@ const { processMessage } = require(`${__base}/message-processor`) class SyncMailboxOperation { - constructor(category) { + constructor(category, options) { this._category = category; + this._options = options; if (!this._category) { throw new Error("SyncMailboxOperation requires a category") } } description() { - return `SyncMailboxOperation (${this._category.name} - ${this._category.id})`; + return `SyncMailboxOperation (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`; } - _unlinkAllMessages() { + _getLowerBoundUID() { + const {count} = this._options.limit; + return count ? Math.max(1, this._box.uidnext - count) : 1; + } + + _recoverFromUIDInvalidity() { + // UID invalidity means the server has asked us to delete all the UIDs for + // this folder and start from scratch. We let a garbage collector clean up + // actual Messages, because we may just get new UIDs pointing to the same + // messages. const {MessageUID} = this._db; - return MessageUID.destroy({ - where: { - CategoryId: this._category.id, - }, - }) + return this._db.sequelize.transaction((transaction) => + MessageUID.destroy({ + where: { + CategoryId: this._category.id, + }, + }, {transaction}) + ) } _removeDeletedMessageUIDs(removedUIDs) { @@ -30,7 +42,12 @@ class SyncMailboxOperation { return Promise.resolve(); } return this._db.sequelize.transaction((transaction) => - MessageUID.destroy({where: {uid: removedUIDs}}, {transaction}) + MessageUID.destroy({ + where: { + CategoryId: this._category.id, + uid: removedUIDs, + }, + }, {transaction}) ); } @@ -73,7 +90,7 @@ class SyncMailboxOperation { return processMessage({accountId, attributes, headers, body, hash}) } - _openMailboxAndCheckValidity() { + _openMailboxAndEnsureValidity() { return this._imap.openBox(this._category.name, true).then((box) => { this._box = box; @@ -81,7 +98,7 @@ class SyncMailboxOperation { throw new Error("Mailbox does not support persistentUIDs.") } if (box.uidvalidity !== this._category.syncState.uidvalidity) { - return this._unlinkAllMessages(); + return this._recoverFromUIDInvalidity(); } return Promise.resolve(); }) @@ -94,18 +111,18 @@ class SyncMailboxOperation { uidvalidity: this._box.uidvalidity, } - console.log(" - fetching unseen messages") - - let fetchRange = `1:*`; + let range = `${this._getLowerBoundUID()}:*`; if (savedSyncState.uidnext) { if (savedSyncState.uidnext === currentSyncState.uidnext) { console.log(" --- nothing more to fetch") return Promise.resolve(); } - fetchRange = `${savedSyncState.uidnext}:*` + range = `${savedSyncState.uidnext}:*` } - return this._imap.fetch(fetchRange, this._processMessage.bind(this)).then(() => { + console.log(` - fetching unseen messages ${range}`) + + return this._imap.fetch(range, this._processMessage.bind(this)).then(() => { this._category.syncState = currentSyncState; return this._category.save(); }); @@ -113,15 +130,16 @@ class SyncMailboxOperation { _fetchChangesToMessages() { const {MessageUID} = this._db; + const range = `${this._getLowerBoundUID()}:*`; - console.log(" - fetching changes to messages") + console.log(` - fetching changes to messages ${range}`) - return this._imap.fetchUIDAttributes(`1:*`).then((latestUIDAttributes) => { + return this._imap.fetchUIDAttributes(range).then((latestUIDAttributes) => { return MessageUID.findAll({where: {CategoryId: this._category.id}}).then((knownUIDs) => { const {removedUIDs, neededUIDs} = this._deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs); - console.log(` - found changed / new UIDs: ${neededUIDs.join(', ')}`) - console.log(` - found removed UIDs: ${removedUIDs.join(', ')}`) + console.log(` - found changed / new UIDs: ${neededUIDs.join(', ') || 'none'}`) + console.log(` - found removed UIDs: ${removedUIDs.join(', ') || 'none'}`) return Promise.props({ deletes: this._removeDeletedMessageUIDs(removedUIDs), @@ -135,7 +153,7 @@ class SyncMailboxOperation { this._db = db; this._imap = imap; - return this._openMailboxAndCheckValidity() + return this._openMailboxAndEnsureValidity() .then(() => this._fetchUnseenMessages() ).then(() => diff --git a/sync/sync-worker-pool.js b/sync/sync-worker-pool.js index 8423d0ceb..e76b7f4c8 100644 --- a/sync/sync-worker-pool.js +++ b/sync/sync-worker-pool.js @@ -7,6 +7,21 @@ class SyncWorkerPool { } addWorkerForAccount(account) { + account.syncPolicy = { + limit: { + after: Date.now() - 7 * 24 * 60 * 60 * 1000, + count: 10000, + }, + afterSync: 'idle', + folderRecentSync: { + every: 60 * 1000, + }, + folderDeepSync: { + every: 5 * 60 * 1000, + }, + expiration: Date.now() + 60 * 60 * 1000, + } + DatabaseConnectionFactory.forAccount(account.id).then((db) => { this._workers[account.id] = new SyncWorker(account, db); }); diff --git a/sync/sync-worker.js b/sync/sync-worker.js index ad148afcc..88bd1bbd2 100644 --- a/sync/sync-worker.js +++ b/sync/sync-worker.js @@ -1,61 +1,202 @@ const IMAPConnection = require('./imap/connection'); const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') const SyncMailboxOperation = require('./imap/sync-mailbox-operation') +// +// account.syncPolicy = { +// afterSync: 'idle', +// limit: { +// after: Date.now() - 7 * 24 * 60 * 60 * 1000, +// count: 10000, +// }, +// folderRecentSync: { +// every: 60 * 1000, +// }, +// folderDeepSync: { +// every: 5 * 60 * 1000, +// }, +// expiration: Date.now() + 60 * 60 * 1000, +// } class SyncWorker { + constructor(account, db) { - const main = new IMAPConnection(db, { - user: 'inboxapptest1@fastmail.fm', - password: 'trar2e', - host: 'mail.messagingengine.com', - port: 993, - tls: true, - }); - - // Todo: SyncWorker should decide what operations to queue and what params - // to pass them, and how often, based on SyncPolicy model (TBD). - - main.on('ready', () => { - main.runOperation(new RefreshMailboxesOperation()) - .then(() => - this._db.Category.find({where: {role: 'inbox'}}) - ).then((inboxCategory) => { - if (!inboxCategory) { - throw new Error("Unable to find an inbox category.") - } - main.on('mail', () => { - main.runOperation(new SyncMailboxOperation(inboxCategory)); - }) - main.on('update', () => { - main.runOperation(new SyncMailboxOperation(inboxCategory)); - }) - main.on('queue-empty', () => { - main.openBox(inboxCategory.name, true).then(() => { - console.log("Idling on inbox category"); - }); - }); - - setInterval(() => this.syncAllMailboxes(), 120 * 1000); - this.syncAllMailboxes(); - }); - }); - this._db = db; - this._main = main; + this._conn = null; + this._account = account; + this._lastFolderRecentSync = null; + this._lastFolderDeepSync = null; + + this._syncTimer = null; + this._expirationTimer = null; + + this.syncNow(); + this.scheduleExpiration(); } - syncAllMailboxes() { - const {Category} = this._db; - Category.findAll().then((categories) => { - const priority = ['inbox', 'drafts', 'sent']; - const sorted = categories.sort((a, b) => { - return priority.indexOf(b.role) - priority.indexOf(a.role); - }) - for (const cat of sorted) { - this._main.runOperation(new SyncMailboxOperation(cat)); - } + // TODO: How does this get called? + onAccountChanged() { + this.syncNow(); + this.scheduleExpiration(); + } + + onExpired() { + // Returning syncs to the unclaimed queue every so often is healthy. + // TODO: That. + } + + onSyncDidComplete() { + const {afterSync} = this._account.syncPolicy; + + if (afterSync === 'idle') { + this.getInboxCategory().then((inboxCategory) => { + this._conn.openBox(inboxCategory.name, true).then(() => { + console.log(" - Idling on inbox category"); + }); + }); + } else if (afterSync === 'close') { + console.log(" - Closing connection"); + this._conn.end(); + this._conn = null; + } else { + throw new Error(`onSyncDidComplete: Unknown afterSync behavior: ${afterSync}`) + } + } + + onConnectionIdleUpdate() { + this.getInboxCategory((inboxCategory) => { + this._conn.runOperation(new SyncMailboxOperation(inboxCategory, { + scanAllUIDs: false, + limit: this.account.syncPolicy.options, + })); }); } + + getInboxCategory() { + return this._db.Category.find({where: {role: 'inbox'}}) + } + + getCurrentFolderSyncOptionsForPolicy() { + const {folderRecentSync, folderDeepSync, limit} = this._account.syncPolicy; + + if (Date.now() - this._lastFolderDeepSync > folderDeepSync.every) { + return { + mode: 'deep', + options: { + scanAllUIDs: true, + limit: limit, + }, + }; + } + if (Date.now() - this._lastFolderRecentSync > folderRecentSync.every) { + return { + mode: 'shallow', + options: { + scanAllUIDs: false, + limit: limit, + }, + }; + } + return { + mode: 'none', + }; + } + + ensureConnection() { + if (!this._conn) { + const conn = new IMAPConnection(this._db, { + user: 'inboxapptest1@fastmail.fm', + password: 'trar2e', + host: 'mail.messagingengine.com', + port: 993, + tls: true, + }); + conn.on('mail', () => { + this.onConnectionIdleUpdate(); + }) + conn.on('update', () => { + this.onConnectionIdleUpdate(); + }) + conn.on('queue-empty', () => { + }); + + this._conn = conn; + } + + return this._conn.connect(); + } + + queueOperationsForUpdates() { + // todo: syncback operations belong here! + return this._conn.runOperation(new RefreshMailboxesOperation()) + } + + queueOperationsForFolderSyncs() { + const {Category} = this._db; + const {mode, options} = this.getCurrentFolderSyncOptionsForPolicy(); + + if (mode === 'none') { + return Promise.resolve(); + } + + return Category.findAll().then((categories) => { + const priority = ['inbox', 'drafts', 'sent']; + const sorted = categories.sort((a, b) => + priority.indexOf(b.role) - priority.indexOf(a.role) + ) + return Promise.all(sorted.map((cat) => + this._conn.runOperation(new SyncMailboxOperation(cat, options)) + )).then(() => { + if (mode === 'deep') { + this._lastFolderDeepSync = Date.now(); + this._lastFolderRecentSync = Date.now(); + } else if (mode === 'shallow') { + this._lastFolderRecentSync = Date.now(); + } + }); + }); + } + + syncNow() { + clearTimeout(this._syncTimer); + + this.ensureConnection().then(() => + this.queueOperationsForUpdates().then(() => + this.queueOperationsForFolderSyncs() + ) + ).catch((err) => { + // Sync has failed for some reason. What do we do?! + console.error(err); + }).finally(() => { + this.onSyncDidComplete(); + this.scheduleNextSync(); + }); + } + + scheduleExpiration() { + const {expiration} = this._account.syncPolicy; + + clearTimeout(this._expirationTimer); + this._expirationTimer = setTimeout(() => this.onExpired(), expiration); + } + + scheduleNextSync() { + const {folderRecentSync, folderDeepSync} = this._account.syncPolicy; + + let target = Number.MAX_SAFE_INTEGER; + + if (folderRecentSync) { + target = Math.min(target, this._lastFolderRecentSync + folderRecentSync.every); + } + if (folderDeepSync) { + target = Math.min(target, this._lastFolderDeepSync + folderDeepSync.every); + } + + console.log(`Next sync scheduled for ${new Date(target).toLocaleString()}`); + + this._syncTimer = setTimeout(() => { + this.syncNow(); + }, target - Date.now()); + } } module.exports = SyncWorker; From 2d4b17ee52e397b90ff48dfa9511b76b37da7092 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 15:57:50 -0700 Subject: [PATCH 011/800] Add a delta streaming endpoint --- api/app.js | 2 ++ api/routes/delta.js | 37 +++++++++++++++++++++++++++++++++++++ core/delta-stream-queue.js | 13 +++++++------ 3 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 api/routes/delta.js diff --git a/api/app.js b/api/app.js index a763f6a9d..6359393d2 100644 --- a/api/app.js +++ b/api/app.js @@ -64,6 +64,8 @@ server.register(plugins, (err) => { server.auth.strategy('api-consumer', 'basic', { validateFunc: validate }); server.auth.default('api-consumer'); + DatabaseConnectionFactory.setup() + server.start((startErr) => { if (startErr) { throw startErr; } console.log('Server running at:', server.info.uri); diff --git a/api/routes/delta.js b/api/routes/delta.js new file mode 100644 index 000000000..f06f400bf --- /dev/null +++ b/api/routes/delta.js @@ -0,0 +1,37 @@ +const DeltaStreamQueue = require(`${__base}/core/delta-stream-queue`); + +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/delta/streaming', + config: { + description: 'Returns deltas since timestamp then streams deltas', + notes: 'Returns deltas since timestamp then streams deltas', + tags: ['threads'], + validate: { + params: { + }, + }, + response: { + schema: null, + }, + }, + handler: (request, reply) => { + const outputStream = require('stream').Readable(); + outputStream._read = () => { return }; + const pushMsg = (msg = "\n") => outputStream.push(msg) + + request.getAccountDatabase() + .then((db) => { + return db.Transaction.findAll().then((transactions = []) => { + transactions.map(JSON.stringify).forEach(pushMsg); + DeltaStreamQueue.subscribe(db.accountId, pushMsg) + }) + }).then(() => { + const keepAlive = setInterval(pushMsg, 1000); + request.on("disconnect", () => { clearTimeout(keepAlive) }) + return reply(outputStream) + }) + }, + }); +}; diff --git a/core/delta-stream-queue.js b/core/delta-stream-queue.js index 7da47b556..c1a0f87d9 100644 --- a/core/delta-stream-queue.js +++ b/core/delta-stream-queue.js @@ -14,15 +14,16 @@ class DeltaStreamQueue { return `delta-${accountId}` } - hasSubscribers(accountId) { - return this.client.existsAsync(this.key(accountId)) + notify(accountId, data) { + this.client.publish(this.key(accountId), JSON.stringify(data)) } - notify(accountId, data) { - return this.hasSubscribers(accountId).then((hasSubscribers) => { - if (!hasSubscribers) return Promise.resolve() - return this.client.rpushAsync(this.key(accountId), JSON.stringify(data)) + subscribe(accountId, callback) { + this.client.on("message", (channel, message) => { + if (channel !== this.key(accountId)) { return } + callback(message) }) + this.client.subscribe(this.key(accountId)) } } From fa4ed23621ad9df8063718314a7b3a4a5d50a249 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 16:11:47 -0700 Subject: [PATCH 012/800] Put single token in shared db --- storage/shared.sqlite | Bin 4096 -> 4096 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/storage/shared.sqlite b/storage/shared.sqlite index 677d35ba3b4e2a681f44ff8cbc09eb8d9345d43a..cd8f2a479eed16af481d063068916bdf5ff607ac 100644 GIT binary patch delta 98 zcmZorXi%6S&B!uQ#+j9cLH8Ng#*_u@98Am)8JIsaKin+Ha+O(AfQeb1(=j Date: Tue, 21 Jun 2016 16:32:11 -0700 Subject: [PATCH 013/800] Add params --- api/routes/delta.js | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/api/routes/delta.js b/api/routes/delta.js index f06f400bf..428665fdf 100644 --- a/api/routes/delta.js +++ b/api/routes/delta.js @@ -1,29 +1,23 @@ const DeltaStreamQueue = require(`${__base}/core/delta-stream-queue`); +function findParams(queryParams = {}) { + const since = new Date(queryParams.since || Date.now()) + return {where: {createdAt: {$gte: since}}} +} + module.exports = (server) => { server.route({ method: 'GET', path: '/delta/streaming', - config: { - description: 'Returns deltas since timestamp then streams deltas', - notes: 'Returns deltas since timestamp then streams deltas', - tags: ['threads'], - validate: { - params: { - }, - }, - response: { - schema: null, - }, - }, handler: (request, reply) => { const outputStream = require('stream').Readable(); outputStream._read = () => { return }; - const pushMsg = (msg = "\n") => outputStream.push(msg) + const pushMsg = (msg = "\n") => outputStream.push(msg); request.getAccountDatabase() .then((db) => { - return db.Transaction.findAll().then((transactions = []) => { + return db.Transaction.findAll(findParams(request.query)) + .then((transactions = []) => { transactions.map(JSON.stringify).forEach(pushMsg); DeltaStreamQueue.subscribe(db.accountId, pushMsg) }) From 76026207423f70490fa1cad3cac131cea7d77424 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 17:10:34 -0700 Subject: [PATCH 014/800] Add Procfile and heroku readiness --- .gitignore | 1 + Procfile | 2 ++ api/routes/delta.js | 8 ++++---- core/delta-stream-queue.js | 2 +- package.json | 30 ++++++++++++++++++++++++++++++ sync/package.json | 1 - 6 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 Procfile create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 4cd9e26d8..b3a88b97f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store node_modules storage/a-1.sqlite +.env diff --git a/Procfile b/Procfile new file mode 100644 index 000000000..4487bbcab --- /dev/null +++ b/Procfile @@ -0,0 +1,2 @@ +web: node api/app.js +worker: node sync/app.js diff --git a/api/routes/delta.js b/api/routes/delta.js index 428665fdf..36f6cccf2 100644 --- a/api/routes/delta.js +++ b/api/routes/delta.js @@ -12,17 +12,17 @@ module.exports = (server) => { handler: (request, reply) => { const outputStream = require('stream').Readable(); outputStream._read = () => { return }; - const pushMsg = (msg = "\n") => outputStream.push(msg); + const sendMsg = (msg = "\n") => outputStream.push(msg); request.getAccountDatabase() .then((db) => { return db.Transaction.findAll(findParams(request.query)) .then((transactions = []) => { - transactions.map(JSON.stringify).forEach(pushMsg); - DeltaStreamQueue.subscribe(db.accountId, pushMsg) + transactions.map(JSON.stringify).forEach(sendMsg); + DeltaStreamQueue.subscribe(db.accountId, sendMsg) }) }).then(() => { - const keepAlive = setInterval(pushMsg, 1000); + const keepAlive = setInterval(sendMsg, 1000); request.on("disconnect", () => { clearTimeout(keepAlive) }) return reply(outputStream) }) diff --git a/core/delta-stream-queue.js b/core/delta-stream-queue.js index c1a0f87d9..7caf31972 100644 --- a/core/delta-stream-queue.js +++ b/core/delta-stream-queue.js @@ -5,7 +5,7 @@ bluebird.promisifyAll(redis.Multi.prototype); class DeltaStreamQueue { setup() { - this.client = redis.createClient(); + this.client = redis.createClient(process.env.REDIS_URL); this.client.on("error", console.error); this.client.on("ready", () => console.log("Redis ready")); } diff --git a/package.json b/package.json new file mode 100644 index 000000000..9fbcd1b51 --- /dev/null +++ b/package.json @@ -0,0 +1,30 @@ +{ + "name": "k2", + "version": "0.0.1", + "description": "Sync Engine ++", + "main": "", + "dependencies": {}, + "devDependencies": {}, + "scripts": { + "start": "heroku local", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "imap-experiment": "./sync", + "api": "./api" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/nylas/K2.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/nylas/K2/issues" + }, + "homepage": "https://github.com/nylas/K2#readme", + "engines": { + "node": "6.2.2", + "npm": "3.9.5" + } +} diff --git a/sync/package.json b/sync/package.json index 2524ddb1c..320f03fc8 100644 --- a/sync/package.json +++ b/sync/package.json @@ -12,7 +12,6 @@ }, "devDependencies": {}, "scripts": { - "postinstall": "brew install redis", "start": "node app.js", "test": "echo \"Error: no test specified\" && exit 1" }, From 02fbd6e6251035405ad4d84cc12a8d74ca4105f5 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 17:13:48 -0700 Subject: [PATCH 015/800] Add core as package dependency --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 9fbcd1b51..aab3bd8bf 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "imap-experiment": "./sync", + "core": "./core", "api": "./api" }, "repository": { From 05795692f1e8eedb9a451b9ac3b44e38e736b801 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 21 Jun 2016 14:58:20 -0700 Subject: [PATCH 016/800] More sync improvements, use CONDSTORE, gmail coming soon --- core/models/account/category.js | 4 +- core/models/account/message.js | 25 ++- core/models/account/message_uid.js | 29 --- message-processor/index.js | 42 +++-- message-processor/processors/index.js | 1 - sync/imap/connection.js | 38 ++-- sync/imap/refresh-mailboxes-operation.js | 3 + sync/imap/sync-mailbox-operation.js | 219 ++++++++++++++++------- sync/package.json | 3 +- sync/sync-worker-pool.js | 12 +- sync/sync-worker.js | 103 ++++------- 11 files changed, 253 insertions(+), 226 deletions(-) delete mode 100644 core/models/account/message_uid.js diff --git a/core/models/account/category.js b/core/models/account/category.js index b56b5c9d9..39f416829 100644 --- a/core/models/account/category.js +++ b/core/models/account/category.js @@ -14,8 +14,8 @@ module.exports = (sequelize, Sequelize) => { }, }, { classMethods: { - associate: ({MessageUID}) => { - Category.hasMany(MessageUID) + associate: ({Message}) => { + Category.hasMany(Message) }, }, }); diff --git a/core/models/account/message.js b/core/models/account/message.js index ec78ec946..a72d736d1 100644 --- a/core/models/account/message.js +++ b/core/models/account/message.js @@ -2,20 +2,33 @@ const crypto = require('crypto'); module.exports = (sequelize, Sequelize) => { const Message = sequelize.define('Message', { + messageId: Sequelize.STRING, + body: Sequelize.STRING, + rawBody: Sequelize.STRING, + headers: Sequelize.JSONTYPE('headers'), + rawHeaders: Sequelize.STRING, subject: Sequelize.STRING, snippet: Sequelize.STRING, - body: Sequelize.STRING, hash: Sequelize.STRING, - headers: Sequelize.STRING, date: Sequelize.DATE, unread: Sequelize.BOOLEAN, starred: Sequelize.BOOLEAN, + processed: Sequelize.INTEGER, + to: Sequelize.JSONTYPE('to'), + from: Sequelize.JSONTYPE('from'), + cc: Sequelize.JSONTYPE('cc'), + bcc: Sequelize.JSONTYPE('bcc'), + CategoryUID: { type: Sequelize.STRING, allowNull: true}, }, { + indexes: [ + { + unique: true, + fields: ['hash'], + }, + ], classMethods: { - associate: ({MessageUID}) => { - // is this really a good idea? - // Message.hasMany(Contact, {as: 'from'}) - Message.hasMany(MessageUID, {as: 'uids'}) + associate: ({Category}) => { + Message.belongsTo(Category) }, hashForHeaders: (headers) => { return crypto.createHash('sha256').update(headers, 'utf8').digest('hex'); diff --git a/core/models/account/message_uid.js b/core/models/account/message_uid.js deleted file mode 100644 index 0c7a2352d..000000000 --- a/core/models/account/message_uid.js +++ /dev/null @@ -1,29 +0,0 @@ -module.exports = (sequelize, Sequelize) => { - const MessageUID = sequelize.define('MessageUID', { - uid: Sequelize.STRING, - messageHash: Sequelize.STRING, - flags: { - type: Sequelize.STRING, - get: function get() { - return JSON.parse(this.getDataValue('flags')) - }, - set: function set(val) { - this.setDataValue('flags', JSON.stringify(val)); - }, - }, - }, { - indexes: [ - { - unique: true, - fields: ['uid', 'CategoryId', 'messageHash'], - }, - ], - classMethods: { - associate: ({Category}) => { - MessageUID.belongsTo(Category) - }, - }, - }); - - return MessageUID; -}; diff --git a/message-processor/index.js b/message-processor/index.js index 719f877f7..63d594e3a 100644 --- a/message-processor/index.js +++ b/message-processor/index.js @@ -1,17 +1,11 @@ const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) -const processors = require('./processors') +const {processors} = require('./processors') -function createMessage({headers, body, attributes, hash, db}) { - const {Message} = db - return Message.create({ - hash: hash, - unread: attributes.flags.includes('\\Unseen'), - starred: attributes.flags.includes('\\Flagged'), - date: attributes.date, - headers: headers, - body: body, - }) -} +// List of the attributes of Message that the processor should b allowed to change. +// The message may move between folders, get starred, etc. while it's being +// processed, and it shouldn't overwrite changes to those fields. +const MessageAttributes = ['body', 'processed'] +const MessageProcessorVersion = 1; function runPipeline(message) { return processors.reduce((prevPromise, processor) => { @@ -19,12 +13,24 @@ function runPipeline(message) { }, Promise.resolve(message)) } -function processMessage({headers, body, attributes, hash, accountId}) { - return DatabaseConnectionFactory.forAccount(accountId) - .then((db) => createMessage({headers, body, attributes, hash, db})) - .then((message) => runPipeline(message)) - .then((processedMessage) => processedMessage) - .catch((err) => console.log('oh no')) +function processMessage({messageId, accountId}) { + DatabaseConnectionFactory.forAccount(accountId).then((db) => + db.Message.find({where: {id: messageId}}).then((message) => + runPipeline(message) + .then((transformedMessage) => { + transformedMessage.processed = MessageProcessorVersion; + return transformedMessage.save({ + fields: MessageAttributes, + }); + }) + .catch((err) => + console.error(`MessageProcessor Failed: ${err}`) + ) + ) + .catch((err) => + console.error(`MessageProcessor: Couldn't find message id ${messageId} in accountId: ${accountId}: ${err}`) + ) + ) } module.exports = { diff --git a/message-processor/processors/index.js b/message-processor/processors/index.js index a13b95158..2e8c031c8 100644 --- a/message-processor/processors/index.js +++ b/message-processor/processors/index.js @@ -1,5 +1,4 @@ const fs = require('fs') -const path = require('path') const processors = fs.readdirSync(__dirname) .filter((file) => file !== 'index.js') diff --git a/sync/imap/connection.js b/sync/imap/connection.js index a7ec840c9..3097e5102 100644 --- a/sync/imap/connection.js +++ b/sync/imap/connection.js @@ -11,13 +11,13 @@ const Capabilities = { } class IMAPConnection extends EventEmitter { + constructor(db, settings) { super(); this._db = db; this._queue = []; this._current = null; - this._capabilities = []; this._imap = Promise.promisifyAll(new Imap(settings)); this._imap.once('error', (err) => { @@ -50,23 +50,14 @@ class IMAPConnection extends EventEmitter { this._imap.on('update', () => this.emit('update')) } - populateCapabilities() { - this._capabilities = []; - for (const key of Object.keys(Capabilities)) { - const val = Capabilities[key]; - if (this._imap.serverSupports(val)) { - this._capabilities.push(val); - } - } + serverSupports(cap) { + this._imap.serverSupports(cap); } connect() { if (!this._connectPromise) { this._connectPromise = new Promise((resolve) => { - this._imap.once('ready', () => { - this.populateCapabilities(); - resolve(); - }); + this._imap.once('ready', resolve); this._imap.connect(); }); } @@ -86,32 +77,33 @@ class IMAPConnection extends EventEmitter { return this._imap.getBoxesAsync(); } - fetch(range, messageReadyCallback) { + fetch(range, messageCallback) { return new Promise((resolve, reject) => { const f = this._imap.fetch(range, { bodies: ['HEADER', 'TEXT'], }); - f.on('message', (msg, uid) => - this._receiveMessage(msg, uid, messageReadyCallback)); + f.on('message', (msg) => + this._receiveMessage(msg, messageCallback) + ) f.once('error', reject); f.once('end', resolve); }); } - fetchMessages(uids, messageReadyCallback) { + fetchMessages(uids, messageCallback) { if (uids.length === 0) { return Promise.resolve(); } - return this.fetch(uids, messageReadyCallback); + return this.fetch(uids, messageCallback); } fetchUIDAttributes(range) { return new Promise((resolve, reject) => { const latestUIDAttributes = {}; const f = this._imap.fetch(range, {}); - f.on('message', (msg, uid) => { + f.on('message', (msg) => { msg.on('attributes', (attrs) => { - latestUIDAttributes[uid] = attrs; + latestUIDAttributes[attrs.uid] = attrs; }) }); f.once('error', reject); @@ -121,7 +113,7 @@ class IMAPConnection extends EventEmitter { }); } - _receiveMessage(msg, uid, callback) { + _receiveMessage(msg, callback) { let attributes = null; let body = null; let headers = null; @@ -146,7 +138,7 @@ class IMAPConnection extends EventEmitter { }); }); msg.once('end', () => { - callback(attributes, headers, body, uid); + callback(attributes, headers, body); }); } @@ -192,4 +184,6 @@ class IMAPConnection extends EventEmitter { } } +IMAPConnection.Capabilities = Capabilities; + module.exports = IMAPConnection diff --git a/sync/imap/refresh-mailboxes-operation.js b/sync/imap/refresh-mailboxes-operation.js index 1efaf9f8a..f0d475d5c 100644 --- a/sync/imap/refresh-mailboxes-operation.js +++ b/sync/imap/refresh-mailboxes-operation.js @@ -9,6 +9,9 @@ class RefreshMailboxesOperation { '\\Sent': 'sent', '\\Drafts': 'drafts', '\\Junk': 'junk', + '\\Trash': 'trash', + '\\All': 'all', + '\\Important': 'important', '\\Flagged': 'flagged', }[attrib]; if (role) { diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js index ca7c4f962..e1d25a463 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/sync/imap/sync-mailbox-operation.js @@ -1,6 +1,7 @@ -const _ = require('underscore'); -const { processMessage } = require(`${__base}/message-processor`) +const {processMessage} = require(`${__base}/message-processor`); +const {Capabilities} = require('./connection.js'); +const MessageFlagAttributes = ['id', 'CategoryUID', 'unread', 'starred'] class SyncMailboxOperation { constructor(category, options) { @@ -15,79 +16,120 @@ class SyncMailboxOperation { return `SyncMailboxOperation (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`; } - _getLowerBoundUID() { - const {count} = this._options.limit; + _getLowerBoundUID(count) { return count ? Math.max(1, this._box.uidnext - count) : 1; } _recoverFromUIDInvalidity() { - // UID invalidity means the server has asked us to delete all the UIDs for + // UID invalidity means the server has asked us to delete all the UIDs for // this folder and start from scratch. We let a garbage collector clean up // actual Messages, because we may just get new UIDs pointing to the same // messages. - const {MessageUID} = this._db; + const {Message} = this._db; return this._db.sequelize.transaction((transaction) => - MessageUID.destroy({ + Message.update({ + CategoryUID: null, + CategoryId: null, + }, { + transaction: transaction, where: { CategoryId: this._category.id, }, - }, {transaction}) + }) ) } - _removeDeletedMessageUIDs(removedUIDs) { - const {MessageUID} = this._db; + _createAndUpdateMessages(remoteUIDAttributes, localMessageAttributes) { + const messageAttributesMap = {}; + for (const msg of localMessageAttributes) { + messageAttributesMap[msg.CategoryUID] = msg; + } + + const createdUIDs = []; + const changedMessages = []; + + Object.keys(remoteUIDAttributes).forEach((uid) => { + const msg = messageAttributesMap[uid]; + const flags = remoteUIDAttributes[uid].flags; + + if (!msg) { + createdUIDs.push(uid); + return; + } + + const unread = !flags.includes('\\Seen'); + const starred = flags.includes('\\Flagged'); + + if (msg.unread !== unread || msg.starred !== starred) { + msg.unread = unread; + msg.starred = starred; + changedMessages.push(msg); + } + }) + + console.log(` -- found ${createdUIDs.length} new messages`) + console.log(` -- found ${changedMessages.length} flag changes`) + + return Promise.props({ + creates: this._imap.fetchMessages(createdUIDs, this._processMessage.bind(this)), + updates: this._db.sequelize.transaction((transaction) => + Promise.all(changedMessages.map(m => m.save({ + fields: MessageFlagAttributes, + transaction, + }))) + ), + }) + } + + _removeDeletedMessages(remoteUIDAttributes, localMessageAttributes) { + const {Message} = this._db; + + const removedUIDs = localMessageAttributes + .filter(msg => !remoteUIDAttributes[msg.CategoryUID]) + .map(msg => msg.CategoryUID) + + console.log(` -- found ${removedUIDs.length} messages no longer in the folder`) if (removedUIDs.length === 0) { return Promise.resolve(); } return this._db.sequelize.transaction((transaction) => - MessageUID.destroy({ + Message.update({ + CategoryUID: null, + CategoryId: null, + }, { + transaction, where: { CategoryId: this._category.id, - uid: removedUIDs, + CategoryUID: removedUIDs, }, - }, {transaction}) + }) ); } - _deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs) { - const removedUIDs = []; - const neededUIDs = []; - - for (const known of knownUIDs) { - if (!latestUIDAttributes[known.uid]) { - removedUIDs.push(known.uid); - continue; - } - if (!_.isEqual(latestUIDAttributes[known.uid].flags, known.flags)) { - known.flags = latestUIDAttributes[known.uid].flags; - neededUIDs.push(known.uid); - } - - // delete entries from the attributes hash as we go. At the end, - // remaining keys will be the ones that we don't have locally. - delete latestUIDAttributes[known.uid]; - } - - return { - neededUIDs: neededUIDs.concat(Object.keys(latestUIDAttributes)), - removedUIDs: removedUIDs, - }; - } - _processMessage(attributes, headers, body) { - const {Message, MessageUID, accountId} = this._db; + const {Message, accountId} = this._db; const hash = Message.hashForHeaders(headers); - - MessageUID.create({ - messageHash: hash, + const values = { + hash: hash, + rawHeaders: headers, + rawBody: body, + unread: !attributes.flags.includes('\\Seen'), + starred: attributes.flags.includes('\\Flagged'), + date: attributes.date, + CategoryUID: attributes.uid, CategoryId: this._category.id, - flags: attributes.flags, - uid: attributes.uid, - }); - return processMessage({accountId, attributes, headers, body, hash}) + } + Message.find({where: {hash}}).then((existing) => { + if (existing) { + Object.assign(existing, values); + return existing.save(); + } + return Message.create(values).then((created) => { + processMessage({accountId, messageId: created.id, messageBody: body}) + }) + }) } _openMailboxAndEnsureValidity() { @@ -106,47 +148,94 @@ class SyncMailboxOperation { _fetchUnseenMessages() { const savedSyncState = this._category.syncState; - const currentSyncState = { + const boxSyncState = { uidnext: this._box.uidnext, uidvalidity: this._box.uidvalidity, } - let range = `${this._getLowerBoundUID()}:*`; + const {limit} = this._options; + let range = `${this._getLowerBoundUID(limit)}:*`; + + console.log(` - fetching unseen messages ${range}`) + if (savedSyncState.uidnext) { - if (savedSyncState.uidnext === currentSyncState.uidnext) { - console.log(" --- nothing more to fetch") + if (savedSyncState.uidnext === boxSyncState.uidnext) { + console.log(" --- uidnext matches, nothing more to fetch") return Promise.resolve(); } range = `${savedSyncState.uidnext}:*` } - console.log(` - fetching unseen messages ${range}`) - return this._imap.fetch(range, this._processMessage.bind(this)).then(() => { - this._category.syncState = currentSyncState; + console.log(` - finished fetching unseen messages`); + this._category.syncState = Object.assign(this._category.syncState, { + uidnext: boxSyncState.uidnext, + uidvalidity: boxSyncState.uidvalidity, + timeFetchedUnseen: Date.now(), + }); return this._category.save(); }); } _fetchChangesToMessages() { - const {MessageUID} = this._db; - const range = `${this._getLowerBoundUID()}:*`; + const {highestmodseq, timeDeepScan} = this._category.syncState; + const nextHighestmodseq = this._box.highestmodseq; + + const {Message} = this._db; + const {limit} = this._options; + const range = `${this._getLowerBoundUID(limit)}:*`; console.log(` - fetching changes to messages ${range}`) - return this._imap.fetchUIDAttributes(range).then((latestUIDAttributes) => { - return MessageUID.findAll({where: {CategoryId: this._category.id}}).then((knownUIDs) => { - const {removedUIDs, neededUIDs} = this._deltasInUIDsAndFlags(latestUIDAttributes, knownUIDs); + const shouldRunDeepScan = Date.now() - (timeDeepScan || 0) > this._options.deepFolderScan - console.log(` - found changed / new UIDs: ${neededUIDs.join(', ') || 'none'}`) - console.log(` - found removed UIDs: ${removedUIDs.join(', ') || 'none'}`) + if (shouldRunDeepScan) { + return this._imap.fetchUIDAttributes(range).then((remoteUIDAttributes) => + Message.findAll({ + where: {CategoryId: this._category.id}, + attributes: MessageFlagAttributes, + }).then((localMessageAttributes) => + Promise.props({ + upserts: this._createAndUpdateMessages(remoteUIDAttributes, localMessageAttributes), + deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes), + }) + ).then(() => { + this._category.syncState = Object.assign(this._category.syncState, { + highestmodseq: nextHighestmodseq, + timeDeepScan: Date.now(), + timeShallowScan: Date.now(), + }); + return this._category.save(); + }) + ); + } - return Promise.props({ - deletes: this._removeDeletedMessageUIDs(removedUIDs), - changes: this._imap.fetchMessages(neededUIDs, this._processMessage.bind(this)), + let shallowFetch = null; + + if (this._imap.serverSupports(Capabilities.Condstore)) { + if (nextHighestmodseq === highestmodseq) { + console.log(" --- highestmodseq matches, nothing more to fetch") + return Promise.resolve(); + } + shallowFetch = this._imap.fetchUIDAttributes(range, {changedsince: highestmodseq}); + } else { + shallowFetch = this._imap.fetchUIDAttributes(`${this._getLowerBoundUID(1000)}:*`); + } + + return shallowFetch.then((remoteUIDAttributes) => + Message.findAll({ + where: {CategoryId: this._category.id}, + attributes: MessageFlagAttributes, + }).then((localMessageAttributes) => + this._createAndUpdateMessages(remoteUIDAttributes, localMessageAttributes) + ).then(() => { + this._category.syncState = Object.assign(this._category.syncState, { + highestmodseq: nextHighestmodseq, + timeShallowScan: Date.now(), }); - }); - }); + return this._category.save(); + }) + ) } run(db, imap) { diff --git a/sync/package.json b/sync/package.json index 320f03fc8..81af66795 100644 --- a/sync/package.json +++ b/sync/package.json @@ -8,7 +8,8 @@ "imap": "^0.8.17", "redis": "2.x.x", "redis-server": "0.x.x", - "underscore": "^1.8.3" + "underscore": "^1.8.3", + "xoauth2": "^1.1.0" }, "devDependencies": {}, "scripts": { diff --git a/sync/sync-worker-pool.js b/sync/sync-worker-pool.js index e76b7f4c8..2154e5d22 100644 --- a/sync/sync-worker-pool.js +++ b/sync/sync-worker-pool.js @@ -8,16 +8,10 @@ class SyncWorkerPool { addWorkerForAccount(account) { account.syncPolicy = { - limit: { - after: Date.now() - 7 * 24 * 60 * 60 * 1000, - count: 10000, - }, afterSync: 'idle', - folderRecentSync: { - every: 60 * 1000, - }, - folderDeepSync: { - every: 5 * 60 * 1000, + interval: 30 * 1000, + folderSyncOptions: { + deepFolderScan: 5 * 60 * 1000, }, expiration: Date.now() + 60 * 60 * 1000, } diff --git a/sync/sync-worker.js b/sync/sync-worker.js index 88bd1bbd2..9d53fad7a 100644 --- a/sync/sync-worker.js +++ b/sync/sync-worker.js @@ -8,11 +8,9 @@ const SyncMailboxOperation = require('./imap/sync-mailbox-operation') // after: Date.now() - 7 * 24 * 60 * 60 * 1000, // count: 10000, // }, -// folderRecentSync: { -// every: 60 * 1000, -// }, -// folderDeepSync: { -// every: 5 * 60 * 1000, +// interval: 60 * 1000, +// folderSyncOptions: { +// deepFolderScan: 5 * 60 * 1000, // }, // expiration: Date.now() + 60 * 60 * 1000, // } @@ -23,8 +21,7 @@ class SyncWorker { this._db = db; this._conn = null; this._account = account; - this._lastFolderRecentSync = null; - this._lastFolderDeepSync = null; + this._lastSyncTime = null; this._syncTimer = null; this._expirationTimer = null; @@ -63,46 +60,18 @@ class SyncWorker { } onConnectionIdleUpdate() { - this.getInboxCategory((inboxCategory) => { - this._conn.runOperation(new SyncMailboxOperation(inboxCategory, { - scanAllUIDs: false, - limit: this.account.syncPolicy.options, - })); - }); + this.syncNow(); } getInboxCategory() { return this._db.Category.find({where: {role: 'inbox'}}) } - getCurrentFolderSyncOptionsForPolicy() { - const {folderRecentSync, folderDeepSync, limit} = this._account.syncPolicy; - - if (Date.now() - this._lastFolderDeepSync > folderDeepSync.every) { - return { - mode: 'deep', - options: { - scanAllUIDs: true, - limit: limit, - }, - }; - } - if (Date.now() - this._lastFolderRecentSync > folderRecentSync.every) { - return { - mode: 'shallow', - options: { - scanAllUIDs: false, - limit: limit, - }, - }; - } - return { - mode: 'none', - }; - } - ensureConnection() { - if (!this._conn) { + if (this._conn) { + return this._conn.connect(); + } + return new Promise((resolve) => { const conn = new IMAPConnection(this._db, { user: 'inboxapptest1@fastmail.fm', password: 'trar2e', @@ -120,9 +89,8 @@ class SyncWorker { }); this._conn = conn; - } - - return this._conn.connect(); + resolve(this._conn.connect()); + }); } queueOperationsForUpdates() { @@ -132,26 +100,22 @@ class SyncWorker { queueOperationsForFolderSyncs() { const {Category} = this._db; - const {mode, options} = this.getCurrentFolderSyncOptionsForPolicy(); - - if (mode === 'none') { - return Promise.resolve(); - } + const {folderSyncOptions} = this._account.syncPolicy; return Category.findAll().then((categories) => { - const priority = ['inbox', 'drafts', 'sent']; - const sorted = categories.sort((a, b) => - priority.indexOf(b.role) - priority.indexOf(a.role) + const priority = ['inbox', 'drafts', 'sent'].reverse(); + const categoriesToSync = categories.sort((a, b) => + (priority.indexOf(a.role) - priority.indexOf(b.role)) * -1 ) - return Promise.all(sorted.map((cat) => - this._conn.runOperation(new SyncMailboxOperation(cat, options)) + + // const filtered = sorted.filter(cat => + // ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam'].includes(cat.name) + // ) + + return Promise.all(categoriesToSync.map((cat) => + this._conn.runOperation(new SyncMailboxOperation(cat, folderSyncOptions)) )).then(() => { - if (mode === 'deep') { - this._lastFolderDeepSync = Date.now(); - this._lastFolderRecentSync = Date.now(); - } else if (mode === 'shallow') { - this._lastFolderRecentSync = Date.now(); - } + this._lastSyncTime = Date.now(); }); }); } @@ -180,22 +144,15 @@ class SyncWorker { } scheduleNextSync() { - const {folderRecentSync, folderDeepSync} = this._account.syncPolicy; + const {interval} = this._account.syncPolicy; - let target = Number.MAX_SAFE_INTEGER; - - if (folderRecentSync) { - target = Math.min(target, this._lastFolderRecentSync + folderRecentSync.every); + if (interval) { + const target = this._lastSyncTime + interval; + console.log(`Next sync scheduled for ${new Date(target).toLocaleString()}`); + this._syncTimer = setTimeout(() => { + this.syncNow(); + }, target - Date.now()); } - if (folderDeepSync) { - target = Math.min(target, this._lastFolderDeepSync + folderDeepSync.every); - } - - console.log(`Next sync scheduled for ${new Date(target).toLocaleString()}`); - - this._syncTimer = setTimeout(() => { - this.syncNow(); - }, target - Date.now()); } } From b4e05fcb301d5c3bf82d811c714bd6ca04b78372 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 17:17:48 -0700 Subject: [PATCH 017/800] Bind to proper port --- api/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/app.js b/api/app.js index 6359393d2..84484697c 100644 --- a/api/app.js +++ b/api/app.js @@ -9,7 +9,7 @@ const path = require('path'); global.__base = path.join(__dirname, '..') const server = new Hapi.Server(); -server.connection({ port: 3000 }); +server.connection({ port: process.env.PORT || 3000 }); const plugins = [Inert, Vision, HapiBasicAuth, { register: HapiSwagger, From 2170ee41430831914be21212a5124b2ad995bba4 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Tue, 21 Jun 2016 17:15:42 -0700 Subject: [PATCH 018/800] WIP message-processor: building the message --- core/database-connection-factory.js | 1 + core/database-types.js | 12 ++++++++++++ core/models/account/category.js | 11 +---------- 3 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 core/database-types.js diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index 76cebb316..a60cf8400 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -8,6 +8,7 @@ const STORAGE_DIR = path.join(__base, 'storage'); if (!fs.existsSync(STORAGE_DIR)) { fs.mkdirSync(STORAGE_DIR); } +Object.assign(Sequelize, require('./database-connection-types')(Sequelize)) class DatabaseConnectionFactory { constructor() { diff --git a/core/database-types.js b/core/database-types.js new file mode 100644 index 000000000..f851fa489 --- /dev/null +++ b/core/database-types.js @@ -0,0 +1,12 @@ +module.exports = (Sequelize) => ({ + JSONTYPE: (fieldName) => ({ + type: Sequelize.STRING, + defaultValue: '{}', + get: function get() { + return JSON.parse(this.getDataValue('syncState')) + }, + set: function set(val) { + this.setDataValue('syncState', JSON.stringify(val)); + }, + }), +}) diff --git a/core/models/account/category.js b/core/models/account/category.js index 39f416829..cede313b4 100644 --- a/core/models/account/category.js +++ b/core/models/account/category.js @@ -2,16 +2,7 @@ module.exports = (sequelize, Sequelize) => { const Category = sequelize.define('Category', { name: Sequelize.STRING, role: Sequelize.STRING, - syncState: { - type: Sequelize.STRING, - defaultValue: '{}', - get: function get() { - return JSON.parse(this.getDataValue('syncState')) - }, - set: function set(val) { - this.setDataValue('syncState', JSON.stringify(val)); - }, - }, + syncState: Sequelize.JSONTYPE('syncState'), }, { classMethods: { associate: ({Message}) => { From 644af22d40e194462ef94516a18c4dac7767b9ae Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 21 Jun 2016 17:25:25 -0700 Subject: [PATCH 019/800] Fix merge conflicts --- core/database-connection-factory.js | 1 - core/database-types.js | 12 +++++++----- core/models/account/category.js | 4 +++- core/models/account/message.js | 11 ++++++----- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index a60cf8400..76cebb316 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -8,7 +8,6 @@ const STORAGE_DIR = path.join(__base, 'storage'); if (!fs.existsSync(STORAGE_DIR)) { fs.mkdirSync(STORAGE_DIR); } -Object.assign(Sequelize, require('./database-connection-types')(Sequelize)) class DatabaseConnectionFactory { constructor() { diff --git a/core/database-types.js b/core/database-types.js index f851fa489..66259c812 100644 --- a/core/database-types.js +++ b/core/database-types.js @@ -1,12 +1,14 @@ -module.exports = (Sequelize) => ({ - JSONTYPE: (fieldName) => ({ +const Sequelize = require('sequelize'); + +module.exports = { + JSONType: (fieldName) => ({ type: Sequelize.STRING, defaultValue: '{}', get: function get() { - return JSON.parse(this.getDataValue('syncState')) + return JSON.parse(this.getDataValue(fieldName)) }, set: function set(val) { - this.setDataValue('syncState', JSON.stringify(val)); + this.setDataValue(fieldName, JSON.stringify(val)); }, }), -}) +} diff --git a/core/models/account/category.js b/core/models/account/category.js index cede313b4..810a01bbd 100644 --- a/core/models/account/category.js +++ b/core/models/account/category.js @@ -1,8 +1,10 @@ +const {JSONType} = require('../../database-types'); + module.exports = (sequelize, Sequelize) => { const Category = sequelize.define('Category', { name: Sequelize.STRING, role: Sequelize.STRING, - syncState: Sequelize.JSONTYPE('syncState'), + syncState: JSONType('syncState'), }, { classMethods: { associate: ({Message}) => { diff --git a/core/models/account/message.js b/core/models/account/message.js index a72d736d1..8698677a9 100644 --- a/core/models/account/message.js +++ b/core/models/account/message.js @@ -1,11 +1,12 @@ const crypto = require('crypto'); +const {JSONType} = require('../../database-types'); module.exports = (sequelize, Sequelize) => { const Message = sequelize.define('Message', { messageId: Sequelize.STRING, body: Sequelize.STRING, rawBody: Sequelize.STRING, - headers: Sequelize.JSONTYPE('headers'), + headers: JSONType('headers'), rawHeaders: Sequelize.STRING, subject: Sequelize.STRING, snippet: Sequelize.STRING, @@ -14,10 +15,10 @@ module.exports = (sequelize, Sequelize) => { unread: Sequelize.BOOLEAN, starred: Sequelize.BOOLEAN, processed: Sequelize.INTEGER, - to: Sequelize.JSONTYPE('to'), - from: Sequelize.JSONTYPE('from'), - cc: Sequelize.JSONTYPE('cc'), - bcc: Sequelize.JSONTYPE('bcc'), + to: JSONType('to'), + from: JSONType('from'), + cc: JSONType('cc'), + bcc: JSONType('bcc'), CategoryUID: { type: Sequelize.STRING, allowNull: true}, }, { indexes: [ From e75d0ea16be1f94c767c97c5da6c07f452ff4828 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 17:44:09 -0700 Subject: [PATCH 020/800] Add procfile.dev with external redis launcher --- .gitignore | 1 + Procfile.dev | 3 +++ README.md | 30 ++++++++++++++++-------------- core/delta-stream-queue.js | 2 +- sync/app.js | 25 ++++++------------------- 5 files changed, 27 insertions(+), 34 deletions(-) create mode 100644 Procfile.dev diff --git a/.gitignore b/.gitignore index b3a88b97f..2f7bfd77e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules storage/a-1.sqlite .env +dump.rdb diff --git a/Procfile.dev b/Procfile.dev new file mode 100644 index 000000000..71a64c463 --- /dev/null +++ b/Procfile.dev @@ -0,0 +1,3 @@ +redis: redis-server +web: node api/app.js +worker: node sync/app.js diff --git a/README.md b/README.md index 7c9a6c915..eb7e9151c 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,26 @@ # K2 - Sync Engine Experiment # Initial Setup + +1. Download https://toolbelt.heroku.com/ + +2. Install redis for local dev: + ``` +brew install redis +``` + +3. Make sure you're using the latest node: + +``` +nvm install 6 nvm use 6 -cd core -npm instal ``` -# Running the API +4. `npm install` + +# Running locally ``` -cd api -npm install -node app.js -``` - -# Running the Sync Engine - -``` -cd sync -npm install -node app.js +foreman start -f Procfile.dev ``` diff --git a/core/delta-stream-queue.js b/core/delta-stream-queue.js index 7caf31972..ca9d7856d 100644 --- a/core/delta-stream-queue.js +++ b/core/delta-stream-queue.js @@ -5,7 +5,7 @@ bluebird.promisifyAll(redis.Multi.prototype); class DeltaStreamQueue { setup() { - this.client = redis.createClient(process.env.REDIS_URL); + this.client = redis.createClient(process.env.REDIS_URL || null); this.client.on("error", console.error); this.client.on("ready", () => console.log("Redis ready")); } diff --git a/sync/app.js b/sync/app.js index f3f1e31e9..8f0783900 100644 --- a/sync/app.js +++ b/sync/app.js @@ -8,27 +8,14 @@ const DatabaseConnectionFactory = require(`${__base}/core/database-connection-fa const SyncWorkerPool = require('./sync-worker-pool'); const workerPool = new SyncWorkerPool(); -const RedisServer = require('redis-server'); -const redisServerInstance = new RedisServer(6379); - -const start = () => { - DatabaseConnectionFactory.setup() - DatabaseConnectionFactory.forShared().then((db) => { - const {Account} = db - Account.findAll().then((accounts) => { - accounts.forEach((account) => { - workerPool.addWorkerForAccount(account); - }); +DatabaseConnectionFactory.setup() +DatabaseConnectionFactory.forShared().then((db) => { + const {Account} = db + Account.findAll().then((accounts) => { + accounts.forEach((account) => { + workerPool.addWorkerForAccount(account); }); }); -} - -redisServerInstance.open((error) => { - if (error) { - console.error(error) - process.exit(1); - } - start() }); global.workerPool = workerPool; From 647dc1f78c3e725fb831c27f18f4f088ac354a7f Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 21 Jun 2016 17:51:47 -0700 Subject: [PATCH 021/800] Default to port 5100 --- api/app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/app.js b/api/app.js index 84484697c..6fe7e9f28 100644 --- a/api/app.js +++ b/api/app.js @@ -9,7 +9,8 @@ const path = require('path'); global.__base = path.join(__dirname, '..') const server = new Hapi.Server(); -server.connection({ port: process.env.PORT || 3000 }); +console.log(process.env.PORT) +server.connection({ port: process.env.PORT || 5100 }); const plugins = [Inert, Vision, HapiBasicAuth, { register: HapiSwagger, From b7cd644a83b45a621da4e64605e5d293ad67d63b Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 21 Jun 2016 17:51:24 -0700 Subject: [PATCH 022/800] More quietly update category sync state --- sync/imap/sync-mailbox-operation.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js index e1d25a463..e86a4d713 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/sync/imap/sync-mailbox-operation.js @@ -1,3 +1,4 @@ +const _ = require('underscore'); const {processMessage} = require(`${__base}/message-processor`); const {Capabilities} = require('./connection.js'); @@ -168,12 +169,11 @@ class SyncMailboxOperation { return this._imap.fetch(range, this._processMessage.bind(this)).then(() => { console.log(` - finished fetching unseen messages`); - this._category.syncState = Object.assign(this._category.syncState, { + return this.updateCategorySyncState({ uidnext: boxSyncState.uidnext, uidvalidity: boxSyncState.uidvalidity, timeFetchedUnseen: Date.now(), }); - return this._category.save(); }); } @@ -200,12 +200,11 @@ class SyncMailboxOperation { deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes), }) ).then(() => { - this._category.syncState = Object.assign(this._category.syncState, { + return this.updateCategorySyncState({ highestmodseq: nextHighestmodseq, timeDeepScan: Date.now(), timeShallowScan: Date.now(), }); - return this._category.save(); }) ); } @@ -229,15 +228,22 @@ class SyncMailboxOperation { }).then((localMessageAttributes) => this._createAndUpdateMessages(remoteUIDAttributes, localMessageAttributes) ).then(() => { - this._category.syncState = Object.assign(this._category.syncState, { + return this.updateCategorySyncState({ highestmodseq: nextHighestmodseq, timeShallowScan: Date.now(), }); - return this._category.save(); }) ) } + updateCategorySyncState(newState) { + if (_.isMatch(this._category.syncState, newState)) { + return Promise.resolve(); + } + this._category.syncState = Object.assign(this._category.syncState, newState); + return this._category.save(); + } + run(db, imap) { this._db = db; this._imap = imap; From 19de776e008b1b93d633a4e44c567a02a4959c65 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 21 Jun 2016 18:29:58 -0700 Subject: [PATCH 023/800] Encrypt connction credentials, move conn settings to acct --- core/database-connection-factory.js | 3 -- core/models/shared/account-token.js | 5 ++- core/models/shared/account.js | 41 +++++++++++++++---- storage/shared.sqlite | Bin 4096 -> 0 bytes sync/app.js | 61 ++++++++++++++++++++++++---- sync/sync-worker-pool.js | 9 ---- sync/sync-worker.js | 18 ++++---- 7 files changed, 101 insertions(+), 36 deletions(-) delete mode 100644 storage/shared.sqlite diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index 76cebb316..de1b354d9 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -78,9 +78,6 @@ class DatabaseConnectionFactory { db.sequelize = sequelize; db.Sequelize = Sequelize; - const transactionLog = new TransactionLog(db); - transactionLog.setupSQLHooks(sequelize) - return sequelize.authenticate().then(() => sequelize.sync() ).thenReturn(db); diff --git a/core/models/shared/account-token.js b/core/models/shared/account-token.js index a9ff17986..1d3a002f4 100644 --- a/core/models/shared/account-token.js +++ b/core/models/shared/account-token.js @@ -1,6 +1,9 @@ module.exports = (sequelize, Sequelize) => { const AccountToken = sequelize.define('AccountToken', { - value: Sequelize.STRING, + value: { + type: Sequelize.UUID, + defaultValue: Sequelize.UUIDV4, + }, }, { classMethods: { associate: ({Account}) => { diff --git a/core/models/shared/account.js b/core/models/shared/account.js index 0f5a56940..72edad611 100644 --- a/core/models/shared/account.js +++ b/core/models/shared/account.js @@ -1,15 +1,15 @@ +const crypto = require('crypto'); +const {JSONType} = require('../../database-types'); + +const algorithm = 'aes-256-ctr'; +const password = 'd6F3Efeq'; + module.exports = (sequelize, Sequelize) => { const Account = sequelize.define('Account', { emailAddress: Sequelize.STRING, - syncPolicy: { - type: Sequelize.STRING, - get: function get() { - return JSON.parse(this.getDataValue('syncPolicy')) - }, - set: function set(val) { - this.setDataValue('syncPolicy', JSON.stringify(val)); - }, - }, + connectionSettings: JSONType('connectionSettings'), + connectionCredentials: Sequelize.STRING, + syncPolicy: JSONType('syncPolicy'), }, { classMethods: { associate: ({AccountToken}) => { @@ -23,6 +23,29 @@ module.exports = (sequelize, Sequelize) => { email_address: this.emailAddress, } }, + + setCredentials: function setCredentials(json) { + if (!(json instanceof Object)) { + throw new Error("Call setCredentials with JSON!") + } + const cipher = crypto.createCipher(algorithm, password) + let crypted = cipher.update(JSON.stringify(json), 'utf8', 'hex') + crypted += cipher.final('hex'); + + this.connectionCredentials = crypted; + }, + + decryptedCredentials: function decryptedCredentials() { + const decipher = crypto.createDecipher(algorithm, password) + let dec = decipher.update(this.connectionCredentials, 'hex', 'utf8') + dec += decipher.final('utf8'); + + try { + return JSON.parse(dec); + } catch (err) { + return null; + } + }, }, }); diff --git a/storage/shared.sqlite b/storage/shared.sqlite deleted file mode 100644 index cd8f2a479eed16af481d063068916bdf5ff607ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmeH}K~LK-6oBox0x`ru;?PsYyR`v!0K^@Wnk^ztyEL8D(-IORRiw#i(xDzWto$c_ z%dWd{ul5J_0%RfuAtY{$x|pSdPiHT?(ES1J)iV4!jJrWDlf+}4&>aFl^D%A z8?ir>yBMke>eA(EtwyJlr_2l7saASQLApr#O_1f#A4EJ1M>55@Io9bU4mwFR2qw>5 z@7M-CuT9GS5^PEo0YzZ(39Qm3vcA6YaW9v={x$Xo>mNHMihv@pSOhjSR$nRo_pMS^ jD=g{5`9C9Uwphznt5F0Lffop@<){DV=E`22Tw(hMMUd1g diff --git a/sync/app.js b/sync/app.js index 8f0783900..8397cfc8e 100644 --- a/sync/app.js +++ b/sync/app.js @@ -8,14 +8,61 @@ const DatabaseConnectionFactory = require(`${__base}/core/database-connection-fa const SyncWorkerPool = require('./sync-worker-pool'); const workerPool = new SyncWorkerPool(); -DatabaseConnectionFactory.setup() -DatabaseConnectionFactory.forShared().then((db) => { - const {Account} = db - Account.findAll().then((accounts) => { - accounts.forEach((account) => { - workerPool.addWorkerForAccount(account); +const seed = (db) => { + const {Account, AccountToken} = db; + + const account = Account.build({ + emailAddress: 'inboxapptest1@fastmail.fm', + connectionSettings: { + imap: { + host: 'mail.messagingengine.com', + port: 993, + tls: true, + }, + }, + syncPolicy: { + afterSync: 'idle', + interval: 30 * 1000, + folderSyncOptions: { + deepFolderScan: 5 * 60 * 1000, + }, + expiration: Date.now() + 60 * 60 * 1000, + }, + }) + account.setCredentials({ + imap: { + user: 'inboxapptest1@fastmail.fm', + password: 'trar2e', + }, + smtp: { + user: 'inboxapptest1@fastmail.fm', + password: 'trar2e', + }, + }); + return account.save().then((obj) => + AccountToken.create({ + AccountId: obj.id, + }).then((token) => { + console.log(`Created seed data. Your API token is ${token.value}`) + }) + ); +} + +const start = () => { + DatabaseConnectionFactory.forShared().then((db) => { + const {Account} = db; + Account.findAll().then((accounts) => { + if (accounts.length === 0) { + seed(db).then(start); + } + accounts.forEach((account) => { + workerPool.addWorkerForAccount(account); + }); }); }); -}); +} + +DatabaseConnectionFactory.setup() +start(); global.workerPool = workerPool; diff --git a/sync/sync-worker-pool.js b/sync/sync-worker-pool.js index 2154e5d22..8423d0ceb 100644 --- a/sync/sync-worker-pool.js +++ b/sync/sync-worker-pool.js @@ -7,15 +7,6 @@ class SyncWorkerPool { } addWorkerForAccount(account) { - account.syncPolicy = { - afterSync: 'idle', - interval: 30 * 1000, - folderSyncOptions: { - deepFolderScan: 5 * 60 * 1000, - }, - expiration: Date.now() + 60 * 60 * 1000, - } - DatabaseConnectionFactory.forAccount(account.id).then((db) => { this._workers[account.id] = new SyncWorker(account, db); }); diff --git a/sync/sync-worker.js b/sync/sync-worker.js index 9d53fad7a..01e537969 100644 --- a/sync/sync-worker.js +++ b/sync/sync-worker.js @@ -72,13 +72,17 @@ class SyncWorker { return this._conn.connect(); } return new Promise((resolve) => { - const conn = new IMAPConnection(this._db, { - user: 'inboxapptest1@fastmail.fm', - password: 'trar2e', - host: 'mail.messagingengine.com', - port: 993, - tls: true, - }); + const settings = this._account.connectionSettings; + const credentials = this._account.decryptedCredentials(); + + if (!settings || !settings.imap) { + throw new Error("ensureConnection: There are no IMAP connection settings for this account.") + } + if (!credentials || !credentials.imap) { + throw new Error("ensureConnection: There are no IMAP connection credentials for this account.") + } + + const conn = new IMAPConnection(this._db, Object.assign({}, settings.imap, credentials.imap)); conn.on('mail', () => { this.onConnectionIdleUpdate(); }) From c3ed7cbdef414cee607cb3a926526ae7210edaec Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Wed, 22 Jun 2016 10:59:22 -0700 Subject: [PATCH 024/800] Adds message parsing module + other updates - Adds an order for the message processing pipeline to ensure that parsing occurs first - Adds JSONARRAYType - Other misc updates --- core/database-types.js | 10 ++++ core/models/account/message.js | 14 +++--- message-processor/index.js | 32 +++++++------ message-processor/package.json | 8 ++-- message-processor/processors/index.js | 7 ++- message-processor/processors/parsing.js | 53 +++++++++++++++++++++ message-processor/processors/quoted-text.js | 3 +- message-processor/processors/threading.js | 3 +- message-processor/spec/parsing-spec.js | 35 ++++++++++++++ package.json | 19 +++++--- sync/imap/sync-mailbox-operation.js | 5 +- 11 files changed, 153 insertions(+), 36 deletions(-) create mode 100644 message-processor/processors/parsing.js create mode 100644 message-processor/spec/parsing-spec.js diff --git a/core/database-types.js b/core/database-types.js index 66259c812..35f99bf46 100644 --- a/core/database-types.js +++ b/core/database-types.js @@ -11,4 +11,14 @@ module.exports = { this.setDataValue(fieldName, JSON.stringify(val)); }, }), + JSONARRAYType: (fieldName) => ({ + type: Sequelize.STRING, + defaultValue: '[]', + get: function get() { + return JSON.parse(this.getDataValue(fieldName)) + }, + set: function set(val) { + this.setDataValue(fieldName, JSON.stringify(val)); + }, + }), } diff --git a/core/models/account/message.js b/core/models/account/message.js index 8698677a9..e9e66db35 100644 --- a/core/models/account/message.js +++ b/core/models/account/message.js @@ -1,13 +1,13 @@ const crypto = require('crypto'); -const {JSONType} = require('../../database-types'); +const {JSONType, JSONARRAYType} = require('../../database-types'); module.exports = (sequelize, Sequelize) => { const Message = sequelize.define('Message', { + rawBody: Sequelize.STRING, + rawHeaders: Sequelize.STRING, messageId: Sequelize.STRING, body: Sequelize.STRING, - rawBody: Sequelize.STRING, headers: JSONType('headers'), - rawHeaders: Sequelize.STRING, subject: Sequelize.STRING, snippet: Sequelize.STRING, hash: Sequelize.STRING, @@ -15,10 +15,10 @@ module.exports = (sequelize, Sequelize) => { unread: Sequelize.BOOLEAN, starred: Sequelize.BOOLEAN, processed: Sequelize.INTEGER, - to: JSONType('to'), - from: JSONType('from'), - cc: JSONType('cc'), - bcc: JSONType('bcc'), + to: JSONARRAYType('to'), + from: JSONARRAYType('from'), + cc: JSONARRAYType('cc'), + bcc: JSONARRAYType('bcc'), CategoryUID: { type: Sequelize.STRING, allowNull: true}, }, { indexes: [ diff --git a/message-processor/index.js b/message-processor/index.js index 63d594e3a..c50e32bfe 100644 --- a/message-processor/index.js +++ b/message-processor/index.js @@ -1,28 +1,32 @@ const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) const {processors} = require('./processors') -// List of the attributes of Message that the processor should b allowed to change. +// List of the attributes of Message that the processor should be allowed to change. // The message may move between folders, get starred, etc. while it's being // processed, and it shouldn't overwrite changes to those fields. const MessageAttributes = ['body', 'processed'] const MessageProcessorVersion = 1; -function runPipeline(message) { - return processors.reduce((prevPromise, processor) => { - return prevPromise.then((msg) => processor(msg)) - }, Promise.resolve(message)) + +function runPipeline(accountId, message) { + return processors.reduce((prevPromise, processor) => ( + prevPromise.then((msg) => processor({message: msg, accountId})) + ), Promise.resolve(message)) +} + +function saveMessage(message) { + message.processed = MessageProcessorVersion; + return message.save({ + fields: MessageAttributes, + }); } function processMessage({messageId, accountId}) { - DatabaseConnectionFactory.forAccount(accountId).then((db) => - db.Message.find({where: {id: messageId}}).then((message) => - runPipeline(message) - .then((transformedMessage) => { - transformedMessage.processed = MessageProcessorVersion; - return transformedMessage.save({ - fields: MessageAttributes, - }); - }) + DatabaseConnectionFactory.forAccount(accountId) + .then(({Message}) => + Message.find({where: {id: messageId}}).then((message) => + runPipeline(accountId, message) + .then((processedMessage) => saveMessage(processedMessage)) .catch((err) => console.error(`MessageProcessor Failed: ${err}`) ) diff --git a/message-processor/package.json b/message-processor/package.json index 900e7509a..fda89a02e 100644 --- a/message-processor/package.json +++ b/message-processor/package.json @@ -2,10 +2,12 @@ "name": "message-processor", "version": "1.0.0", "description": "Message processing pipeline", - "main": "y", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, - "author": "Juan Tejada ", - "license": "ISC" + "author": "", + "license": "ISC", + "dependencies": { + "mailparser": "^0.6.0" + } } diff --git a/message-processor/processors/index.js b/message-processor/processors/index.js index 2e8c031c8..048a50e1a 100644 --- a/message-processor/processors/index.js +++ b/message-processor/processors/index.js @@ -3,7 +3,12 @@ const fs = require('fs') const processors = fs.readdirSync(__dirname) .filter((file) => file !== 'index.js') .map((file) => { - return require(`./${file}`).processMessage + const {processMessage, order} = require(`./${file}`) + return { + order, + processMessage: processMessage || (msg) => msg, + } }) +.sort(({order: o1}, {order: o2}) => o1 - o2) module.exports = {processors} diff --git a/message-processor/processors/parsing.js b/message-processor/processors/parsing.js new file mode 100644 index 000000000..60fd47842 --- /dev/null +++ b/message-processor/processors/parsing.js @@ -0,0 +1,53 @@ +const {MailParser} = require('mailparser') +const SNIPPET_SIZE = 100 + +function Contact({name, address}) { + return { + name, + email: address, + } +} + +const extractContacts = (field) => (field ? field.map(Contact) : []) + +function processMessage({message}) { + return new Promise((resolve, reject) => { + const {rawHeaders, rawBody} = message + const parser = new MailParser() + parser.on('error', reject) + parser.on('end', (mailObject) => { + const { + html, + text, + subject, + from, + to, + cc, + bcc, + headers, + } = mailObject + + // TODO pull attachments + Object.assign(message, { + subject, + body: html, + headers, + from: extractContacts(from), + to: extractContacts(to), + cc: extractContacts(cc), + bcc: extractContacts(bcc), + messageId: headers['message-id'], + snippet: text.slice(0, SNIPPET_SIZE), + }) + resolve(message) + }); + parser.write(rawHeaders) + parser.write(rawBody); + parser.end(); + }) +} + +module.exports = { + order: 0, + processMessage, +} diff --git a/message-processor/processors/quoted-text.js b/message-processor/processors/quoted-text.js index 39ffd71c6..8216d1bdb 100644 --- a/message-processor/processors/quoted-text.js +++ b/message-processor/processors/quoted-text.js @@ -1,3 +1,4 @@ module.exports = { - processMessage: (message) => message, + order: 2, + processMessage: ({message}) => message, } diff --git a/message-processor/processors/threading.js b/message-processor/processors/threading.js index 39ffd71c6..7812f9346 100644 --- a/message-processor/processors/threading.js +++ b/message-processor/processors/threading.js @@ -1,3 +1,4 @@ module.exports = { - processMessage: (message) => message, + order: 1, + processMessage: ({message}) => message, } diff --git a/message-processor/spec/parsing-spec.js b/message-processor/spec/parsing-spec.js new file mode 100644 index 000000000..ac903a39d --- /dev/null +++ b/message-processor/spec/parsing-spec.js @@ -0,0 +1,35 @@ +const path = require('path') +const fs = require('fs') +const assert = require('assert') +const {processMessage} = require('../processors/parsing') + +const BASE_PATH = path.join('/', 'Users', 'juan', 'Downloads', 'sample data') + +const tests = [] + +function it(name, testFn) { + tests.push(testFn) +} + +function test() { + tests.reduce((prev, t) => prev.then(() => t()), Promise.resolve()) + .then(() => console.log('Success!')) + .catch((err) => console.log(err)) +} + +it('parses the message correctly', () => { + const bodyPath = path.join(BASE_PATH, '1-99174-body.txt') + const headersPath = path.join(BASE_PATH, '1-99174-headers.txt') + const rawBody = fs.readFileSync(bodyPath, 'utf8') + const rawHeaders = fs.readFileSync(headersPath, 'utf8') + const message = { rawHeaders, rawBody } + return processMessage({message}).then((processed) => { + const bodyPart = `` + assert.equal(processed.headers['in-reply-to'], '') + assert.equal(processed.messageId, '') + assert.equal(processed.subject, 'Re: [electron/electron.atom.io] Add Jasper app (#352)') + assert.equal(processed.body.includes(bodyPart), true) + }) +}) + +test() diff --git a/package.json b/package.json index aab3bd8bf..57fdd13c6 100644 --- a/package.json +++ b/package.json @@ -3,17 +3,24 @@ "version": "0.0.1", "description": "Sync Engine ++", "main": "", - "dependencies": {}, - "devDependencies": {}, - "scripts": { - "start": "heroku local", - "test": "echo \"Error: no test specified\" && exit 1" - }, "dependencies": { "imap-experiment": "./sync", "core": "./core", "api": "./api" }, + "devDependencies": { + "babel-eslint": "^6.0.5", + "eslint": "^2.13.1", + "eslint-config-airbnb": "8.0.0", + "eslint-plugin-import": "1.7.0", + "eslint-plugin-jsx-a11y": "1.0.4", + "eslint-plugin-react": "5.0.1", + "eslint_d": "^3.1.1" + }, + "scripts": { + "start": "heroku local", + "test": "echo \"Error: no test specified\" && exit 1" + }, "repository": { "type": "git", "url": "git+https://github.com/nylas/K2.git" diff --git a/sync/imap/sync-mailbox-operation.js b/sync/imap/sync-mailbox-operation.js index e86a4d713..6dbdc026c 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/sync/imap/sync-mailbox-operation.js @@ -127,9 +127,8 @@ class SyncMailboxOperation { Object.assign(existing, values); return existing.save(); } - return Message.create(values).then((created) => { - processMessage({accountId, messageId: created.id, messageBody: body}) - }) + return Message.create(values) + .then((created) => processMessage({accountId, messageId: created.id})) }) } From 8172cad6228322fca4e2eb8213513949a24ca52d Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Wed, 22 Jun 2016 11:01:51 -0700 Subject: [PATCH 025/800] Fix syntax error --- message-processor/processors/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/message-processor/processors/index.js b/message-processor/processors/index.js index 048a50e1a..10cec9b4b 100644 --- a/message-processor/processors/index.js +++ b/message-processor/processors/index.js @@ -6,7 +6,7 @@ const processors = fs.readdirSync(__dirname) const {processMessage, order} = require(`./${file}`) return { order, - processMessage: processMessage || (msg) => msg, + processMessage: processMessage || ((msg) => msg), } }) .sort(({order: o1}, {order: o2}) => o1 - o2) From 5bf63f8ea6be0eacc750fee1f109e474955a517d Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Wed, 22 Jun 2016 11:57:24 -0700 Subject: [PATCH 026/800] Fix message processor pipeline initialization - Was not actually returning an array of functions from `processors/index.js` --- message-processor/processors/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/message-processor/processors/index.js b/message-processor/processors/index.js index 10cec9b4b..1afb1cf1e 100644 --- a/message-processor/processors/index.js +++ b/message-processor/processors/index.js @@ -10,5 +10,6 @@ const processors = fs.readdirSync(__dirname) } }) .sort(({order: o1}, {order: o2}) => o1 - o2) +.map(({processMessage}) => processMessage) module.exports = {processors} From 135f16f8d680ff5c36303514500265b3f48d5446 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 12:16:28 -0700 Subject: [PATCH 027/800] Add initial observable based delta stream endpoint --- .gitignore | 1 + api/package.json | 2 + api/routes/delta.js | 68 +++++++++++++++++++++-------- core/database-connection-factory.js | 2 + core/database-extensions.js | 25 +++++++++++ core/delta-stream-queue.js | 1 - core/package.json | 1 + core/transaction-log.js | 1 + 8 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 core/database-extensions.js diff --git a/.gitignore b/.gitignore index 2f7bfd77e..5a3f69aeb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ node_modules storage/a-1.sqlite .env dump.rdb +*npm-debug.log diff --git a/api/package.json b/api/package.json index 33e8d55fe..5ecb10c7c 100644 --- a/api/package.json +++ b/api/package.json @@ -14,6 +14,8 @@ "hapi-swagger": "^6.1.0", "inert": "^4.0.0", "joi": "^8.4.2", + "rxjs": "5.0.0-beta.9", + "underscore": "1.x.x", "vision": "^4.1.0" } } diff --git a/api/routes/delta.js b/api/routes/delta.js index 36f6cccf2..ee4b1d431 100644 --- a/api/routes/delta.js +++ b/api/routes/delta.js @@ -1,8 +1,45 @@ +const Rx = require('rxjs/Rx') +const _ = require('underscore'); const DeltaStreamQueue = require(`${__base}/core/delta-stream-queue`); -function findParams(queryParams = {}) { - const since = new Date(queryParams.since || Date.now()) - return {where: {createdAt: {$gte: since}}} +function keepAlive(request) { + const until = Rx.Observable.fromCallback(request.on)("disconnect") + return Rx.Observable.interval(1000).map(() => "\n").takeUntil(until) +} + +function inflateTransactions(db, transactions = []) { + const byModel = _.groupBy(transactions, "modelName"); + const byObjectIds = _.groupBy(transactions, "objectId"); + + return Promise.all(Object.keys(byModel).map((modelName) => { + const ids = _.pluck(byModel[modelName], "objectId"); + const ModelKlass = db[modelName] + return ModelKlass.findAll({id: ids}).then((models = []) => { + for (const model of models) { + const tsForId = byObjectIds[model.id]; + if (!tsForId || tsForId.length === 0) { continue; } + for (const t of tsForId) { t.object = model; } + } + }) + })).then(() => transactions) +} + +function createOutputStream() { + const outputStream = require('stream').Readable(); + outputStream._read = () => { return }; + outputStream.pushJSON = (msg) => { + const jsonMsg = typeof msg === 'string' ? msg : JSON.stringify(msg); + outputStream.push(jsonMsg); + } + return outputStream +} + +function initialTransactions(db, request) { + const getParams = request.query || {} + const since = new Date(getParams.since || Date.now()) + return db.Transaction + .streamAll({where: {createdAt: {$gte: since}}}) + .flatMap((objs) => inflateTransactions(db, objs)) } module.exports = (server) => { @@ -10,22 +47,17 @@ module.exports = (server) => { method: 'GET', path: '/delta/streaming', handler: (request, reply) => { - const outputStream = require('stream').Readable(); - outputStream._read = () => { return }; - const sendMsg = (msg = "\n") => outputStream.push(msg); + const outputStream = createOutputStream() - request.getAccountDatabase() - .then((db) => { - return db.Transaction.findAll(findParams(request.query)) - .then((transactions = []) => { - transactions.map(JSON.stringify).forEach(sendMsg); - DeltaStreamQueue.subscribe(db.accountId, sendMsg) - }) - }).then(() => { - const keepAlive = setInterval(sendMsg, 1000); - request.on("disconnect", () => { clearTimeout(keepAlive) }) - return reply(outputStream) - }) + request.getAccountDatabase().then((db) => { + Rx.Observable.merge( + DeltaStreamQueue.fromAccountId(db.accountId), + initialTransactions(db, request), + keepAlive(request) + ).subscribe(outputStream.pushJSON) + }); + + reply(outputStream) }, }); }; diff --git a/core/database-connection-factory.js b/core/database-connection-factory.js index de1b354d9..1675df2d8 100644 --- a/core/database-connection-factory.js +++ b/core/database-connection-factory.js @@ -4,6 +4,8 @@ const path = require('path'); const TransactionLog = require('./transaction-log') const DeltaStreamQueue = require('./delta-stream-queue.js') +require('./database-extensions'); // Extends Sequelize on require + const STORAGE_DIR = path.join(__base, 'storage'); if (!fs.existsSync(STORAGE_DIR)) { fs.mkdirSync(STORAGE_DIR); diff --git a/core/database-extensions.js b/core/database-extensions.js new file mode 100644 index 000000000..c8b926e50 --- /dev/null +++ b/core/database-extensions.js @@ -0,0 +1,25 @@ +const Rx = require('rxjs/Rx'); +const Sequelize = require('sequelize'); + +Sequelize.Model.prototype.streamAll = function streamAll(options = {}) { + return Rx.Observable.create((observer) => { + const chunkSize = options.chunkSize || 1000; + options.offset = 0; + options.limit = chunkSize; + + const findFn = (opts) => { + this.findAll(opts).then((models = []) => { + observer.onNext(models) + if (models.length === chunkSize) { + opts.offset = chunkSize; + findFn(opts) + } else { + observer.onCompleted() + } + }) + } + + findFn(options) + }) +} + diff --git a/core/delta-stream-queue.js b/core/delta-stream-queue.js index ca9d7856d..94ef784f5 100644 --- a/core/delta-stream-queue.js +++ b/core/delta-stream-queue.js @@ -7,7 +7,6 @@ class DeltaStreamQueue { setup() { this.client = redis.createClient(process.env.REDIS_URL || null); this.client.on("error", console.error); - this.client.on("ready", () => console.log("Redis ready")); } key(accountId) { diff --git a/core/package.json b/core/package.json index cd7564760..0fdb37fa3 100644 --- a/core/package.json +++ b/core/package.json @@ -7,6 +7,7 @@ "bluebird": "3.x.x", "mysql": "^2.10.2", "redis": "2.x.x", + "rxjs": "5.0.0-beta.9", "sequelize": "^3.23.3", "sqlite3": "^3.1.4" }, diff --git a/core/transaction-log.js b/core/transaction-log.js index 2462a2926..a9a9bd401 100644 --- a/core/transaction-log.js +++ b/core/transaction-log.js @@ -24,6 +24,7 @@ class TransactionLog { this.parseHookData(sequelizeHookData) ); this.db.Transaction.create(transactionData); + transactionData.object = sequelizeHookData.dataValues DeltaStreamQueue.notify(this.db.accountId, transactionData) } } From 54599a21c0d39830184b12e5f27a99bea82d072a Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 12:19:17 -0700 Subject: [PATCH 028/800] Ignore storage folder --- .gitignore | 2 +- package.json | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 5a3f69aeb..f4fc643a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .DS_Store node_modules -storage/a-1.sqlite .env dump.rdb *npm-debug.log +storage/ diff --git a/package.json b/package.json index 57fdd13c6..924e7cfd4 100644 --- a/package.json +++ b/package.json @@ -6,19 +6,20 @@ "dependencies": { "imap-experiment": "./sync", "core": "./core", - "api": "./api" + "api": "./api", + "message-processor": "./message-processor" }, "devDependencies": { - "babel-eslint": "^6.0.5", - "eslint": "^2.13.1", - "eslint-config-airbnb": "8.0.0", - "eslint-plugin-import": "1.7.0", - "eslint-plugin-jsx-a11y": "1.0.4", - "eslint-plugin-react": "5.0.1", - "eslint_d": "^3.1.1" + "babel-eslint": "6.x", + "eslint": "2.x", + "eslint-config-airbnb": "8.x", + "eslint-plugin-import": "1.x", + "eslint-plugin-jsx-a11y": "1.x", + "eslint-plugin-react": "5.x", + "eslint_d": "3.x" }, "scripts": { - "start": "heroku local", + "start": "heroku local -f Procfile.dev", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { From 02f07882327e17c41e082c7d5d54246bf019e53b Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 13:35:55 -0700 Subject: [PATCH 029/800] Improving observable based delta stream --- api/package.json | 2 +- api/routes/delta.js | 8 +++++--- core/database-extensions.js | 2 +- core/delta-stream-queue.js | 14 +++++++++----- core/package.json | 2 +- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/api/package.json b/api/package.json index 5ecb10c7c..c6509d9d5 100644 --- a/api/package.json +++ b/api/package.json @@ -14,7 +14,7 @@ "hapi-swagger": "^6.1.0", "inert": "^4.0.0", "joi": "^8.4.2", - "rxjs": "5.0.0-beta.9", + "rx": "4.x.x", "underscore": "1.x.x", "vision": "^4.1.0" } diff --git a/api/routes/delta.js b/api/routes/delta.js index ee4b1d431..b0c9c7a49 100644 --- a/api/routes/delta.js +++ b/api/routes/delta.js @@ -1,4 +1,4 @@ -const Rx = require('rxjs/Rx') +const Rx = require('rx') const _ = require('underscore'); const DeltaStreamQueue = require(`${__base}/core/delta-stream-queue`); @@ -47,14 +47,16 @@ module.exports = (server) => { method: 'GET', path: '/delta/streaming', handler: (request, reply) => { - const outputStream = createOutputStream() + const outputStream = createOutputStream(); request.getAccountDatabase().then((db) => { - Rx.Observable.merge( + const source = Rx.Observable.merge( DeltaStreamQueue.fromAccountId(db.accountId), initialTransactions(db, request), keepAlive(request) ).subscribe(outputStream.pushJSON) + + request.on("disconnect", () => source.dispose()); }); reply(outputStream) diff --git a/core/database-extensions.js b/core/database-extensions.js index c8b926e50..4c64073fb 100644 --- a/core/database-extensions.js +++ b/core/database-extensions.js @@ -1,4 +1,4 @@ -const Rx = require('rxjs/Rx'); +const Rx = require('rx'); const Sequelize = require('sequelize'); Sequelize.Model.prototype.streamAll = function streamAll(options = {}) { diff --git a/core/delta-stream-queue.js b/core/delta-stream-queue.js index 94ef784f5..b85102a2f 100644 --- a/core/delta-stream-queue.js +++ b/core/delta-stream-queue.js @@ -1,3 +1,4 @@ +const Rx = require('rx') const bluebird = require('bluebird') const redis = require("redis"); bluebird.promisifyAll(redis.RedisClient.prototype); @@ -17,12 +18,15 @@ class DeltaStreamQueue { this.client.publish(this.key(accountId), JSON.stringify(data)) } - subscribe(accountId, callback) { - this.client.on("message", (channel, message) => { - if (channel !== this.key(accountId)) { return } - callback(message) + fromAccountId(accountId) { + return Rx.Observable.create((observer) => { + this.client.on("message", (channel, message) => { + if (channel !== this.key(accountId)) { return } + observer.onNext(message) + }); + this.client.subscribe(this.key(accountId)); + return () => { this.client.unsubscribe() } }) - this.client.subscribe(this.key(accountId)) } } diff --git a/core/package.json b/core/package.json index 0fdb37fa3..3e307326b 100644 --- a/core/package.json +++ b/core/package.json @@ -7,7 +7,7 @@ "bluebird": "3.x.x", "mysql": "^2.10.2", "redis": "2.x.x", - "rxjs": "5.0.0-beta.9", + "rx": "4.x.x", "sequelize": "^3.23.3", "sqlite3": "^3.1.4" }, From 2c5da59d744feeb5c8af4b785b7d1f14b2b10d19 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 13:57:51 -0700 Subject: [PATCH 030/800] Properly pluck out transactionModel --- api/routes/delta.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/routes/delta.js b/api/routes/delta.js index b0c9c7a49..77373b5fb 100644 --- a/api/routes/delta.js +++ b/api/routes/delta.js @@ -7,7 +7,8 @@ function keepAlive(request) { return Rx.Observable.interval(1000).map(() => "\n").takeUntil(until) } -function inflateTransactions(db, transactions = []) { +function inflateTransactions(db, transactionModels = []) { + const transactions = _.pluck(transactionModels, "dataValues") const byModel = _.groupBy(transactions, "modelName"); const byObjectIds = _.groupBy(transactions, "objectId"); From 18f2925b434b1e018d433cae4f9a12b66c6dec6c Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 14:41:32 -0700 Subject: [PATCH 031/800] Convert to monorepo --- .eslintrc | 3 +-- Procfile | 4 ++-- Procfile.dev | 4 ++-- api/package.json | 21 ---------------- core/package.json | 19 --------------- lerna.json | 4 ++++ message-processor/package.json | 13 ---------- package.json | 22 ++++++++++------- {api => packages/nylas-api}/app.js | 3 +-- .../nylas-api}/decorators/connections.js | 2 +- packages/nylas-api/package.json | 19 +++++++++++++++ .../nylas-api}/routes/accounts.js | 0 {api => packages/nylas-api}/routes/delta.js | 2 +- {api => packages/nylas-api}/routes/threads.js | 0 {api => packages/nylas-api}/serialization.js | 0 .../nylas-core}/config/development.json | 0 .../database-connection-factory.js | 2 +- .../nylas-core}/database-extensions.js | 0 .../nylas-core}/database-types.js | 0 .../nylas-core}/delta-stream-queue.js | 0 packages/nylas-core/index.js | 5 ++++ .../migrations/20160617002207-create-user.js | 0 .../nylas-core}/models/account/category.js | 0 .../nylas-core}/models/account/message.js | 0 .../nylas-core}/models/account/thread.js | 0 .../nylas-core}/models/account/transaction.js | 0 .../models/shared/account-token.js | 0 .../nylas-core}/models/shared/account.js | 0 packages/nylas-core/package.json | 11 +++++++++ .../nylas-core}/transaction-log.js | 0 .../nylas-message-processor}/index.js | 2 +- packages/nylas-message-processor/package.json | 12 ++++++++++ .../processors/index.js | 0 .../processors/parsing.js | 0 .../processors/quoted-text.js | 0 .../processors/threading.js | 0 .../spec/parsing-spec.js | 0 {sync => packages/nylas-sync}/app.js | 6 +---- .../nylas-sync}/imap/connection.js | 0 .../imap/refresh-mailboxes-operation.js | 0 .../imap/sync-mailbox-operation.js | 2 +- packages/nylas-sync/package.json | 16 +++++++++++++ .../nylas-sync}/sync-worker-pool.js | 2 +- {sync => packages/nylas-sync}/sync-worker.js | 0 process.json | 24 ------------------- sync/package.json | 21 ---------------- 46 files changed, 93 insertions(+), 126 deletions(-) delete mode 100644 api/package.json delete mode 100644 core/package.json create mode 100644 lerna.json delete mode 100644 message-processor/package.json rename {api => packages/nylas-api}/app.js (92%) rename {api => packages/nylas-api}/decorators/connections.js (72%) create mode 100644 packages/nylas-api/package.json rename {api => packages/nylas-api}/routes/accounts.js (100%) rename {api => packages/nylas-api}/routes/delta.js (96%) rename {api => packages/nylas-api}/routes/threads.js (100%) rename {api => packages/nylas-api}/serialization.js (100%) rename {core => packages/nylas-core}/config/development.json (100%) rename {core => packages/nylas-core}/database-connection-factory.js (97%) rename {core => packages/nylas-core}/database-extensions.js (100%) rename {core => packages/nylas-core}/database-types.js (100%) rename {core => packages/nylas-core}/delta-stream-queue.js (100%) create mode 100644 packages/nylas-core/index.js rename {core => packages/nylas-core}/migrations/20160617002207-create-user.js (100%) rename {core => packages/nylas-core}/models/account/category.js (100%) rename {core => packages/nylas-core}/models/account/message.js (100%) rename {core => packages/nylas-core}/models/account/thread.js (100%) rename {core => packages/nylas-core}/models/account/transaction.js (100%) rename {core => packages/nylas-core}/models/shared/account-token.js (100%) rename {core => packages/nylas-core}/models/shared/account.js (100%) create mode 100644 packages/nylas-core/package.json rename {core => packages/nylas-core}/transaction-log.js (100%) rename {message-processor => packages/nylas-message-processor}/index.js (93%) create mode 100644 packages/nylas-message-processor/package.json rename {message-processor => packages/nylas-message-processor}/processors/index.js (100%) rename {message-processor => packages/nylas-message-processor}/processors/parsing.js (100%) rename {message-processor => packages/nylas-message-processor}/processors/quoted-text.js (100%) rename {message-processor => packages/nylas-message-processor}/processors/threading.js (100%) rename {message-processor => packages/nylas-message-processor}/spec/parsing-spec.js (100%) rename {sync => packages/nylas-sync}/app.js (85%) rename {sync => packages/nylas-sync}/imap/connection.js (100%) rename {sync => packages/nylas-sync}/imap/refresh-mailboxes-operation.js (100%) rename {sync => packages/nylas-sync}/imap/sync-mailbox-operation.js (99%) create mode 100644 packages/nylas-sync/package.json rename {sync => packages/nylas-sync}/sync-worker-pool.js (78%) rename {sync => packages/nylas-sync}/sync-worker.js (100%) delete mode 100644 process.json delete mode 100644 sync/package.json diff --git a/.eslintrc b/.eslintrc index 002cc8677..3b98b88de 100644 --- a/.eslintrc +++ b/.eslintrc @@ -8,8 +8,7 @@ "advanceClock": false, "TEST_ACCOUNT_ID": false, "TEST_ACCOUNT_NAME": false, - "TEST_ACCOUNT_EMAIL": false, - "__base": false + "TEST_ACCOUNT_EMAIL": false }, "env": { "browser": true, diff --git a/Procfile b/Procfile index 4487bbcab..bc443adf9 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: node api/app.js -worker: node sync/app.js +web: node packages/nylas-api/app.js +worker: node packages/nylas-sync/app.js diff --git a/Procfile.dev b/Procfile.dev index 71a64c463..62b30759f 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,3 +1,3 @@ redis: redis-server -web: node api/app.js -worker: node sync/app.js +web: node packages/nylas-api/app.js +worker: node packages/nylas-sync/app.js diff --git a/api/package.json b/api/package.json deleted file mode 100644 index c6509d9d5..000000000 --- a/api/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "api", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC", - "dependencies": { - "hapi": "^13.4.1", - "hapi-auth-basic": "^4.2.0", - "hapi-swagger": "^6.1.0", - "inert": "^4.0.0", - "joi": "^8.4.2", - "rx": "4.x.x", - "underscore": "1.x.x", - "vision": "^4.1.0" - } -} diff --git a/core/package.json b/core/package.json deleted file mode 100644 index 3e307326b..000000000 --- a/core/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "core", - "version": "1.0.0", - "description": "", - "main": "database-connection-factory.js", - "dependencies": { - "bluebird": "3.x.x", - "mysql": "^2.10.2", - "redis": "2.x.x", - "rx": "4.x.x", - "sequelize": "^3.23.3", - "sqlite3": "^3.1.4" - }, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC" -} diff --git a/lerna.json b/lerna.json new file mode 100644 index 000000000..56dbae74a --- /dev/null +++ b/lerna.json @@ -0,0 +1,4 @@ +{ + "lerna": "2.0.0-beta.20", + "version": "0.0.1" +} diff --git a/message-processor/package.json b/message-processor/package.json deleted file mode 100644 index fda89a02e..000000000 --- a/message-processor/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "message-processor", - "version": "1.0.0", - "description": "Message processing pipeline", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC", - "dependencies": { - "mailparser": "^0.6.0" - } -} diff --git a/package.json b/package.json index 924e7cfd4..d75708d33 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,17 @@ { "name": "k2", "version": "0.0.1", - "description": "Sync Engine ++", + "description": "k2", "main": "", "dependencies": { - "imap-experiment": "./sync", - "core": "./core", - "api": "./api", - "message-processor": "./message-processor" + "bluebird": "3.x.x", + "nylas-core": "0.x.x", + "nylas-api": "0.x.x", + "nylas-sync": "0.x.x", + "redis": "2.x.x", + "rx": "4.x.x", + "sequelize": "3.x.x", + "underscore": "1.x.x" }, "devDependencies": { "babel-eslint": "6.x", @@ -16,17 +20,17 @@ "eslint-plugin-import": "1.x", "eslint-plugin-jsx-a11y": "1.x", "eslint-plugin-react": "5.x", - "eslint_d": "3.x" + "eslint_d": "3.x", + "lerna": "2.0.0-beta.20" }, "scripts": { - "start": "heroku local -f Procfile.dev", - "test": "echo \"Error: no test specified\" && exit 1" + "start": "heroku local -f Procfile.dev" }, "repository": { "type": "git", "url": "git+https://github.com/nylas/K2.git" }, - "author": "", + "author": "Nylas", "license": "ISC", "bugs": { "url": "https://github.com/nylas/K2/issues" diff --git a/api/app.js b/packages/nylas-api/app.js similarity index 92% rename from api/app.js rename to packages/nylas-api/app.js index 6fe7e9f28..2b36fdd72 100644 --- a/api/app.js +++ b/packages/nylas-api/app.js @@ -6,7 +6,6 @@ const Vision = require('vision'); const Package = require('./package'); const fs = require('fs'); const path = require('path'); -global.__base = path.join(__dirname, '..') const server = new Hapi.Server(); console.log(process.env.PORT) @@ -23,7 +22,7 @@ const plugins = [Inert, Vision, HapiBasicAuth, { }]; let sharedDb = null; -const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +const {DatabaseConnectionFactory} = require(`nylas-core`) DatabaseConnectionFactory.forShared().then((db) => { sharedDb = db; }); diff --git a/api/decorators/connections.js b/packages/nylas-api/decorators/connections.js similarity index 72% rename from api/decorators/connections.js rename to packages/nylas-api/decorators/connections.js index 509d969ee..2190f2c67 100644 --- a/api/decorators/connections.js +++ b/packages/nylas-api/decorators/connections.js @@ -1,6 +1,6 @@ /* eslint func-names:0 */ -const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`); +const {DatabaseConnectionFactory} = require(`nylas-core`); module.exports = (server) => { server.decorate('request', 'getAccountDatabase', function () { diff --git a/packages/nylas-api/package.json b/packages/nylas-api/package.json new file mode 100644 index 000000000..d023ff6f6 --- /dev/null +++ b/packages/nylas-api/package.json @@ -0,0 +1,19 @@ +{ + "name": "nylas-api", + "version": "0.0.1", + "description": "Nylas API", + "scripts": { + "start": "node app.js" + }, + "author": "Nylas", + "license": "ISC", + "dependencies": { + "hapi": "13.4.1", + "hapi-auth-basic": "4.2.0", + "hapi-swagger": "6.1.0", + "inert": "4.0.0", + "joi": "8.4.2", + "nylas-core": "0.x.x", + "vision": "4.1.0" + } +} diff --git a/api/routes/accounts.js b/packages/nylas-api/routes/accounts.js similarity index 100% rename from api/routes/accounts.js rename to packages/nylas-api/routes/accounts.js diff --git a/api/routes/delta.js b/packages/nylas-api/routes/delta.js similarity index 96% rename from api/routes/delta.js rename to packages/nylas-api/routes/delta.js index 77373b5fb..c98471e19 100644 --- a/api/routes/delta.js +++ b/packages/nylas-api/routes/delta.js @@ -1,6 +1,6 @@ const Rx = require('rx') const _ = require('underscore'); -const DeltaStreamQueue = require(`${__base}/core/delta-stream-queue`); +const {DeltaStreamQueue} = require(`nylas-core`); function keepAlive(request) { const until = Rx.Observable.fromCallback(request.on)("disconnect") diff --git a/api/routes/threads.js b/packages/nylas-api/routes/threads.js similarity index 100% rename from api/routes/threads.js rename to packages/nylas-api/routes/threads.js diff --git a/api/serialization.js b/packages/nylas-api/serialization.js similarity index 100% rename from api/serialization.js rename to packages/nylas-api/serialization.js diff --git a/core/config/development.json b/packages/nylas-core/config/development.json similarity index 100% rename from core/config/development.json rename to packages/nylas-core/config/development.json diff --git a/core/database-connection-factory.js b/packages/nylas-core/database-connection-factory.js similarity index 97% rename from core/database-connection-factory.js rename to packages/nylas-core/database-connection-factory.js index 1675df2d8..23bfaac71 100644 --- a/core/database-connection-factory.js +++ b/packages/nylas-core/database-connection-factory.js @@ -6,7 +6,7 @@ const DeltaStreamQueue = require('./delta-stream-queue.js') require('./database-extensions'); // Extends Sequelize on require -const STORAGE_DIR = path.join(__base, 'storage'); +const STORAGE_DIR = path.join(__dirname, 'storage'); if (!fs.existsSync(STORAGE_DIR)) { fs.mkdirSync(STORAGE_DIR); } diff --git a/core/database-extensions.js b/packages/nylas-core/database-extensions.js similarity index 100% rename from core/database-extensions.js rename to packages/nylas-core/database-extensions.js diff --git a/core/database-types.js b/packages/nylas-core/database-types.js similarity index 100% rename from core/database-types.js rename to packages/nylas-core/database-types.js diff --git a/core/delta-stream-queue.js b/packages/nylas-core/delta-stream-queue.js similarity index 100% rename from core/delta-stream-queue.js rename to packages/nylas-core/delta-stream-queue.js diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js new file mode 100644 index 000000000..1f3d95a06 --- /dev/null +++ b/packages/nylas-core/index.js @@ -0,0 +1,5 @@ +module.exports = { + DatabaseConnectionFactory: require('./database-connection-factory'), + DeltaStreamQueue: require('./delta-stream-queue'), + Config: require(`./config/${process.env.ENV || 'development'}`), +} diff --git a/core/migrations/20160617002207-create-user.js b/packages/nylas-core/migrations/20160617002207-create-user.js similarity index 100% rename from core/migrations/20160617002207-create-user.js rename to packages/nylas-core/migrations/20160617002207-create-user.js diff --git a/core/models/account/category.js b/packages/nylas-core/models/account/category.js similarity index 100% rename from core/models/account/category.js rename to packages/nylas-core/models/account/category.js diff --git a/core/models/account/message.js b/packages/nylas-core/models/account/message.js similarity index 100% rename from core/models/account/message.js rename to packages/nylas-core/models/account/message.js diff --git a/core/models/account/thread.js b/packages/nylas-core/models/account/thread.js similarity index 100% rename from core/models/account/thread.js rename to packages/nylas-core/models/account/thread.js diff --git a/core/models/account/transaction.js b/packages/nylas-core/models/account/transaction.js similarity index 100% rename from core/models/account/transaction.js rename to packages/nylas-core/models/account/transaction.js diff --git a/core/models/shared/account-token.js b/packages/nylas-core/models/shared/account-token.js similarity index 100% rename from core/models/shared/account-token.js rename to packages/nylas-core/models/shared/account-token.js diff --git a/core/models/shared/account.js b/packages/nylas-core/models/shared/account.js similarity index 100% rename from core/models/shared/account.js rename to packages/nylas-core/models/shared/account.js diff --git a/packages/nylas-core/package.json b/packages/nylas-core/package.json new file mode 100644 index 000000000..bd7fd3df7 --- /dev/null +++ b/packages/nylas-core/package.json @@ -0,0 +1,11 @@ +{ + "name": "nylas-core", + "version": "0.0.1", + "description": "Core shared packages", + "main": "index.js", + "dependencies": { + "sqlite3": "3.1.4" + }, + "author": "Nylas", + "license": "ISC" +} diff --git a/core/transaction-log.js b/packages/nylas-core/transaction-log.js similarity index 100% rename from core/transaction-log.js rename to packages/nylas-core/transaction-log.js diff --git a/message-processor/index.js b/packages/nylas-message-processor/index.js similarity index 93% rename from message-processor/index.js rename to packages/nylas-message-processor/index.js index c50e32bfe..6028adb2f 100644 --- a/message-processor/index.js +++ b/packages/nylas-message-processor/index.js @@ -1,4 +1,4 @@ -const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +const {DatabaseConnectionFactory} = require(`nylas-core`) const {processors} = require('./processors') // List of the attributes of Message that the processor should be allowed to change. diff --git a/packages/nylas-message-processor/package.json b/packages/nylas-message-processor/package.json new file mode 100644 index 000000000..0748d7c9d --- /dev/null +++ b/packages/nylas-message-processor/package.json @@ -0,0 +1,12 @@ +{ + "name": "nylas-message-processor", + "version": "0.0.1", + "description": "Message processing pipeline", + "main": "index.js", + "author": "Nylas", + "license": "ISC", + "dependencies": { + "mailparser": "0.6.0", + "nylas-core": "0.x.x" + } +} diff --git a/message-processor/processors/index.js b/packages/nylas-message-processor/processors/index.js similarity index 100% rename from message-processor/processors/index.js rename to packages/nylas-message-processor/processors/index.js diff --git a/message-processor/processors/parsing.js b/packages/nylas-message-processor/processors/parsing.js similarity index 100% rename from message-processor/processors/parsing.js rename to packages/nylas-message-processor/processors/parsing.js diff --git a/message-processor/processors/quoted-text.js b/packages/nylas-message-processor/processors/quoted-text.js similarity index 100% rename from message-processor/processors/quoted-text.js rename to packages/nylas-message-processor/processors/quoted-text.js diff --git a/message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js similarity index 100% rename from message-processor/processors/threading.js rename to packages/nylas-message-processor/processors/threading.js diff --git a/message-processor/spec/parsing-spec.js b/packages/nylas-message-processor/spec/parsing-spec.js similarity index 100% rename from message-processor/spec/parsing-spec.js rename to packages/nylas-message-processor/spec/parsing-spec.js diff --git a/sync/app.js b/packages/nylas-sync/app.js similarity index 85% rename from sync/app.js rename to packages/nylas-sync/app.js index 8397cfc8e..78025f3a0 100644 --- a/sync/app.js +++ b/packages/nylas-sync/app.js @@ -1,10 +1,6 @@ -const path = require('path'); - -global.__base = path.join(__dirname, '..') -global.config = require(`${__base}/core/config/${process.env.ENV || 'development'}.json`); global.Promise = require('bluebird'); -const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +const {DatabaseConnectionFactory} = require(`nylas-core`) const SyncWorkerPool = require('./sync-worker-pool'); const workerPool = new SyncWorkerPool(); diff --git a/sync/imap/connection.js b/packages/nylas-sync/imap/connection.js similarity index 100% rename from sync/imap/connection.js rename to packages/nylas-sync/imap/connection.js diff --git a/sync/imap/refresh-mailboxes-operation.js b/packages/nylas-sync/imap/refresh-mailboxes-operation.js similarity index 100% rename from sync/imap/refresh-mailboxes-operation.js rename to packages/nylas-sync/imap/refresh-mailboxes-operation.js diff --git a/sync/imap/sync-mailbox-operation.js b/packages/nylas-sync/imap/sync-mailbox-operation.js similarity index 99% rename from sync/imap/sync-mailbox-operation.js rename to packages/nylas-sync/imap/sync-mailbox-operation.js index 6dbdc026c..892a3d2cb 100644 --- a/sync/imap/sync-mailbox-operation.js +++ b/packages/nylas-sync/imap/sync-mailbox-operation.js @@ -1,5 +1,5 @@ const _ = require('underscore'); -const {processMessage} = require(`${__base}/message-processor`); +const {processMessage} = require(`nylas-message-processor`); const {Capabilities} = require('./connection.js'); const MessageFlagAttributes = ['id', 'CategoryUID', 'unread', 'starred'] diff --git a/packages/nylas-sync/package.json b/packages/nylas-sync/package.json new file mode 100644 index 000000000..aaf58b528 --- /dev/null +++ b/packages/nylas-sync/package.json @@ -0,0 +1,16 @@ +{ + "name": "nylas-sync", + "version": "0.0.1", + "description": "Nylas Sync Engine", + "dependencies": { + "imap": "0.8.17", + "nylas-core": "0.x.x", + "nylas-message-processor": "0.x.x", + "xoauth2": "1.1.0" + }, + "scripts": { + "start": "node app.js" + }, + "author": "Nylas", + "license": "ISC" +} diff --git a/sync/sync-worker-pool.js b/packages/nylas-sync/sync-worker-pool.js similarity index 78% rename from sync/sync-worker-pool.js rename to packages/nylas-sync/sync-worker-pool.js index 8423d0ceb..aeca79a18 100644 --- a/sync/sync-worker-pool.js +++ b/packages/nylas-sync/sync-worker-pool.js @@ -1,5 +1,5 @@ const SyncWorker = require('./sync-worker'); -const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +const {DatabaseConnectionFactory} = require(`nylas-core`) class SyncWorkerPool { constructor() { diff --git a/sync/sync-worker.js b/packages/nylas-sync/sync-worker.js similarity index 100% rename from sync/sync-worker.js rename to packages/nylas-sync/sync-worker.js diff --git a/process.json b/process.json deleted file mode 100644 index ee13cea43..000000000 --- a/process.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "apps": [ - { - "name": "api", - "script": "api/app.js", - "watch": true, - "instances": "max", - "exec_mode": "cluster", - "env": { - "NODE_ENV": "development" - }, - "env_production": { - "NODE_ENV": "production" - } - }, - { - "name": "sync", - "script": "sync/app.js", - "watch": true, - "instances": "max", - "exec_mode": "cluster" - } - ] -} diff --git a/sync/package.json b/sync/package.json deleted file mode 100644 index 81af66795..000000000 --- a/sync/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "imap-experiment", - "version": "1.0.0", - "description": "", - "main": "app.js", - "dependencies": { - "bluebird": "^3.4.1", - "imap": "^0.8.17", - "redis": "2.x.x", - "redis-server": "0.x.x", - "underscore": "^1.8.3", - "xoauth2": "^1.1.0" - }, - "devDependencies": {}, - "scripts": { - "start": "node app.js", - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC" -} From 95fbb64d17b94f8a27c444939666e86dd49778cf Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Wed, 22 Jun 2016 14:17:45 -0700 Subject: [PATCH 032/800] Add /messages, /folders, /account --- api/routes/categories.js | 38 +++++++++++++++++++ api/routes/messages.js | 36 ++++++++++++++++++ packages/nylas-api/serialization.js | 12 ++++++ .../nylas-core/models/account/category.js | 9 +++++ packages/nylas-core/models/account/message.js | 14 +++++++ packages/nylas-core/models/shared/account.js | 2 + 6 files changed, 111 insertions(+) create mode 100644 api/routes/categories.js create mode 100644 api/routes/messages.js diff --git a/api/routes/categories.js b/api/routes/categories.js new file mode 100644 index 000000000..65d9ccdaa --- /dev/null +++ b/api/routes/categories.js @@ -0,0 +1,38 @@ +const Joi = require('joi'); +const Serialization = require('../serialization'); + +module.exports = (server) => { + ['folders', 'labels'].forEach((term) => { + server.route({ + method: 'GET', + path: `/${term}`, + config: { + description: `${term}`, + notes: 'Notes go here', + tags: [term], + validate: { + query: { + limit: Joi.number().integer().min(1).max(2000).default(100), + offset: Joi.number().integer().min(0).default(0), + }, + }, + response: { + schema: Joi.array().items( + Serialization.jsonSchema('Category') + ), + }, + }, + handler: (request, reply) => { + request.getAccountDatabase().then((db) => { + const {Category} = db; + Category.findAll({ + limit: request.query.limit, + offset: request.query.offset, + }).then((categories) => { + reply(Serialization.jsonStringify(categories)); + }) + }) + }, + }); + }); +}; diff --git a/api/routes/messages.js b/api/routes/messages.js new file mode 100644 index 000000000..fb898371c --- /dev/null +++ b/api/routes/messages.js @@ -0,0 +1,36 @@ +const Joi = require('joi'); +const Serialization = require('../serialization'); + +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/messages', + config: { + description: 'Returns all your messages.', + notes: 'Notes go here', + tags: ['messages'], + validate: { + query: { + limit: Joi.number().integer().min(1).max(2000).default(100), + offset: Joi.number().integer().min(0).default(0), + }, + }, + response: { + schema: Joi.array().items( + Serialization.jsonSchema('Message') + ), + }, + }, + handler: (request, reply) => { + request.getAccountDatabase().then((db) => { + const {Message} = db; + Message.findAll({ + limit: request.query.limit, + offset: request.query.offset, + }).then((messages) => { + reply(Serialization.jsonStringify(messages)); + }) + }) + }, + }); +}; diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 6e0da9b90..3609d1c67 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -10,6 +10,18 @@ function jsonSchema(modelName) { return Joi.object().keys({ id: Joi.number(), email_address: Joi.string(), + connection_settings: Joi.object(), + sync_policy: Joi.object(), + }) + } + if (modelName === 'Message') { + return Joi.object(); + } + if (modelName === 'Category') { + return Joi.object().keys({ + id: Joi.number(), + name: Joi.string().allow(null), + display_name: Joi.string(), }) } return null; diff --git a/packages/nylas-core/models/account/category.js b/packages/nylas-core/models/account/category.js index 810a01bbd..5d2c97408 100644 --- a/packages/nylas-core/models/account/category.js +++ b/packages/nylas-core/models/account/category.js @@ -11,6 +11,15 @@ module.exports = (sequelize, Sequelize) => { Category.hasMany(Message) }, }, + instanceMethods: { + toJSON: function toJSON() { + return { + id: this.id, + name: this.role, + display_name: this.name, + }; + }, + }, }); return Category; diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js index e9e66db35..17405e9bd 100644 --- a/packages/nylas-core/models/account/message.js +++ b/packages/nylas-core/models/account/message.js @@ -35,6 +35,20 @@ module.exports = (sequelize, Sequelize) => { return crypto.createHash('sha256').update(headers, 'utf8').digest('hex'); }, }, + instanceMethods: { + toJSON: function toJSON() { + return { + id: this.id, + body: this.body, + subject: this.subject, + snippet: this.snippet, + date: this.date.getTime() / 1000.0, + unread: this.unread, + starred: this.starred, + category_id: this.CategoryId, + }; + }, + }, }); return Message; diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index 72edad611..259060f52 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -21,6 +21,8 @@ module.exports = (sequelize, Sequelize) => { return { id: this.id, email_address: this.emailAddress, + connection_settings: this.connectionSettings, + sync_policy: this.syncPolicy, } }, From 06393dd07c5ded496268bd24fbad2c9c9e1f94e4 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 14:43:20 -0700 Subject: [PATCH 033/800] Move categories & messages to new monorepo structure --- packages/nylas-api/app.js | 1 - {api => packages/nylas-api}/routes/categories.js | 0 {api => packages/nylas-api}/routes/messages.js | 0 3 files changed, 1 deletion(-) rename {api => packages/nylas-api}/routes/categories.js (100%) rename {api => packages/nylas-api}/routes/messages.js (100%) diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index 2b36fdd72..a07e10bf9 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -8,7 +8,6 @@ const fs = require('fs'); const path = require('path'); const server = new Hapi.Server(); -console.log(process.env.PORT) server.connection({ port: process.env.PORT || 5100 }); const plugins = [Inert, Vision, HapiBasicAuth, { diff --git a/api/routes/categories.js b/packages/nylas-api/routes/categories.js similarity index 100% rename from api/routes/categories.js rename to packages/nylas-api/routes/categories.js diff --git a/api/routes/messages.js b/packages/nylas-api/routes/messages.js similarity index 100% rename from api/routes/messages.js rename to packages/nylas-api/routes/messages.js From 71167328ddaa5830dcd11b6dc1d765db1caebf27 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 14:49:36 -0700 Subject: [PATCH 034/800] Update readme --- README.md | 14 +++----------- package.json | 2 +- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index eb7e9151c..e3dccf6c4 100644 --- a/README.md +++ b/README.md @@ -4,23 +4,15 @@ 1. Download https://toolbelt.heroku.com/ -2. Install redis for local dev: - ``` brew install redis -``` - -3. Make sure you're using the latest node: - -``` nvm install 6 -nvm use 6 +npm install -g lerna@2.0.0-beta +lerna bootstrap ``` -4. `npm install` - # Running locally ``` -foreman start -f Procfile.dev +npm start ``` diff --git a/package.json b/package.json index d75708d33..1f154dbc7 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "lerna": "2.0.0-beta.20" }, "scripts": { - "start": "heroku local -f Procfile.dev" + "start": "foreman start -f Procfile.dev" }, "repository": { "type": "git", From d5d019f9d9ddf636673d51312eeec46f2cecad82 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 22 Jun 2016 14:53:23 -0700 Subject: [PATCH 035/800] Plain npm install --- README.md | 3 +-- package.json | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e3dccf6c4..9457940eb 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,7 @@ ``` brew install redis nvm install 6 -npm install -g lerna@2.0.0-beta -lerna bootstrap +npm install ``` # Running locally diff --git a/package.json b/package.json index 1f154dbc7..fc04ce51c 100644 --- a/package.json +++ b/package.json @@ -5,12 +5,10 @@ "main": "", "dependencies": { "bluebird": "3.x.x", - "nylas-core": "0.x.x", - "nylas-api": "0.x.x", - "nylas-sync": "0.x.x", "redis": "2.x.x", "rx": "4.x.x", "sequelize": "3.x.x", + "sqlite3": "3.1.4", "underscore": "1.x.x" }, "devDependencies": { @@ -24,7 +22,8 @@ "lerna": "2.0.0-beta.20" }, "scripts": { - "start": "foreman start -f Procfile.dev" + "start": "foreman start -f Procfile.dev", + "postinstall": "lerna bootstrap" }, "repository": { "type": "git", From f5f435236fea7a7bcb3cff1d0cd7bf2c2aae3d7d Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Wed, 22 Jun 2016 17:19:48 -0700 Subject: [PATCH 036/800] /auth + remove hardcoded stub in favor of curl request --- packages/nylas-api/routes/auth.js | 106 ++++++++++++++++++ packages/nylas-api/serialization.js | 3 + .../nylas-core/database-connection-factory.js | 2 +- .../imap-connection.js} | 16 ++- packages/nylas-core/index.js | 1 + packages/nylas-core/models/shared/account.js | 1 + packages/nylas-sync/app.js | 43 +------ .../nylas-sync/imap/sync-mailbox-operation.js | 3 +- packages/nylas-sync/sync-worker.js | 8 +- 9 files changed, 130 insertions(+), 53 deletions(-) create mode 100644 packages/nylas-api/routes/auth.js rename packages/{nylas-sync/imap/connection.js => nylas-core/imap-connection.js} (92%) diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js new file mode 100644 index 000000000..7f9adaa57 --- /dev/null +++ b/packages/nylas-api/routes/auth.js @@ -0,0 +1,106 @@ +const Joi = require('Joi'); +const _ = require('underscore'); + +const Serialization = require('../serialization'); +const {IMAPConnection, DatabaseConnectionFactory} = require('nylas-core'); + +const imapSmtpSettings = Joi.object().keys({ + imap_host: [Joi.string().ip().required(), Joi.string().hostname().required()], + imap_port: Joi.number().integer().required(), + imap_username: Joi.string().required(), + imap_password: Joi.string().required(), + smtp_host: [Joi.string().ip().required(), Joi.string().hostname().required()], + smtp_port: Joi.number().integer().required(), + smtp_username: Joi.string().required(), + smtp_password: Joi.string().required(), + ssl_required: Joi.boolean().required(), +}).required(); + +const exchangeSettings = Joi.object().keys({ + username: Joi.string().required(), + password: Joi.string().required(), + eas_server_host: [Joi.string().ip().required(), Joi.string().hostname().required()], +}).required(); + +const defaultSyncPolicy = { + afterSync: 'idle', + interval: 30 * 1000, + folderSyncOptions: { + deepFolderScan: 5 * 60 * 1000, + }, + expiration: Date.now() + 60 * 60 * 1000, +}; + +module.exports = (server) => { + server.route({ + method: 'POST', + path: '/auth', + config: { + description: 'Authenticates a new account.', + notes: 'Notes go here', + tags: ['accounts'], + auth: false, + validate: { + query: { + client_id: Joi.string().required(), + }, + payload: { + email: Joi.string().email().required(), + name: Joi.string().required(), + provider: Joi.string().required(), + settings: Joi.alternatives().try(imapSmtpSettings, exchangeSettings), + }, + }, + response: { + schema: Joi.alternatives().try( + Serialization.jsonSchema('Account'), + Serialization.jsonSchema('Error') + ), + }, + }, + handler: (request, reply) => { + const connectionChecks = []; + const {settings, email, provider, name} = request.payload; + + if (provider === 'imap') { + const dbStub = {}; + const conn = new IMAPConnection(dbStub, settings); + connectionChecks.push(conn.connect()) + } + + Promise.all(connectionChecks).then(() => { + DatabaseConnectionFactory.forShared().then((db) => { + const {AccountToken, Account} = db; + + const account = Account.build({ + name: name, + emailAddress: email, + syncPolicy: defaultSyncPolicy, + connectionSettings: _.pick(settings, [ + 'imap_host', 'imap_port', + 'smtp_host', 'smtp_port', + 'ssl_required', + ]), + }) + account.setCredentials(_.pick(settings, [ + 'imap_username', 'imap_password', + 'smtp_username', 'smtp_password', + ])); + account.save().then((saved) => + AccountToken.create({ + AccountId: saved.id, + }).then((token) => { + const response = Serialization.jsonStringify(saved); + response.token = token; + reply(response); + }) + ); + }) + }) + .catch((err) => { + // TODO: Lots more of this + reply({error: err.toString()}); + }) + }, + }); +}; diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 3609d1c67..5304984d9 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -17,6 +17,9 @@ function jsonSchema(modelName) { if (modelName === 'Message') { return Joi.object(); } + if (modelName === 'Error') { + return Joi.object(); + } if (modelName === 'Category') { return Joi.object().keys({ id: Joi.number(), diff --git a/packages/nylas-core/database-connection-factory.js b/packages/nylas-core/database-connection-factory.js index 23bfaac71..18ed00156 100644 --- a/packages/nylas-core/database-connection-factory.js +++ b/packages/nylas-core/database-connection-factory.js @@ -6,7 +6,7 @@ const DeltaStreamQueue = require('./delta-stream-queue.js') require('./database-extensions'); // Extends Sequelize on require -const STORAGE_DIR = path.join(__dirname, 'storage'); +const STORAGE_DIR = path.join(__dirname, '..', '..', 'storage'); if (!fs.existsSync(STORAGE_DIR)) { fs.mkdirSync(STORAGE_DIR); } diff --git a/packages/nylas-sync/imap/connection.js b/packages/nylas-core/imap-connection.js similarity index 92% rename from packages/nylas-sync/imap/connection.js rename to packages/nylas-core/imap-connection.js index 3097e5102..11782518a 100644 --- a/packages/nylas-sync/imap/connection.js +++ b/packages/nylas-core/imap-connection.js @@ -1,5 +1,6 @@ const Imap = require('imap'); const EventEmitter = require('events'); +const Promise = require('bluebird'); const Capabilities = { Gmail: 'X-GM-EXT-1', @@ -18,11 +19,13 @@ class IMAPConnection extends EventEmitter { this._db = db; this._queue = []; this._current = null; - this._imap = Promise.promisifyAll(new Imap(settings)); - - this._imap.once('error', (err) => { - console.log(err); - }); + this._imap = Promise.promisifyAll(new Imap({ + host: settings.imap_host, + port: settings.imap_port, + user: settings.imap_username, + password: settings.imap_password, + tls: settings.ssl_required, + })); this._imap.once('end', () => { console.log('Connection ended'); @@ -56,8 +59,9 @@ class IMAPConnection extends EventEmitter { connect() { if (!this._connectPromise) { - this._connectPromise = new Promise((resolve) => { + this._connectPromise = new Promise((resolve, reject) => { this._imap.once('ready', resolve); + this._imap.once('error', reject); this._imap.connect(); }); } diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index 1f3d95a06..26077677d 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -1,5 +1,6 @@ module.exports = { DatabaseConnectionFactory: require('./database-connection-factory'), DeltaStreamQueue: require('./delta-stream-queue'), + IMAPConnection: require('./imap-connection'), Config: require(`./config/${process.env.ENV || 'development'}`), } diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index 259060f52..fff33f885 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -6,6 +6,7 @@ const password = 'd6F3Efeq'; module.exports = (sequelize, Sequelize) => { const Account = sequelize.define('Account', { + name: Sequelize.STRING, emailAddress: Sequelize.STRING, connectionSettings: JSONType('connectionSettings'), connectionCredentials: Sequelize.STRING, diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 78025f3a0..522fe13d6 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -4,52 +4,13 @@ const {DatabaseConnectionFactory} = require(`nylas-core`) const SyncWorkerPool = require('./sync-worker-pool'); const workerPool = new SyncWorkerPool(); -const seed = (db) => { - const {Account, AccountToken} = db; - - const account = Account.build({ - emailAddress: 'inboxapptest1@fastmail.fm', - connectionSettings: { - imap: { - host: 'mail.messagingengine.com', - port: 993, - tls: true, - }, - }, - syncPolicy: { - afterSync: 'idle', - interval: 30 * 1000, - folderSyncOptions: { - deepFolderScan: 5 * 60 * 1000, - }, - expiration: Date.now() + 60 * 60 * 1000, - }, - }) - account.setCredentials({ - imap: { - user: 'inboxapptest1@fastmail.fm', - password: 'trar2e', - }, - smtp: { - user: 'inboxapptest1@fastmail.fm', - password: 'trar2e', - }, - }); - return account.save().then((obj) => - AccountToken.create({ - AccountId: obj.id, - }).then((token) => { - console.log(`Created seed data. Your API token is ${token.value}`) - }) - ); -} - const start = () => { DatabaseConnectionFactory.forShared().then((db) => { const {Account} = db; Account.findAll().then((accounts) => { if (accounts.length === 0) { - seed(db).then(start); + console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) + console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.amessagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) } accounts.forEach((account) => { workerPool.addWorkerForAccount(account); diff --git a/packages/nylas-sync/imap/sync-mailbox-operation.js b/packages/nylas-sync/imap/sync-mailbox-operation.js index 892a3d2cb..56aab99f3 100644 --- a/packages/nylas-sync/imap/sync-mailbox-operation.js +++ b/packages/nylas-sync/imap/sync-mailbox-operation.js @@ -1,6 +1,7 @@ const _ = require('underscore'); const {processMessage} = require(`nylas-message-processor`); -const {Capabilities} = require('./connection.js'); +const {IMAPConnection} = require('nylas-core'); +const {Capabilities} = IMAPConnection; const MessageFlagAttributes = ['id', 'CategoryUID', 'unread', 'starred'] diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 01e537969..1e3566c26 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -1,4 +1,4 @@ -const IMAPConnection = require('./imap/connection'); +const {IMAPConnection} = require('nylas-core'); const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') const SyncMailboxOperation = require('./imap/sync-mailbox-operation') // @@ -75,14 +75,14 @@ class SyncWorker { const settings = this._account.connectionSettings; const credentials = this._account.decryptedCredentials(); - if (!settings || !settings.imap) { + if (!settings || !settings.imap_host) { throw new Error("ensureConnection: There are no IMAP connection settings for this account.") } - if (!credentials || !credentials.imap) { + if (!credentials || !credentials.imap_username) { throw new Error("ensureConnection: There are no IMAP connection credentials for this account.") } - const conn = new IMAPConnection(this._db, Object.assign({}, settings.imap, credentials.imap)); + const conn = new IMAPConnection(this._db, Object.assign({}, settings, credentials)); conn.on('mail', () => { this.onConnectionIdleUpdate(); }) From e64c2ae4c0856b9e3e381fdb9cf168535a0bc0a8 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Wed, 22 Jun 2016 11:49:53 -0700 Subject: [PATCH 037/800] Add basic threading --- packages/nylas-core/models/account/thread.js | 10 ++-- packages/nylas-message-processor/index.js | 1 - .../processors/threading.js | 57 ++++++++++++++++++- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js index 1c63b9ccc..208bcbcf7 100644 --- a/packages/nylas-core/models/account/thread.js +++ b/packages/nylas-core/models/account/thread.js @@ -1,12 +1,12 @@ module.exports = (sequelize, Sequelize) => { const Thread = sequelize.define('Thread', { - first_name: Sequelize.STRING, - last_name: Sequelize.STRING, - bio: Sequelize.TEXT, + threadId: Sequelize.STRING, + subject: Sequelize.STRING, + cleanedSubject: Sequelize.STRING, }, { classMethods: { - associate: (models) => { - // associations can be defined here + associate: ({Message}) => { + Thread.hasMany(Message, {as: 'messages'}) }, }, }); diff --git a/packages/nylas-message-processor/index.js b/packages/nylas-message-processor/index.js index 6028adb2f..b50d83cd1 100644 --- a/packages/nylas-message-processor/index.js +++ b/packages/nylas-message-processor/index.js @@ -7,7 +7,6 @@ const {processors} = require('./processors') const MessageAttributes = ['body', 'processed'] const MessageProcessorVersion = 1; - function runPipeline(accountId, message) { return processors.reduce((prevPromise, processor) => ( prevPromise.then((msg) => processor({message: msg, accountId})) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 7812f9346..9502e634b 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -1,4 +1,55 @@ -module.exports = { - order: 1, - processMessage: ({message}) => message, +const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) + +function processMessage({message, accountId}) { + return DatabaseConnectionFactory.forAccount(accountId) + .then((db) => addMessageToThread({db, accountId, message})) + .then((thread) => { + thread.setMessages([message]) + return message + }) +} + +function addMessageToThread({db, accountId, message}) { + const {Thread, Message} = db + if (message.threadId) { + return Thread.findOne({ + where: { + threadId: message.threadId, + } + }) + } + return matchThread({db, accountId, message}) + .then((thread) => { + if (thread) { + return thread + } + return Thread.create({ + subject: message.subject, + cleanedSubject: cleanSubject(message.subject), + }) + }) +} + +function matchThread({db, accountId, message}) { + if (message.headers.inReplyTo) { + return getThreadFromHeader() + .then((thread) => { + if (thread) { + return thread + } + return Thread.create({ + subject: message.subject, + cleanedSubject: cleanSubject(message.subject), + }) + }) + } + return Thread.create({ + subject: message.subject, + cleanedSubject: cleanSubject(message.subject), + }) +} + +module.exports = { + order: 0, + processMessage, } From cb9557437856c1ccc15e48fff63b0110d132c755 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Wed, 22 Jun 2016 17:34:21 -0700 Subject: [PATCH 038/800] Fix associations --- packages/nylas-core/models/account/message.js | 3 +- .../processors/threading.js | 132 +++++++++++++++--- 2 files changed, 117 insertions(+), 18 deletions(-) diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js index 17405e9bd..d43f8ce3b 100644 --- a/packages/nylas-core/models/account/message.js +++ b/packages/nylas-core/models/account/message.js @@ -28,8 +28,9 @@ module.exports = (sequelize, Sequelize) => { }, ], classMethods: { - associate: ({Category}) => { + associate: ({Category, Thread}) => { Message.belongsTo(Category) + Message.belongsTo(Thread) }, hashForHeaders: (headers) => { return crypto.createHash('sha256').update(headers, 'utf8').digest('hex'); diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 9502e634b..3d98bea52 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -4,7 +4,8 @@ function processMessage({message, accountId}) { return DatabaseConnectionFactory.forAccount(accountId) .then((db) => addMessageToThread({db, accountId, message})) .then((thread) => { - thread.setMessages([message]) + thread.addMessage(message) + message.setThread(thread) return message }) } @@ -12,27 +13,19 @@ function processMessage({message, accountId}) { function addMessageToThread({db, accountId, message}) { const {Thread, Message} = db if (message.threadId) { - return Thread.findOne({ - where: { - threadId: message.threadId, - } - }) + return Thread.find({where: {threadId: message.threadId}}) } return matchThread({db, accountId, message}) - .then((thread) => { - if (thread) { - return thread - } - return Thread.create({ - subject: message.subject, - cleanedSubject: cleanSubject(message.subject), - }) - }) + .then((thread) => (thread)) } function matchThread({db, accountId, message}) { - if (message.headers.inReplyTo) { - return getThreadFromHeader() + const {Thread} = db + + // TODO: Add once we have some test data with this header + /* + if (message.headers['in-reply-to']) { + return getThreadFromHeader() // Doesn't exist yet .then((thread) => { if (thread) { return thread @@ -43,12 +36,117 @@ function matchThread({db, accountId, message}) { }) }) } + */ return Thread.create({ subject: message.subject, cleanedSubject: cleanSubject(message.subject), }) } +function cleanSubject(subject) { + if (subject === null) + return "" + const regex = new RegExp(/^((re|fw|fwd|aw|wg|undeliverable|undelivered):\s*)+/ig) + const cleanedSubject = subject.replace(regex, (match) => "") + return cleanedSubject +} + +// TODO: Incorporate this more elaborate threading algorithm +/* +function fetchCorrespondingThread({db, accountId, message}) { + const cleanedSubject = cleanSubject(message.subject) + return getThreads({db, message, cleanedSubject}) + .then((threads) => { + return findCorrespondingThread({message, threads}) + }) +} + +function getThreads({db, message, cleanedSubject}) { + const {Thread} = db + return Thread.findAll({ + where: { + threadId: message.headers.threadId, + cleanedSubject: cleanedSubject, + }, + order: [ + ['id', 'DESC'] + ], + }) +} + +function findCorrespondingThread({message, threads}) { + for (const thread of threads) { + for (const match of thread.messages) { + // Ignore BCC + const {matchEmails, messageEmails} = removeBccParticipants({message, match}) + + // A thread is probably related if it has more than two common participants + const intersectingParticipants = getIntersectingParticipants({messageEmails, matchEmails}) + if (intersectingParticipants.length >= 2) { + if (thread.messages.length >= MAX_THREAD_LENGTH) + break + return match.thread + } + + // Handle case for self-sent emails + if (!message.from || !message.to) + return + if (isSentToSelf({message, match})) { + if (thread.messages.length >= MAX_THREAD_LENGTH) + break + return match.thread + } + } + } +} + +function removeBccParticipants({message, match}) { + const matchBcc = match.bcc ? match.bcc : [] + const messageBcc = message.bcc ? message.bcc : [] + let matchEmails = match.participants.filter((participant) => { + return matchBcc.find(bcc => bcc === participant) + }) + matchEmails.map((email) => { + return email[1] + }) + let messageEmails = message.participants.filter((participant) => { + return messageBcc.find(bcc => bcc === participant) + }) + messageEmails.map((email) => { + return email[1] + }) + return {messageEmails, matchEmails} +} + +function getIntersectingParticipants({messageEmails, matchEmails}) { + const matchParticipants = new Set(matchEmails) + const messageParticipants = new Set(messageEmails) + const intersectingParticipants = new Set([...matchParticipants] + .filter(participant => messageParticipants.has(participant))) + return intersectingParticipants +} + +function isSentToSelf({message, match}) { + const matchFrom = match.from.map((participant) => { + return participant[1] + }) + const matchTo = match.to.map((participant) => { + return participant[1] + }) + const messageFrom = message.from.map((participant) => { + return participant[1] + }) + const messageTo = message.to.map((participant) => { + return participant[1] + }) + + return (messageTo.length === 1 && + messageFrom === messageTo && + matchFrom === matchTo && + messageTo === matchFrom) +} +*/ + module.exports = { order: 0, processMessage, From 2e9bfa68b56dbcc5c86c202eb76bc379c2e79cad Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Wed, 22 Jun 2016 18:34:00 -0700 Subject: [PATCH 039/800] Refactor DeltaStreamQueue (connection per subscription) --- packages/nylas-api/app.js | 2 - packages/nylas-api/routes/auth.js | 8 ++-- packages/nylas-api/routes/delta.js | 4 +- packages/nylas-core/account-pubsub.js | 47 +++++++++++++++++++ .../nylas-core/database-connection-factory.js | 5 -- packages/nylas-core/delta-stream-queue.js | 33 ------------- packages/nylas-core/index.js | 2 +- packages/nylas-core/transaction-log.js | 6 +-- packages/nylas-sync/app.js | 1 - 9 files changed, 57 insertions(+), 51 deletions(-) create mode 100644 packages/nylas-core/account-pubsub.js delete mode 100644 packages/nylas-core/delta-stream-queue.js diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index a07e10bf9..d752b9bb9 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -63,8 +63,6 @@ server.register(plugins, (err) => { server.auth.strategy('api-consumer', 'basic', { validateFunc: validate }); server.auth.default('api-consumer'); - DatabaseConnectionFactory.setup() - server.start((startErr) => { if (startErr) { throw startErr; } console.log('Server running at:', server.info.uri); diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index 7f9adaa57..7e76ff6de 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -89,10 +89,10 @@ module.exports = (server) => { account.save().then((saved) => AccountToken.create({ AccountId: saved.id, - }).then((token) => { - const response = Serialization.jsonStringify(saved); - response.token = token; - reply(response); + }).then((accountToken) => { + const response = saved.toJSON(); + response.token = accountToken.value; + reply(Serialization.jsonStringify(response)); }) ); }) diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js index c98471e19..93497ba6e 100644 --- a/packages/nylas-api/routes/delta.js +++ b/packages/nylas-api/routes/delta.js @@ -1,6 +1,6 @@ const Rx = require('rx') const _ = require('underscore'); -const {DeltaStreamQueue} = require(`nylas-core`); +const {AccountPubsub} = require(`nylas-core`); function keepAlive(request) { const until = Rx.Observable.fromCallback(request.on)("disconnect") @@ -52,7 +52,7 @@ module.exports = (server) => { request.getAccountDatabase().then((db) => { const source = Rx.Observable.merge( - DeltaStreamQueue.fromAccountId(db.accountId), + AccountPubsub.observableForAccountId(db.accountId), initialTransactions(db, request), keepAlive(request) ).subscribe(outputStream.pushJSON) diff --git a/packages/nylas-core/account-pubsub.js b/packages/nylas-core/account-pubsub.js new file mode 100644 index 000000000..57642565b --- /dev/null +++ b/packages/nylas-core/account-pubsub.js @@ -0,0 +1,47 @@ +const Rx = require('rx') +const bluebird = require('bluebird') +const redis = require("redis"); +bluebird.promisifyAll(redis.RedisClient.prototype); +bluebird.promisifyAll(redis.Multi.prototype); + +class AccountPubsub { + constructor() { + this._broadcastClient = null; + } + + buildClient() { + const client = redis.createClient(process.env.REDIS_URL || null); + client.on("error", console.error); + return client; + } + + keyForAccountId(accountId) { + return `delta-${accountId}`; + } + + notify(accountId, data) { + if (!this._broadcastClient) { + this._broadcastClient = this.buildClient(); + } + const key = this.keyForAccountId(accountId); + this._broadcastClient.publish(key, JSON.stringify(data)) + } + + observableForAccountId(accountId) { + return Rx.Observable.create((observer) => { + const sub = this.buildClient(); + const key = this.keyForAccountId(accountId); + sub.on("message", (channel, message) => { + if (channel !== key) { return } + observer.onNext(message) + }); + sub.subscribe(key); + return () => { + sub.unsubscribe() + sub.quit() + } + }) + } +} + +module.exports = new AccountPubsub() diff --git a/packages/nylas-core/database-connection-factory.js b/packages/nylas-core/database-connection-factory.js index 18ed00156..6e0e9f3bd 100644 --- a/packages/nylas-core/database-connection-factory.js +++ b/packages/nylas-core/database-connection-factory.js @@ -2,7 +2,6 @@ const Sequelize = require('sequelize'); const fs = require('fs'); const path = require('path'); const TransactionLog = require('./transaction-log') -const DeltaStreamQueue = require('./delta-stream-queue.js') require('./database-extensions'); // Extends Sequelize on require @@ -16,10 +15,6 @@ class DatabaseConnectionFactory { this._pools = {}; } - setup() { - DeltaStreamQueue.setup() - } - _readModelsInDirectory(sequelize, dirname) { const db = {}; for (const filename of fs.readdirSync(dirname)) { diff --git a/packages/nylas-core/delta-stream-queue.js b/packages/nylas-core/delta-stream-queue.js deleted file mode 100644 index b85102a2f..000000000 --- a/packages/nylas-core/delta-stream-queue.js +++ /dev/null @@ -1,33 +0,0 @@ -const Rx = require('rx') -const bluebird = require('bluebird') -const redis = require("redis"); -bluebird.promisifyAll(redis.RedisClient.prototype); -bluebird.promisifyAll(redis.Multi.prototype); - -class DeltaStreamQueue { - setup() { - this.client = redis.createClient(process.env.REDIS_URL || null); - this.client.on("error", console.error); - } - - key(accountId) { - return `delta-${accountId}` - } - - notify(accountId, data) { - this.client.publish(this.key(accountId), JSON.stringify(data)) - } - - fromAccountId(accountId) { - return Rx.Observable.create((observer) => { - this.client.on("message", (channel, message) => { - if (channel !== this.key(accountId)) { return } - observer.onNext(message) - }); - this.client.subscribe(this.key(accountId)); - return () => { this.client.unsubscribe() } - }) - } -} - -module.exports = new DeltaStreamQueue() diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index 26077677d..3accf1084 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -1,6 +1,6 @@ module.exports = { DatabaseConnectionFactory: require('./database-connection-factory'), - DeltaStreamQueue: require('./delta-stream-queue'), + AccountPubsub: require('./account-pubsub'), IMAPConnection: require('./imap-connection'), Config: require(`./config/${process.env.ENV || 'development'}`), } diff --git a/packages/nylas-core/transaction-log.js b/packages/nylas-core/transaction-log.js index a9a9bd401..77a3a9474 100644 --- a/packages/nylas-core/transaction-log.js +++ b/packages/nylas-core/transaction-log.js @@ -1,4 +1,4 @@ -const DeltaStreamQueue = require('./delta-stream-queue') +const AccountPubsub = require('./account-pubsub') class TransactionLog { constructor(db) { @@ -24,8 +24,8 @@ class TransactionLog { this.parseHookData(sequelizeHookData) ); this.db.Transaction.create(transactionData); - transactionData.object = sequelizeHookData.dataValues - DeltaStreamQueue.notify(this.db.accountId, transactionData) + transactionData.object = sequelizeHookData.dataValues; + AccountPubsub.notify(this.db.accountId, transactionData); } } diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 522fe13d6..65adcd614 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -19,7 +19,6 @@ const start = () => { }); } -DatabaseConnectionFactory.setup() start(); global.workerPool = workerPool; From 12d9db8dd924fffa9bd8bd209843fca805f66942 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 23 Jun 2016 00:49:22 -0700 Subject: [PATCH 040/800] Redis coordination of sync processes / assignment --- packages/nylas-api/app.js | 4 +- packages/nylas-api/decorators/connections.js | 4 +- packages/nylas-api/routes/auth.js | 9 +- packages/nylas-api/routes/delta.js | 4 +- packages/nylas-core/account-pubsub.js | 47 ----- ...ction-factory.js => database-connector.js} | 4 +- packages/nylas-core/index.js | 4 +- packages/nylas-core/pubsub-connector.js | 91 +++++++++ packages/nylas-core/transaction-log.js | 5 +- packages/nylas-message-processor/index.js | 4 +- packages/nylas-sync/app.js | 35 ++-- packages/nylas-sync/sync-process-manager.js | 179 ++++++++++++++++++ packages/nylas-sync/sync-worker-pool.js | 16 -- packages/nylas-sync/sync-worker.js | 27 ++- 14 files changed, 329 insertions(+), 104 deletions(-) delete mode 100644 packages/nylas-core/account-pubsub.js rename packages/nylas-core/{database-connection-factory.js => database-connector.js} (96%) create mode 100644 packages/nylas-core/pubsub-connector.js create mode 100644 packages/nylas-sync/sync-process-manager.js delete mode 100644 packages/nylas-sync/sync-worker-pool.js diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index d752b9bb9..365e53904 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -21,8 +21,8 @@ const plugins = [Inert, Vision, HapiBasicAuth, { }]; let sharedDb = null; -const {DatabaseConnectionFactory} = require(`nylas-core`) -DatabaseConnectionFactory.forShared().then((db) => { +const {DatabaseConnector} = require(`nylas-core`) +DatabaseConnector.forShared().then((db) => { sharedDb = db; }); diff --git a/packages/nylas-api/decorators/connections.js b/packages/nylas-api/decorators/connections.js index 2190f2c67..2003696cf 100644 --- a/packages/nylas-api/decorators/connections.js +++ b/packages/nylas-api/decorators/connections.js @@ -1,10 +1,10 @@ /* eslint func-names:0 */ -const {DatabaseConnectionFactory} = require(`nylas-core`); +const {DatabaseConnector} = require(`nylas-core`); module.exports = (server) => { server.decorate('request', 'getAccountDatabase', function () { const account = this.auth.credentials; - return DatabaseConnectionFactory.forAccount(account.id); + return DatabaseConnector.forAccount(account.id); }); } diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index 7e76ff6de..6ccb02118 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -2,7 +2,7 @@ const Joi = require('Joi'); const _ = require('underscore'); const Serialization = require('../serialization'); -const {IMAPConnection, DatabaseConnectionFactory} = require('nylas-core'); +const {IMAPConnection, PubsubConnector, DatabaseConnector} = require('nylas-core'); const imapSmtpSettings = Joi.object().keys({ imap_host: [Joi.string().ip().required(), Joi.string().hostname().required()], @@ -69,7 +69,7 @@ module.exports = (server) => { } Promise.all(connectionChecks).then(() => { - DatabaseConnectionFactory.forShared().then((db) => { + DatabaseConnector.forShared().then((db) => { const {AccountToken, Account} = db; const account = Account.build({ @@ -90,6 +90,11 @@ module.exports = (server) => { AccountToken.create({ AccountId: saved.id, }).then((accountToken) => { + const client = PubsubConnector.broadcastClient(); + client.lpushAsync('accounts:unclaimed', saved.id).catch((err) => { + console.error(`Auth: Could not queue account sync! ${err.message}`) + }); + const response = saved.toJSON(); response.token = accountToken.value; reply(Serialization.jsonStringify(response)); diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js index 93497ba6e..fc5e603bf 100644 --- a/packages/nylas-api/routes/delta.js +++ b/packages/nylas-api/routes/delta.js @@ -1,6 +1,6 @@ const Rx = require('rx') const _ = require('underscore'); -const {AccountPubsub} = require(`nylas-core`); +const {PubsubConnector} = require(`nylas-core`); function keepAlive(request) { const until = Rx.Observable.fromCallback(request.on)("disconnect") @@ -52,7 +52,7 @@ module.exports = (server) => { request.getAccountDatabase().then((db) => { const source = Rx.Observable.merge( - AccountPubsub.observableForAccountId(db.accountId), + PubsubConnector.observableForAccountDeltas(db.accountId), initialTransactions(db, request), keepAlive(request) ).subscribe(outputStream.pushJSON) diff --git a/packages/nylas-core/account-pubsub.js b/packages/nylas-core/account-pubsub.js deleted file mode 100644 index 57642565b..000000000 --- a/packages/nylas-core/account-pubsub.js +++ /dev/null @@ -1,47 +0,0 @@ -const Rx = require('rx') -const bluebird = require('bluebird') -const redis = require("redis"); -bluebird.promisifyAll(redis.RedisClient.prototype); -bluebird.promisifyAll(redis.Multi.prototype); - -class AccountPubsub { - constructor() { - this._broadcastClient = null; - } - - buildClient() { - const client = redis.createClient(process.env.REDIS_URL || null); - client.on("error", console.error); - return client; - } - - keyForAccountId(accountId) { - return `delta-${accountId}`; - } - - notify(accountId, data) { - if (!this._broadcastClient) { - this._broadcastClient = this.buildClient(); - } - const key = this.keyForAccountId(accountId); - this._broadcastClient.publish(key, JSON.stringify(data)) - } - - observableForAccountId(accountId) { - return Rx.Observable.create((observer) => { - const sub = this.buildClient(); - const key = this.keyForAccountId(accountId); - sub.on("message", (channel, message) => { - if (channel !== key) { return } - observer.onNext(message) - }); - sub.subscribe(key); - return () => { - sub.unsubscribe() - sub.quit() - } - }) - } -} - -module.exports = new AccountPubsub() diff --git a/packages/nylas-core/database-connection-factory.js b/packages/nylas-core/database-connector.js similarity index 96% rename from packages/nylas-core/database-connection-factory.js rename to packages/nylas-core/database-connector.js index 6e0e9f3bd..8c573431b 100644 --- a/packages/nylas-core/database-connection-factory.js +++ b/packages/nylas-core/database-connector.js @@ -10,7 +10,7 @@ if (!fs.existsSync(STORAGE_DIR)) { fs.mkdirSync(STORAGE_DIR); } -class DatabaseConnectionFactory { +class DatabaseConnector { constructor() { this._pools = {}; } @@ -86,4 +86,4 @@ class DatabaseConnectionFactory { } } -module.exports = new DatabaseConnectionFactory() +module.exports = new DatabaseConnector() diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index 3accf1084..f5ebeae89 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -1,6 +1,6 @@ module.exports = { - DatabaseConnectionFactory: require('./database-connection-factory'), - AccountPubsub: require('./account-pubsub'), + DatabaseConnector: require('./database-connector'), + PubsubConnector: require('./pubsub-connector'), IMAPConnection: require('./imap-connection'), Config: require(`./config/${process.env.ENV || 'development'}`), } diff --git a/packages/nylas-core/pubsub-connector.js b/packages/nylas-core/pubsub-connector.js new file mode 100644 index 000000000..533b30a5f --- /dev/null +++ b/packages/nylas-core/pubsub-connector.js @@ -0,0 +1,91 @@ +const Rx = require('rx') +const bluebird = require('bluebird') +const redis = require("redis"); +bluebird.promisifyAll(redis.RedisClient.prototype); +bluebird.promisifyAll(redis.Multi.prototype); + +class PubsubConnector { + constructor() { + this._broadcastClient = null; + this._listenClient = null; + this._listenClientSubs = {}; + } + + buildClient() { + const client = redis.createClient(process.env.REDIS_URL || null); + client.on("error", console.error); + return client; + } + + broadcastClient() { + if (!this._broadcastClient) { + this._broadcastClient = this.buildClient(); + } + return this._broadcastClient; + } + + channelForAccount(accountId) { + return `a-${accountId}`; + } + + channelForAccountDeltas(accountId) { + return `a-${accountId}-deltas`; + } + + // Shared channel + + notifyAccountChange(accountId) { + const channel = this.channelForAccount(accountId); + this.broadcastClient().publish(channel, 'modified'); + } + + observableForAccountChanges(accountId) { + if (!this._listenClient) { + this._listenClient = this.buildClient(); + this._listenClientSubs = {}; + } + + const channel = this.channelForAccount(accountId); + return Rx.Observable.create((observer) => { + this._listenClient.on("message", (msgChannel, message) => { + if (msgChannel !== channel) { return } + observer.onNext(message) + }); + + if (!this._listenClientSubs[channel]) { + this._listenClientSubs[channel] = 1; + this._listenClient.subscribe(channel); + } else { + this._listenClientSubs[channel] += 1; + } + return () => { + this._listenClientSubs[channel] -= 1; + if (this._listenClientSubs[channel] === 0) { + this._listenClient.unsubscribe(channel); + } + } + }) + } + + + // Account (delta streaming) channels + + notifyAccountDeltas(accountId, data) { + const channel = this.channelForAccountDeltas(accountId); + this.broadcastClient().publish(channel, JSON.stringify(data)) + } + + observableForAccountDeltas(accountId) { + return Rx.Observable.create((observer) => { + const sub = this.buildClient(); + sub.on("message", (channel, message) => observer.onNext(message)); + sub.subscribe(this.channelForAccountDeltas(accountId)); + return () => { + sub.unsubscribe(); + sub.quit(); + } + }) + } +} + +module.exports = new PubsubConnector() diff --git a/packages/nylas-core/transaction-log.js b/packages/nylas-core/transaction-log.js index 77a3a9474..133efad70 100644 --- a/packages/nylas-core/transaction-log.js +++ b/packages/nylas-core/transaction-log.js @@ -1,4 +1,4 @@ -const AccountPubsub = require('./account-pubsub') +const PubsubConnector = require('./pubsub-connector') class TransactionLog { constructor(db) { @@ -25,7 +25,8 @@ class TransactionLog { ); this.db.Transaction.create(transactionData); transactionData.object = sequelizeHookData.dataValues; - AccountPubsub.notify(this.db.accountId, transactionData); + + PubsubConnector.notifyAccountDeltas(this.db.accountId, transactionData); } } diff --git a/packages/nylas-message-processor/index.js b/packages/nylas-message-processor/index.js index b50d83cd1..40a3269fd 100644 --- a/packages/nylas-message-processor/index.js +++ b/packages/nylas-message-processor/index.js @@ -1,4 +1,4 @@ -const {DatabaseConnectionFactory} = require(`nylas-core`) +const {DatabaseConnector} = require(`nylas-core`) const {processors} = require('./processors') // List of the attributes of Message that the processor should be allowed to change. @@ -21,7 +21,7 @@ function saveMessage(message) { } function processMessage({messageId, accountId}) { - DatabaseConnectionFactory.forAccount(accountId) + DatabaseConnector.forAccount(accountId) .then(({Message}) => Message.find({where: {id: messageId}}).then((message) => runPipeline(accountId, message) diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 65adcd614..dc6692444 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -1,24 +1,21 @@ global.Promise = require('bluebird'); -const {DatabaseConnectionFactory} = require(`nylas-core`) -const SyncWorkerPool = require('./sync-worker-pool'); -const workerPool = new SyncWorkerPool(); +const {DatabaseConnector} = require(`nylas-core`) +const SyncProcessManager = require('./sync-process-manager'); -const start = () => { - DatabaseConnectionFactory.forShared().then((db) => { - const {Account} = db; - Account.findAll().then((accounts) => { - if (accounts.length === 0) { - console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) - console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.amessagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) - } - accounts.forEach((account) => { - workerPool.addWorkerForAccount(account); - }); - }); +const manager = new SyncProcessManager(); + +DatabaseConnector.forShared().then((db) => { + const {Account} = db; + Account.findAll().then((accounts) => { + if (accounts.length === 0) { + console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) + console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.amessagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) + } + manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => { + manager.start(); + }) }); -} +}); -start(); - -global.workerPool = workerPool; +global.manager = manager; diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js new file mode 100644 index 000000000..a6bace14f --- /dev/null +++ b/packages/nylas-sync/sync-process-manager.js @@ -0,0 +1,179 @@ +const os = require('os'); +const SyncWorker = require('./sync-worker'); +const {DatabaseConnector, PubsubConnector} = require(`nylas-core`) + +const CPU_COUNT = os.cpus().length; +const IDENTITY = `${os.hostname()}-${process.pid}`; + +const ACCOUNTS_UNCLAIMED = 'accounts:unclaimed'; +const ACCOUNTS_CLAIMED_PREFIX = 'accounts:id-'; +const ACCOUNTS_FOR = (id) => `${ACCOUNTS_CLAIMED_PREFIX}${id}`; +const HEARTBEAT_FOR = (id) => `heartbeat:${id}`; +const HEARTBEAT_EXPIRES = 30; // 2 min in prod? + +/* +Accounts ALWAYS exist in either `accounts:unclaimed` or an `accounts:{id}` list. +They are atomically moved between these sets as they are claimed and returned. + +Periodically, each worker in the pool looks at all the `accounts:{id}` lists. +For each list it finds, it checks for the existence of `heartbeat:{id}`, a key +that expires quickly if the sync process doesn't refresh it. + +If it does not find the key, it moves all of the accounts in the list back to +the unclaimed key. +*/ + +class SyncProcessManager { + constructor() { + this._workers = {}; + this._listenForSyncsClient = null; + this._exiting = false; + } + + start() { + console.log(`SyncWorkerPool: Starting with ID ${IDENTITY}`) + + this.unassignAccountsAssignedTo(IDENTITY).then(() => { + this.unassignAccountsMissingHeartbeats(); + this.update(); + }); + + setInterval(() => this.updateHeartbeat(), HEARTBEAT_EXPIRES / 5.0 * 1000); + this.updateHeartbeat(); + + process.on('SIGINT', () => this.onSigInt()); + } + + updateHeartbeat() { + const key = HEARTBEAT_FOR(IDENTITY); + const client = PubsubConnector.broadcastClient(); + client.setAsync(key, Date.now()).then(() => + client.expireAsync(key, HEARTBEAT_EXPIRES) + ).then(() => + console.log("SyncWorkerPool: Published heartbeat.") + ) + } + + onSigInt() { + console.log(`SyncWorkerPool: Exiting...`) + this._exiting = true; + + this.unassignAccountsAssignedTo(IDENTITY).then(() => + PubsubConnector.broadcastClient().delAsync(ACCOUNTS_FOR(IDENTITY)).then(() => + PubsubConnector.broadcastClient().delAsync(HEARTBEAT_FOR(IDENTITY)) + ) + ).finally(() => { + process.exit(1); + }); + } + + ensureAccountIDsInRedis(accountIds) { + const client = PubsubConnector.broadcastClient(); + + let unseenIds = [].concat(accountIds); + + return Promise.each(client.keysAsync(`accounts:*`), (key) => + client.lrangeAsync(key, 0, 20000).then((foundIds) => { + unseenIds = unseenIds.filter((a) => !foundIds.includes(`${a}`)) + }) + ).finally(() => { + if (unseenIds.length === 0) { + return; + } + console.log(`SyncWorkerPool: Adding account IDs ${unseenIds.join(',')} to redis.`) + unseenIds.map((id) => client.lpushAsync(ACCOUNTS_UNCLAIMED, id)); + }); + } + + unassignAccountsMissingHeartbeats() { + const client = PubsubConnector.broadcastClient(); + + console.log("SyncWorkerPool: Starting unassignment for processes missing heartbeats.") + + Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => { + const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, ''); + return client.existsAsync(HEARTBEAT_FOR(id)).then((exists) => + (exists ? Promise.resolve() : this.unassignAccountsAssignedTo(id)) + ) + }).finally(() => { + const delay = HEARTBEAT_EXPIRES * 1000; + setTimeout(() => this.unassignAccountsMissingHeartbeats(), delay); + }); + } + + unassignAccountsAssignedTo(identity) { + const src = ACCOUNTS_FOR(identity); + const dst = ACCOUNTS_UNCLAIMED; + + const unassignOne = (count) => + PubsubConnector.broadcastClient().rpoplpushAsync(src, dst).then((val) => + (val ? unassignOne(count + 1) : Promise.resolve(count)) + ) + + return unassignOne(0).then((returned) => { + console.log(`SyncWorkerPool: Returned ${returned} accounts assigned to ${identity}.`) + }); + } + + update() { + this.ensureCapacity().then(() => { + console.log(`SyncWorkerPool: Voluntering to sync additional account.`) + this.acceptUnclaimedAccount().finally(() => { + this.update(); + }); + }) + .catch((err) => { + console.log(`SyncWorkerPool: No capacity for additional accounts. ${err.message}`) + setTimeout(() => this.update(), 5000) + }); + } + + ensureCapacity() { + if (os.freemem() < 20 * 1024 * 1024) { + return Promise.reject(new Error(`<20MB RAM free (${os.freemem()} bytes)`)); + } + + const fiveMinuteLoadAvg = os.loadavg()[1]; + if (fiveMinuteLoadAvg > CPU_COUNT * 0.9) { + return Promise.reject(new Error(`CPU load > 90% (${fiveMinuteLoadAvg} - ${CPU_COUNT} cores)`)); + } + + if (this._exiting) { + return Promise.reject(new Error('Quitting...')) + } + + return Promise.resolve(); + } + + acceptUnclaimedAccount() { + if (!this._waitForAccountClient) { + this._waitForAccountClient = PubsubConnector.buildClient(); + } + + const src = ACCOUNTS_UNCLAIMED; + const dst = ACCOUNTS_FOR(IDENTITY); + + return this._waitForAccountClient.brpoplpushAsync(src, dst, 10000) + .then((accountId) => { + if (accountId) { + this.addWorkerForAccountId(accountId); + } + }); + } + + addWorkerForAccountId(accountId) { + DatabaseConnector.forShared().then(({Account}) => { + Account.find({where: {id: accountId}}).then((account) => { + if (!account) { + return; + } + DatabaseConnector.forAccount(account.id).then((db) => { + console.log(`SyncWorkerPool: Starting worker for Account ${accountId}`) + this._workers[account.id] = new SyncWorker(account, db); + }); + }); + }); + } +} + +module.exports = SyncProcessManager; diff --git a/packages/nylas-sync/sync-worker-pool.js b/packages/nylas-sync/sync-worker-pool.js deleted file mode 100644 index aeca79a18..000000000 --- a/packages/nylas-sync/sync-worker-pool.js +++ /dev/null @@ -1,16 +0,0 @@ -const SyncWorker = require('./sync-worker'); -const {DatabaseConnectionFactory} = require(`nylas-core`) - -class SyncWorkerPool { - constructor() { - this._workers = {}; - } - - addWorkerForAccount(account) { - DatabaseConnectionFactory.forAccount(account.id).then((db) => { - this._workers[account.id] = new SyncWorker(account, db); - }); - } -} - -module.exports = SyncWorkerPool; diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 1e3566c26..819cd0bb6 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -1,4 +1,8 @@ -const {IMAPConnection} = require('nylas-core'); +const { + IMAPConnection, + PubsubConnector, + DatabaseConnector, +} = require('nylas-core'); const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') const SyncMailboxOperation = require('./imap/sync-mailbox-operation') // @@ -28,17 +32,28 @@ class SyncWorker { this.syncNow(); this.scheduleExpiration(); + + this._listener = PubsubConnector.observableForAccountChanges(account.id).subscribe(() => { + this.onAccountChanged(); + }); + } + + cleanup() { + this._listener.dispose(); } - // TODO: How does this get called? onAccountChanged() { - this.syncNow(); - this.scheduleExpiration(); + DatabaseConnector.forShared().then(({Account}) => { + Account.find({where: {id: this._account.id}}).then((account) => { + this._account = account; + this.syncNow(); + this.scheduleExpiration(); + }) + }); } onExpired() { - // Returning syncs to the unclaimed queue every so often is healthy. - // TODO: That. + this.cleanup(); } onSyncDidComplete() { From ad1683c9a55a16321c5d0a175a6166414dd0d946 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 23 Jun 2016 10:11:07 -0700 Subject: [PATCH 041/800] Fix reference to __base --- packages/nylas-message-processor/processors/threading.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 3d98bea52..30ef96be9 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -1,4 +1,4 @@ -const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`) +const {DatabaseConnectionFactory} = require('nylas-core') function processMessage({message, accountId}) { return DatabaseConnectionFactory.forAccount(accountId) From a917022505b76504f6c20f05342ec1fb9bb9792f Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Thu, 23 Jun 2016 10:26:41 -0700 Subject: [PATCH 042/800] Update order of threading processor --- packages/nylas-message-processor/processors/threading.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 30ef96be9..d6be145a7 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -44,8 +44,9 @@ function matchThread({db, accountId, message}) { } function cleanSubject(subject) { - if (subject === null) + if (subject === null) { return "" + } const regex = new RegExp(/^((re|fw|fwd|aw|wg|undeliverable|undelivered):\s*)+/ig) const cleanedSubject = subject.replace(regex, (match) => "") return cleanedSubject @@ -148,6 +149,6 @@ function isSentToSelf({message, match}) { */ module.exports = { - order: 0, + order: 1, processMessage, } From ae54192ed65aab992a082e7e229c801c193c91d9 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Thu, 23 Jun 2016 10:33:38 -0700 Subject: [PATCH 043/800] Update threading and fix typo --- .../processors/threading.js | 18 ++++++++++++------ packages/nylas-sync/app.js | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index d6be145a7..ddcb18df5 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -21,11 +21,8 @@ function addMessageToThread({db, accountId, message}) { function matchThread({db, accountId, message}) { const {Thread} = db - - // TODO: Add once we have some test data with this header - /* - if (message.headers['in-reply-to']) { - return getThreadFromHeader() // Doesn't exist yet + if (message.headers.references) { + return getThreadFromReferences() .then((thread) => { if (thread) { return thread @@ -36,13 +33,22 @@ function matchThread({db, accountId, message}) { }) }) } - */ + return Thread.create({ subject: message.subject, cleanedSubject: cleanSubject(message.subject), }) } +function getThreadFromReferences({db, references}) { + const {Message} = db + const messageId = references.split()[references.length - 1] + return Message.find({where: {messageId: messageId}}) + .then((message) => { + return message.getThread() + }) +} + function cleanSubject(subject) { if (subject === null) { return "" diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index dc6692444..5d7319245 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -10,7 +10,7 @@ DatabaseConnector.forShared().then((db) => { Account.findAll().then((accounts) => { if (accounts.length === 0) { console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) - console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.amessagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) + console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) } manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => { manager.start(); From c2e9093b4282a1fa31dbed536465795f8e48656c Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 23 Jun 2016 11:44:57 -0700 Subject: [PATCH 044/800] Escalate sync policy based on number of stream connections --- packages/nylas-api/routes/auth.js | 18 +++++++----------- packages/nylas-api/routes/delta.js | 11 ++++++++--- packages/nylas-core/sync-policy.js | 23 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 14 deletions(-) create mode 100644 packages/nylas-core/sync-policy.js diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index 6ccb02118..9eaf30a67 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -2,7 +2,12 @@ const Joi = require('Joi'); const _ = require('underscore'); const Serialization = require('../serialization'); -const {IMAPConnection, PubsubConnector, DatabaseConnector} = require('nylas-core'); +const { + IMAPConnection, + PubsubConnector, + DatabaseConnector, + SyncPolicy +} = require('nylas-core'); const imapSmtpSettings = Joi.object().keys({ imap_host: [Joi.string().ip().required(), Joi.string().hostname().required()], @@ -22,15 +27,6 @@ const exchangeSettings = Joi.object().keys({ eas_server_host: [Joi.string().ip().required(), Joi.string().hostname().required()], }).required(); -const defaultSyncPolicy = { - afterSync: 'idle', - interval: 30 * 1000, - folderSyncOptions: { - deepFolderScan: 5 * 60 * 1000, - }, - expiration: Date.now() + 60 * 60 * 1000, -}; - module.exports = (server) => { server.route({ method: 'POST', @@ -75,7 +71,7 @@ module.exports = (server) => { const account = Account.build({ name: name, emailAddress: email, - syncPolicy: defaultSyncPolicy, + syncPolicy: SyncPolicy.defaultPolicy(), connectionSettings: _.pick(settings, [ 'imap_host', 'imap_port', 'smtp_host', 'smtp_port', diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js index fc5e603bf..25efafa91 100644 --- a/packages/nylas-api/routes/delta.js +++ b/packages/nylas-api/routes/delta.js @@ -1,6 +1,6 @@ const Rx = require('rx') const _ = require('underscore'); -const {PubsubConnector} = require(`nylas-core`); +const {PubsubConnector, SyncPolicy} = require(`nylas-core`); function keepAlive(request) { const until = Rx.Observable.fromCallback(request.on)("disconnect") @@ -49,15 +49,20 @@ module.exports = (server) => { path: '/delta/streaming', handler: (request, reply) => { const outputStream = createOutputStream(); + const account = request.auth.credentials; + PubsubConnector.incrementActivePolicyLockForAccount(account.id) request.getAccountDatabase().then((db) => { const source = Rx.Observable.merge( - PubsubConnector.observableForAccountDeltas(db.accountId), + PubsubConnector.observableForAccountDeltas(account.id), initialTransactions(db, request), keepAlive(request) ).subscribe(outputStream.pushJSON) - request.on("disconnect", () => source.dispose()); + request.on("disconnect", () => { + PubsubConnector.decrementActivePolicyLockForAccount(account.id) + source.dispose() + }); }); reply(outputStream) diff --git a/packages/nylas-core/sync-policy.js b/packages/nylas-core/sync-policy.js new file mode 100644 index 000000000..5d2d59e0d --- /dev/null +++ b/packages/nylas-core/sync-policy.js @@ -0,0 +1,23 @@ +class SyncPolicy { + static defaultPolicy() { + return { + afterSync: 'close', + interval: 120 * 1000, + folderSyncOptions: { + deepFolderScan: 10 * 60 * 1000, + }, + }; + } + + static activeUserPolicy() { + return { + afterSync: 'idle', + interval: 30 * 1000, + folderSyncOptions: { + deepFolderScan: 5 * 60 * 1000, + }, + }; + } +} + +module.exports = SyncPolicy From f7c647f7ba93e5c694ca19f333474692d3f78a53 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 23 Jun 2016 11:45:24 -0700 Subject: [PATCH 045/800] Return syncs to unclaimed queue after CLAIM_DURATION, just because it's healthy --- packages/nylas-core/database-connector.js | 11 +++++ packages/nylas-core/pubsub-connector.js | 30 ++++++++++++ packages/nylas-sync/sync-process-manager.js | 51 +++++++++++++++------ packages/nylas-sync/sync-worker.js | 24 ++++------ 4 files changed, 85 insertions(+), 31 deletions(-) diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index 8c573431b..812f6427b 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -75,6 +75,17 @@ class DatabaseConnector { db.sequelize = sequelize; db.Sequelize = Sequelize; + const changeObserver = ({dataValues, $modelOptions}) => { + if ($modelOptions.name.singular === 'Account') { + const PubsubConnector = require('./pubsub-connector'); + PubsubConnector.notifyAccountChange(dataValues.id); + } + } + + sequelize.addHook("afterCreate", changeObserver) + sequelize.addHook("afterUpdate", changeObserver) + sequelize.addHook("afterDelete", changeObserver) + return sequelize.authenticate().then(() => sequelize.sync() ).thenReturn(db); diff --git a/packages/nylas-core/pubsub-connector.js b/packages/nylas-core/pubsub-connector.js index 533b30a5f..e251811f7 100644 --- a/packages/nylas-core/pubsub-connector.js +++ b/packages/nylas-core/pubsub-connector.js @@ -1,6 +1,9 @@ const Rx = require('rx') const bluebird = require('bluebird') const redis = require("redis"); + +const SyncPolicy = require('./sync-policy'); + bluebird.promisifyAll(redis.RedisClient.prototype); bluebird.promisifyAll(redis.Multi.prototype); @@ -34,6 +37,33 @@ class PubsubConnector { // Shared channel + assignPolicy(accountId, policy) { + console.log(`Changing policy for ${accountId} to ${JSON.stringify(policy)}`) + const DatabaseConnector = require('./database-connector'); + DatabaseConnector.forShared().then(({Account}) => { + Account.find({where: {id: accountId}}).then((account) => { + account.syncPolicy = policy; + account.save() + }) + }); + } + + incrementActivePolicyLockForAccount(accountId) { + this.broadcastClient().incrAsync(`connections-${accountId}`).then((val) => { + if (val === 1) { + this.assignPolicy(accountId, SyncPolicy.activeUserPolicy()) + } + }) + } + + decrementActivePolicyLockForAccount(accountId) { + this.broadcastClient().decrAsync(`connections-${accountId}`).then((val) => { + if (val === 0) { + this.assignPolicy(accountId, SyncPolicy.defaultPolicy()) + } + }); + } + notifyAccountChange(accountId) { const channel = this.channelForAccount(accountId); this.broadcastClient().publish(channel, 'modified'); diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index a6bace14f..3eeccb074 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -10,6 +10,7 @@ const ACCOUNTS_CLAIMED_PREFIX = 'accounts:id-'; const ACCOUNTS_FOR = (id) => `${ACCOUNTS_CLAIMED_PREFIX}${id}`; const HEARTBEAT_FOR = (id) => `heartbeat:${id}`; const HEARTBEAT_EXPIRES = 30; // 2 min in prod? +const CLAIM_DURATION = 10 * 60 * 1000; // 2 hours on prod? /* Accounts ALWAYS exist in either `accounts:unclaimed` or an `accounts:{id}` list. @@ -31,7 +32,7 @@ class SyncProcessManager { } start() { - console.log(`SyncWorkerPool: Starting with ID ${IDENTITY}`) + console.log(`ProcessManager: Starting with ID ${IDENTITY}`) this.unassignAccountsAssignedTo(IDENTITY).then(() => { this.unassignAccountsMissingHeartbeats(); @@ -50,12 +51,12 @@ class SyncProcessManager { client.setAsync(key, Date.now()).then(() => client.expireAsync(key, HEARTBEAT_EXPIRES) ).then(() => - console.log("SyncWorkerPool: Published heartbeat.") + console.log("ProcessManager: Published heartbeat.") ) } onSigInt() { - console.log(`SyncWorkerPool: Exiting...`) + console.log(`ProcessManager: Exiting...`) this._exiting = true; this.unassignAccountsAssignedTo(IDENTITY).then(() => @@ -80,7 +81,7 @@ class SyncProcessManager { if (unseenIds.length === 0) { return; } - console.log(`SyncWorkerPool: Adding account IDs ${unseenIds.join(',')} to redis.`) + console.log(`ProcessManager: Adding account IDs ${unseenIds.join(',')} to ${ACCOUNTS_UNCLAIMED}.`) unseenIds.map((id) => client.lpushAsync(ACCOUNTS_UNCLAIMED, id)); }); } @@ -88,7 +89,7 @@ class SyncProcessManager { unassignAccountsMissingHeartbeats() { const client = PubsubConnector.broadcastClient(); - console.log("SyncWorkerPool: Starting unassignment for processes missing heartbeats.") + console.log("ProcessManager: Starting unassignment for processes missing heartbeats.") Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => { const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, ''); @@ -111,19 +112,19 @@ class SyncProcessManager { ) return unassignOne(0).then((returned) => { - console.log(`SyncWorkerPool: Returned ${returned} accounts assigned to ${identity}.`) + console.log(`ProcessManager: Returned ${returned} accounts assigned to ${identity}.`) }); } update() { this.ensureCapacity().then(() => { - console.log(`SyncWorkerPool: Voluntering to sync additional account.`) + console.log(`ProcessManager: Voluntering to sync additional account.`) this.acceptUnclaimedAccount().finally(() => { this.update(); }); }) .catch((err) => { - console.log(`SyncWorkerPool: No capacity for additional accounts. ${err.message}`) + console.log(`ProcessManager: Decided not to sync additional account. ${err.message}`) setTimeout(() => this.update(), 5000) }); } @@ -139,7 +140,7 @@ class SyncProcessManager { } if (this._exiting) { - return Promise.reject(new Error('Quitting...')) + return Promise.reject(new Error('Process is exiting.')) } return Promise.resolve(); @@ -153,11 +154,10 @@ class SyncProcessManager { const src = ACCOUNTS_UNCLAIMED; const dst = ACCOUNTS_FOR(IDENTITY); - return this._waitForAccountClient.brpoplpushAsync(src, dst, 10000) - .then((accountId) => { - if (accountId) { - this.addWorkerForAccountId(accountId); - } + return this._waitForAccountClient.brpoplpushAsync(src, dst, 10000).then((accountId) => { + if (!accountId) { return } + this.addWorkerForAccountId(accountId); + setTimeout(() => this.removeWorker(), CLAIM_DURATION); }); } @@ -168,12 +168,33 @@ class SyncProcessManager { return; } DatabaseConnector.forAccount(account.id).then((db) => { - console.log(`SyncWorkerPool: Starting worker for Account ${accountId}`) + if (this._exiting) { + return; + } + console.log(`ProcessManager: Starting worker for Account ${accountId}`) this._workers[account.id] = new SyncWorker(account, db); }); }); }); } + + removeWorker() { + const src = ACCOUNTS_FOR(IDENTITY); + const dst = ACCOUNTS_UNCLAIMED; + + return PubsubConnector.broadcastClient().rpoplpushAsync(src, dst).then((accountId) => { + if (!accountId) { + return; + } + + console.log(`ProcessManager: Returning account ${accountId} to unclaimed pool.`) + + if (this._workers[accountId]) { + this._workers[accountId].cleanup(); + } + this._workers[accountId] = null; + }); + } } module.exports = SyncProcessManager; diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 819cd0bb6..d0c18886c 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -3,6 +3,7 @@ const { PubsubConnector, DatabaseConnector, } = require('nylas-core'); + const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') const SyncMailboxOperation = require('./imap/sync-mailbox-operation') // @@ -29,9 +30,9 @@ class SyncWorker { this._syncTimer = null; this._expirationTimer = null; + this._destroyed = false; this.syncNow(); - this.scheduleExpiration(); this._listener = PubsubConnector.observableForAccountChanges(account.id).subscribe(() => { this.onAccountChanged(); @@ -39,34 +40,32 @@ class SyncWorker { } cleanup() { + this._destroyed = true; this._listener.dispose(); + this._conn.end(); } onAccountChanged() { + console.log("SyncWorker: Detected change to account. Reloading and syncing now.") DatabaseConnector.forShared().then(({Account}) => { Account.find({where: {id: this._account.id}}).then((account) => { this._account = account; this.syncNow(); - this.scheduleExpiration(); }) }); } - onExpired() { - this.cleanup(); - } - onSyncDidComplete() { const {afterSync} = this._account.syncPolicy; if (afterSync === 'idle') { this.getInboxCategory().then((inboxCategory) => { this._conn.openBox(inboxCategory.name, true).then(() => { - console.log(" - Idling on inbox category"); + console.log("SyncWorker: - Idling on inbox category"); }); }); } else if (afterSync === 'close') { - console.log(" - Closing connection"); + console.log("SyncWorker: - Closing connection"); this._conn.end(); this._conn = null; } else { @@ -155,19 +154,12 @@ class SyncWorker { }); } - scheduleExpiration() { - const {expiration} = this._account.syncPolicy; - - clearTimeout(this._expirationTimer); - this._expirationTimer = setTimeout(() => this.onExpired(), expiration); - } - scheduleNextSync() { const {interval} = this._account.syncPolicy; if (interval) { const target = this._lastSyncTime + interval; - console.log(`Next sync scheduled for ${new Date(target).toLocaleString()}`); + console.log(`SyncWorker: Next sync scheduled for ${new Date(target).toLocaleString()}`); this._syncTimer = setTimeout(() => { this.syncNow(); }, target - Date.now()); From 8e692982bb7bf25e63e8bcc774d46a65dc0824b5 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 23 Jun 2016 12:02:57 -0700 Subject: [PATCH 046/800] Remove concept of self-limiting workers, will use cloudwatch metrics collection to scale fleet instead of queue length --- packages/nylas-sync/sync-process-manager.js | 45 +++++++++------------ 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index 3eeccb074..a9d41d38c 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -22,6 +22,16 @@ that expires quickly if the sync process doesn't refresh it. If it does not find the key, it moves all of the accounts in the list back to the unclaimed key. + +Sync processes only claim an account for a fixed period of time. This means that +an engineer can add new sync machines to the pool and the load across instances +will balance on it's own. It also means one bad instance will not permanently +disrupt sync for any accounts. (Eg: instance has faulty network connection.) + +Sync processes periodically claim accounts when they can find them, regardless +of how busy they are. A separate API (`/routes/monitoring`) allows CloudWatch +to decide whether to spin up instances or take them offline based on CPU/RAM +utilization across the pool. */ class SyncProcessManager { @@ -73,6 +83,8 @@ class SyncProcessManager { let unseenIds = [].concat(accountIds); + console.log("ProcessManager: Starting scan for accountIds in database that are not present in Redis.") + return Promise.each(client.keysAsync(`accounts:*`), (key) => client.lrangeAsync(key, 0, 20000).then((foundIds) => { unseenIds = unseenIds.filter((a) => !foundIds.includes(`${a}`)) @@ -117,35 +129,16 @@ class SyncProcessManager { } update() { - this.ensureCapacity().then(() => { - console.log(`ProcessManager: Voluntering to sync additional account.`) - this.acceptUnclaimedAccount().finally(() => { - this.update(); - }); - }) - .catch((err) => { - console.log(`ProcessManager: Decided not to sync additional account. ${err.message}`) - setTimeout(() => this.update(), 5000) + console.log(`ProcessManager: Searching for an unclaimed account to sync.`) + + this.acceptUnclaimedAccount().finally(() => { + if (this._exiting) { + return; + } + this.update(); }); } - ensureCapacity() { - if (os.freemem() < 20 * 1024 * 1024) { - return Promise.reject(new Error(`<20MB RAM free (${os.freemem()} bytes)`)); - } - - const fiveMinuteLoadAvg = os.loadavg()[1]; - if (fiveMinuteLoadAvg > CPU_COUNT * 0.9) { - return Promise.reject(new Error(`CPU load > 90% (${fiveMinuteLoadAvg} - ${CPU_COUNT} cores)`)); - } - - if (this._exiting) { - return Promise.reject(new Error('Process is exiting.')) - } - - return Promise.resolve(); - } - acceptUnclaimedAccount() { if (!this._waitForAccountClient) { this._waitForAccountClient = PubsubConnector.buildClient(); From 1460a0ae9f46fa7fbb6affb871239fdacd6cb637 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 23 Jun 2016 12:20:47 -0600 Subject: [PATCH 047/800] Rename Refresh to Sync --- ...es-operation.js => fetch-category-list.js} | 6 +- ...ation.js => fetch-messages-in-category.js} | 64 ++++++++++--------- packages/nylas-sync/sync-worker.js | 31 ++++----- 3 files changed, 52 insertions(+), 49 deletions(-) rename packages/nylas-sync/imap/{refresh-mailboxes-operation.js => fetch-category-list.js} (95%) rename packages/nylas-sync/imap/{sync-mailbox-operation.js => fetch-messages-in-category.js} (83%) diff --git a/packages/nylas-sync/imap/refresh-mailboxes-operation.js b/packages/nylas-sync/imap/fetch-category-list.js similarity index 95% rename from packages/nylas-sync/imap/refresh-mailboxes-operation.js rename to packages/nylas-sync/imap/fetch-category-list.js index f0d475d5c..9cb46cd7e 100644 --- a/packages/nylas-sync/imap/refresh-mailboxes-operation.js +++ b/packages/nylas-sync/imap/fetch-category-list.js @@ -1,6 +1,6 @@ -class RefreshMailboxesOperation { +class FetchCategoryList { description() { - return `RefreshMailboxesOperation`; + return `FetchCategoryList`; } _roleForMailbox(boxName, box) { @@ -86,4 +86,4 @@ class RefreshMailboxesOperation { } } -module.exports = RefreshMailboxesOperation; +module.exports = FetchCategoryList; diff --git a/packages/nylas-sync/imap/sync-mailbox-operation.js b/packages/nylas-sync/imap/fetch-messages-in-category.js similarity index 83% rename from packages/nylas-sync/imap/sync-mailbox-operation.js rename to packages/nylas-sync/imap/fetch-messages-in-category.js index 56aab99f3..a56dc49b7 100644 --- a/packages/nylas-sync/imap/sync-mailbox-operation.js +++ b/packages/nylas-sync/imap/fetch-messages-in-category.js @@ -5,17 +5,17 @@ const {Capabilities} = IMAPConnection; const MessageFlagAttributes = ['id', 'CategoryUID', 'unread', 'starred'] -class SyncMailboxOperation { +class FetchMessagesInCategory { constructor(category, options) { this._category = category; this._options = options; if (!this._category) { - throw new Error("SyncMailboxOperation requires a category") + throw new Error("FetchMessagesInCategory requires a category") } } description() { - return `SyncMailboxOperation (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`; + return `FetchMessagesInCategory (${this._category.name} - ${this._category.id})\n Options: ${JSON.stringify(this._options)}`; } _getLowerBoundUID(count) { @@ -177,36 +177,42 @@ class SyncMailboxOperation { }); } + _shouldRunDeepScan() { + const {timeDeepScan} = this._category.syncState; + return Date.now() - (timeDeepScan || 0) > this._options.deepFolderScan + } + + _runDeepScan(range) { + const {Message} = this.db; + return this._imap.fetchUIDAttributes(range).then((remoteUIDAttributes) => + Message.findAll({ + where: {CategoryId: this._category.id}, + attributes: MessageFlagAttributes, + }).then((localMessageAttributes) => + Promise.props({ + upserts: this._createAndUpdateMessages(remoteUIDAttributes, localMessageAttributes), + deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes), + }) + ).then(() => { + return this.updateCategorySyncState({ + highestmodseq: this._box.highestmodseq, + timeDeepScan: Date.now(), + timeShallowScan: Date.now(), + }); + }) + ); + } + _fetchChangesToMessages() { - const {highestmodseq, timeDeepScan} = this._category.syncState; + const {highestmodseq} = this._category.syncState; const nextHighestmodseq = this._box.highestmodseq; - const {Message} = this._db; - const {limit} = this._options; - const range = `${this._getLowerBoundUID(limit)}:*`; + const range = `${this._getLowerBoundUID(this._options.limit)}:*`; console.log(` - fetching changes to messages ${range}`) - const shouldRunDeepScan = Date.now() - (timeDeepScan || 0) > this._options.deepFolderScan - - if (shouldRunDeepScan) { - return this._imap.fetchUIDAttributes(range).then((remoteUIDAttributes) => - Message.findAll({ - where: {CategoryId: this._category.id}, - attributes: MessageFlagAttributes, - }).then((localMessageAttributes) => - Promise.props({ - upserts: this._createAndUpdateMessages(remoteUIDAttributes, localMessageAttributes), - deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes), - }) - ).then(() => { - return this.updateCategorySyncState({ - highestmodseq: nextHighestmodseq, - timeDeepScan: Date.now(), - timeShallowScan: Date.now(), - }); - }) - ); + if (this._shouldRunDeepScan()) { + return this._runDeepScan(range) } let shallowFetch = null; @@ -222,7 +228,7 @@ class SyncMailboxOperation { } return shallowFetch.then((remoteUIDAttributes) => - Message.findAll({ + this._db.Message.findAll({ where: {CategoryId: this._category.id}, attributes: MessageFlagAttributes, }).then((localMessageAttributes) => @@ -257,4 +263,4 @@ class SyncMailboxOperation { } } -module.exports = SyncMailboxOperation; +module.exports = FetchMessagesInCategory; diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index d0c18886c..0e9ab3e54 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -4,8 +4,8 @@ const { DatabaseConnector, } = require('nylas-core'); -const RefreshMailboxesOperation = require('./imap/refresh-mailboxes-operation') -const SyncMailboxOperation = require('./imap/sync-mailbox-operation') +const FetchCategoryList = require('./imap/fetch-category-list') +const FetchMessagesInCategory = require('./imap/fetch-messages-in-category') // // account.syncPolicy = { // afterSync: 'idle', @@ -111,12 +111,12 @@ class SyncWorker { }); } - queueOperationsForUpdates() { + fetchCategoryList() { // todo: syncback operations belong here! - return this._conn.runOperation(new RefreshMailboxesOperation()) + return this._conn.runOperation(new FetchCategoryList()) } - queueOperationsForFolderSyncs() { + fetchMessagesInCategory() { const {Category} = this._db; const {folderSyncOptions} = this._account.syncPolicy; @@ -131,24 +131,21 @@ class SyncWorker { // ) return Promise.all(categoriesToSync.map((cat) => - this._conn.runOperation(new SyncMailboxOperation(cat, folderSyncOptions)) - )).then(() => { - this._lastSyncTime = Date.now(); - }); + this._conn.runOperation(new FetchMessagesInCategory(cat, folderSyncOptions)) + )) }); } syncNow() { clearTimeout(this._syncTimer); - this.ensureConnection().then(() => - this.queueOperationsForUpdates().then(() => - this.queueOperationsForFolderSyncs() - ) - ).catch((err) => { - // Sync has failed for some reason. What do we do?! - console.error(err); - }).finally(() => { + this.ensureConnection() + .then(this.fetchCategoryList.bind(this)) + .then(this.syncbackMessageActions.bind(this)) + .then(this.fetchMessagesInCategory.bind(this)) + .then(() => { this._lastSyncTime = Date.now() }) + .catch(console.error) + .finally(() => { this.onSyncDidComplete(); this.scheduleNextSync(); }); From 8160acc81e375ee2c98519ceb371c80f51e780c8 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 23 Jun 2016 13:18:33 -0600 Subject: [PATCH 048/800] Add imap dependency to package.json --- packages/nylas-core/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nylas-core/package.json b/packages/nylas-core/package.json index bd7fd3df7..f8c34791f 100644 --- a/packages/nylas-core/package.json +++ b/packages/nylas-core/package.json @@ -4,6 +4,7 @@ "description": "Core shared packages", "main": "index.js", "dependencies": { + "imap": "0.8.x", "sqlite3": "3.1.4" }, "author": "Nylas", From 6ad9cdd322bf9388b0915409fe49606d75809779 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 23 Jun 2016 14:15:30 -0600 Subject: [PATCH 049/800] Fixing broken processors --- packages/nylas-core/index.js | 1 + .../processors/threading.js | 64 +++++++++---------- .../imap/fetch-messages-in-category.js | 2 +- packages/nylas-sync/sync-worker.js | 5 +- 4 files changed, 38 insertions(+), 34 deletions(-) diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index f5ebeae89..5365e0289 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -2,5 +2,6 @@ module.exports = { DatabaseConnector: require('./database-connector'), PubsubConnector: require('./pubsub-connector'), IMAPConnection: require('./imap-connection'), + SyncPolicy: require('./sync-policy'), Config: require(`./config/${process.env.ENV || 'development'}`), } diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index ddcb18df5..ec676a046 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -1,25 +1,24 @@ -const {DatabaseConnectionFactory} = require('nylas-core') +const {DatabaseConnector} = require('nylas-core') -function processMessage({message, accountId}) { - return DatabaseConnectionFactory.forAccount(accountId) - .then((db) => addMessageToThread({db, accountId, message})) - .then((thread) => { - thread.addMessage(message) - message.setThread(thread) - return message +function cleanSubject(subject) { + if (subject === null) { + return "" + } + const regex = new RegExp(/^((re|fw|fwd|aw|wg|undeliverable|undelivered):\s*)+/ig) + const cleanedSubject = subject.replace(regex, () => "") + return cleanedSubject +} + +function getThreadFromReferences({db, references}) { + const {Message} = db + const messageId = references.split()[references.length - 1] + return Message.find({where: {messageId: messageId}}) + .then((message) => { + return message.getThread() }) } -function addMessageToThread({db, accountId, message}) { - const {Thread, Message} = db - if (message.threadId) { - return Thread.find({where: {threadId: message.threadId}}) - } - return matchThread({db, accountId, message}) - .then((thread) => (thread)) -} - -function matchThread({db, accountId, message}) { +function matchThread({db, message}) { const {Thread} = db if (message.headers.references) { return getThreadFromReferences() @@ -40,22 +39,23 @@ function matchThread({db, accountId, message}) { }) } -function getThreadFromReferences({db, references}) { - const {Message} = db - const messageId = references.split()[references.length - 1] - return Message.find({where: {messageId: messageId}}) - .then((message) => { - return message.getThread() - }) +function addMessageToThread({db, accountId, message}) { + const {Thread} = db + if (message.threadId) { + return Thread.find({where: {threadId: message.threadId}}) + } + return matchThread({db, accountId, message}) + .then((thread) => (thread)) } -function cleanSubject(subject) { - if (subject === null) { - return "" - } - const regex = new RegExp(/^((re|fw|fwd|aw|wg|undeliverable|undelivered):\s*)+/ig) - const cleanedSubject = subject.replace(regex, (match) => "") - return cleanedSubject +function processMessage({message, accountId}) { + return DatabaseConnector.forAccount(accountId) + .then((db) => addMessageToThread({db, accountId, message})) + .then((thread) => { + thread.addMessage(message) + message.setThread(thread) + return message + }) } // TODO: Incorporate this more elaborate threading algorithm diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-category.js index a56dc49b7..38d884469 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-category.js +++ b/packages/nylas-sync/imap/fetch-messages-in-category.js @@ -183,7 +183,7 @@ class FetchMessagesInCategory { } _runDeepScan(range) { - const {Message} = this.db; + const {Message} = this._db; return this._imap.fetchUIDAttributes(range).then((remoteUIDAttributes) => Message.findAll({ where: {CategoryId: this._category.id}, diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 0e9ab3e54..66de25fa5 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -112,10 +112,13 @@ class SyncWorker { } fetchCategoryList() { - // todo: syncback operations belong here! return this._conn.runOperation(new FetchCategoryList()) } + syncbackMessageActions() { + return Promise.resolve() + } + fetchMessagesInCategory() { const {Category} = this._db; const {folderSyncOptions} = this._account.syncPolicy; From 5cc4841ac6fd8c3feb9c9d501cdfa8f1c3383750 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Thu, 23 Jun 2016 15:19:20 -0700 Subject: [PATCH 050/800] Add check for Gmail thread ID --- packages/nylas-message-processor/processors/threading.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index ec676a046..2b21bf72b 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -41,8 +41,9 @@ function matchThread({db, message}) { function addMessageToThread({db, accountId, message}) { const {Thread} = db - if (message.threadId) { - return Thread.find({where: {threadId: message.threadId}}) + // Check for Gmail's own thread ID + if (message.headers['X-GM-THRID']) { + return Thread.find({where: {threadId: message.headers['X-GM-THRID']}) } return matchThread({db, accountId, message}) .then((thread) => (thread)) From f9fe8368992c8c60f8e8d1ecb1a0d24f4d776d28 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Thu, 23 Jun 2016 15:21:54 -0700 Subject: [PATCH 051/800] Fix syntax error --- packages/nylas-message-processor/processors/threading.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 2b21bf72b..199a8c794 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -43,7 +43,7 @@ function addMessageToThread({db, accountId, message}) { const {Thread} = db // Check for Gmail's own thread ID if (message.headers['X-GM-THRID']) { - return Thread.find({where: {threadId: message.headers['X-GM-THRID']}) + return Thread.find({where: {threadId: message.headers['X-GM-THRID']}}) } return matchThread({db, accountId, message}) .then((thread) => (thread)) From b6f57f3ce8ccfecaa87b4b3f25f54348e9fd6e29 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Thu, 23 Jun 2016 15:44:03 -0700 Subject: [PATCH 052/800] Add threading algorithm --- .../processors/threading.js | 133 ++++++++++-------- 1 file changed, 71 insertions(+), 62 deletions(-) diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 199a8c794..8f5922b0f 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -1,66 +1,5 @@ const {DatabaseConnector} = require('nylas-core') -function cleanSubject(subject) { - if (subject === null) { - return "" - } - const regex = new RegExp(/^((re|fw|fwd|aw|wg|undeliverable|undelivered):\s*)+/ig) - const cleanedSubject = subject.replace(regex, () => "") - return cleanedSubject -} - -function getThreadFromReferences({db, references}) { - const {Message} = db - const messageId = references.split()[references.length - 1] - return Message.find({where: {messageId: messageId}}) - .then((message) => { - return message.getThread() - }) -} - -function matchThread({db, message}) { - const {Thread} = db - if (message.headers.references) { - return getThreadFromReferences() - .then((thread) => { - if (thread) { - return thread - } - return Thread.create({ - subject: message.subject, - cleanedSubject: cleanSubject(message.subject), - }) - }) - } - - return Thread.create({ - subject: message.subject, - cleanedSubject: cleanSubject(message.subject), - }) -} - -function addMessageToThread({db, accountId, message}) { - const {Thread} = db - // Check for Gmail's own thread ID - if (message.headers['X-GM-THRID']) { - return Thread.find({where: {threadId: message.headers['X-GM-THRID']}}) - } - return matchThread({db, accountId, message}) - .then((thread) => (thread)) -} - -function processMessage({message, accountId}) { - return DatabaseConnector.forAccount(accountId) - .then((db) => addMessageToThread({db, accountId, message})) - .then((thread) => { - thread.addMessage(message) - message.setThread(thread) - return message - }) -} - -// TODO: Incorporate this more elaborate threading algorithm -/* function fetchCorrespondingThread({db, accountId, message}) { const cleanedSubject = cleanSubject(message.subject) return getThreads({db, message, cleanedSubject}) @@ -153,7 +92,77 @@ function isSentToSelf({message, match}) { matchFrom === matchTo && messageTo === matchFrom) } -*/ + +function cleanSubject(subject) { + if (subject === null) { + return "" + } + const regex = new RegExp(/^((re|fw|fwd|aw|wg|undeliverable|undelivered):\s*)+/ig) + const cleanedSubject = subject.replace(regex, () => "") + return cleanedSubject +} + +function getThreadFromReferences({db, references}) { + const {Message} = db + const messageId = references.split()[references.length - 1] + return Message.find({where: {messageId: messageId}}) + .then((message) => { + return message.getThread() + }) +} + +function matchThread({db, accountId, message}) { + const {Thread} = db + if (message.headers.references) { + return getThreadFromReferences() + .then((thread) => { + if (thread) { + return thread + } + return fetchCorrespondingThread({db, accountId, message}) + .then((thread) => { + if (thread) { + return thread + } + return Thread.create({ + subject: message.subject, + cleanedSubject: cleanSubject(message.subject), + }) + }) + }) + } + + return fetchCorrespondingThread({db, accountId, message}) + .then((thread) => { + if (thread) { + return thread + } + return Thread.create({ + subject: message.subject, + cleanedSubject: cleanSubject(message.subject), + }) + }) +} + +function addMessageToThread({db, accountId, message}) { + const {Thread} = db + // Check for Gmail's own thread ID + if (message.headers['X-GM-THRID']) { + return Thread.find({where: {threadId: message.headers['X-GM-THRID']}}) + } + return matchThread({db, accountId, message}) + .then((thread) => (thread)) +} + +function processMessage({message, accountId}) { + return DatabaseConnector.forAccount(accountId) + .then((db) => addMessageToThread({db, accountId, message})) + .then((thread) => { + thread.addMessage(message) + message.setThread(thread) + return message + }) +} module.exports = { order: 1, From 3f5cac4342366db5c0c4b6c82bfc1663ec00f8c3 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 23 Jun 2016 16:46:52 -0600 Subject: [PATCH 053/800] Adding in syncback message actions --- packages/nylas-api/package.json | 1 + packages/nylas-api/routes/threads.js | 37 ++++++++++++++++++- packages/nylas-api/serialization.js | 3 ++ .../models/account/syncback-request.js | 26 +++++++++++++ packages/nylas-core/pubsub-connector.js | 5 +++ packages/nylas-sync/sync-worker.js | 14 ++++++- packages/nylas-sync/syncback-task-factory.js | 11 ++++++ 7 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 packages/nylas-core/models/account/syncback-request.js create mode 100644 packages/nylas-sync/syncback-task-factory.js diff --git a/packages/nylas-api/package.json b/packages/nylas-api/package.json index d023ff6f6..66c52eb25 100644 --- a/packages/nylas-api/package.json +++ b/packages/nylas-api/package.json @@ -14,6 +14,7 @@ "inert": "4.0.0", "joi": "8.4.2", "nylas-core": "0.x.x", + "nylas-sync": "0.x.x", "vision": "4.1.0" } } diff --git a/packages/nylas-api/routes/threads.js b/packages/nylas-api/routes/threads.js index d9ccb910f..8cd77e62a 100644 --- a/packages/nylas-api/routes/threads.js +++ b/packages/nylas-api/routes/threads.js @@ -15,7 +15,7 @@ module.exports = (server) => { }, response: { schema: Joi.array().items( - Serialization.jsonSchema('Account') + Serialization.jsonSchema('Thread') ), }, }, @@ -28,4 +28,39 @@ module.exports = (server) => { }) }, }); + + server.route({ + method: 'PUT', + path: '/threads/${id}', + config: { + description: 'Update a thread', + notes: 'Can move between folders', + tags: ['threads'], + validate: { + params: { + payload: { + folder_id: Joi.string(), + }, + }, + }, + response: { + schema: Joi.array().items( + Serialization.jsonSchema('Thread') + ), + }, + }, + handler: (request, reply) => { + request.getAccountDatabase().then((db) => { + db.SyncbackRequest.create({ + type: "MoveToFolder", + props: { + folderId: request.params.folder_id, + threadId: request.params.id, + }, + }).then((syncbackRequest) => { + reply(Serialization.jsonStringify(syncbackRequest)) + }) + }) + }, + }); }; diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 5304984d9..043a3a5ba 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -17,6 +17,9 @@ function jsonSchema(modelName) { if (modelName === 'Message') { return Joi.object(); } + if (modelName === 'Thread') { + return Joi.object(); + } if (modelName === 'Error') { return Joi.object(); } diff --git a/packages/nylas-core/models/account/syncback-request.js b/packages/nylas-core/models/account/syncback-request.js new file mode 100644 index 000000000..f120c4872 --- /dev/null +++ b/packages/nylas-core/models/account/syncback-request.js @@ -0,0 +1,26 @@ +module.exports = (sequelize, Sequelize) => { + const SyncbackRequest = sequelize.define('SyncbackRequest', { + type: Sequelize.STRING, + status: Sequelize.STRING, + error: { + type: Sequelize.STRING, + get: function get() { + return JSON.parse(this.getDataValue('error')) + }, + set: function set(val) { + this.setDataValue('error', JSON.stringify(val)); + }, + }, + props: { + type: Sequelize.STRING, + get: function get() { + return JSON.parse(this.getDataValue('props')) + }, + set: function set(val) { + this.setDataValue('props', JSON.stringify(val)); + }, + }, + }); + + return SyncbackRequest; +}; diff --git a/packages/nylas-core/pubsub-connector.js b/packages/nylas-core/pubsub-connector.js index e251811f7..c13d01ccf 100644 --- a/packages/nylas-core/pubsub-connector.js +++ b/packages/nylas-core/pubsub-connector.js @@ -116,6 +116,11 @@ class PubsubConnector { } }) } + + queueSyncbackTask({taskName, props}) { + const channel = this.channelForSyncbackTaskQueue(accountId); + this.broadcastClient().publish(channel, JSON.stringify(data)) + } } module.exports = new PubsubConnector() diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 66de25fa5..262fbdf9c 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -6,6 +6,7 @@ const { const FetchCategoryList = require('./imap/fetch-category-list') const FetchMessagesInCategory = require('./imap/fetch-messages-in-category') +const SyncbackTaskFactory = require('./syncback-task-factory') // // account.syncPolicy = { // afterSync: 'idle', @@ -116,7 +117,18 @@ class SyncWorker { } syncbackMessageActions() { - return Promise.resolve() + return Promise.resolve(); + // TODO + const {SyncbackRequest, accountId, Account} = this._db; + + return Account.find({where: {id: accountId}}).then((account) => { + return Promise.each(SyncbackRequest.findAll().then((reqs = []) => + reqs.map((request) => { + const task = SyncbackTaskFactory.create(account, request); + return this._conn.runOperation(task) + }) + )); + }); } fetchMessagesInCategory() { diff --git a/packages/nylas-sync/syncback-task-factory.js b/packages/nylas-sync/syncback-task-factory.js new file mode 100644 index 000000000..407d702b0 --- /dev/null +++ b/packages/nylas-sync/syncback-task-factory.js @@ -0,0 +1,11 @@ +/** + * Given a `SyncbackRequestObject` it creates the appropriate syncback task. + * + */ +class SyncbackTaskFactory { + static create(account, syncbackRequest) { + if (account) { + + } + } +} From 09bb7874f8b08b9bfa160b055f7818f898b8804a Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 23 Jun 2016 15:52:45 -0700 Subject: [PATCH 054/800] Dashboard with a sweet background. Also realtime assignment / policy view. --- packages/nylas-api/app.js | 2 + packages/nylas-core/imap-connection.js | 1 - packages/nylas-core/index.js | 3 + packages/nylas-core/pubsub-connector.js | 7 +- packages/nylas-core/scheduler-utils.js | 29 + packages/nylas-core/sync-policy.js | 2 +- packages/nylas-dashboard/app.js | 75 + packages/nylas-dashboard/package.json | 17 + packages/nylas-dashboard/public/css/app.css | 17 + packages/nylas-dashboard/public/index.html | 13 + packages/nylas-dashboard/public/js/app.jsx | 91 + .../nylas-dashboard/public/js/react-dom.js | 42 + .../public/js/react-dom.min.js | 12 + packages/nylas-dashboard/public/js/react.js | 19599 ++++++++++++++++ .../nylas-dashboard/public/js/react.min.js | 16 + packages/nylas-message-processor/index.js | 2 + packages/nylas-sync/sync-process-manager.js | 29 +- packages/nylas-sync/sync-worker.js | 2 +- 18 files changed, 19938 insertions(+), 21 deletions(-) create mode 100644 packages/nylas-core/scheduler-utils.js create mode 100644 packages/nylas-dashboard/app.js create mode 100644 packages/nylas-dashboard/package.json create mode 100644 packages/nylas-dashboard/public/css/app.css create mode 100644 packages/nylas-dashboard/public/index.html create mode 100644 packages/nylas-dashboard/public/js/app.jsx create mode 100644 packages/nylas-dashboard/public/js/react-dom.js create mode 100644 packages/nylas-dashboard/public/js/react-dom.min.js create mode 100644 packages/nylas-dashboard/public/js/react.js create mode 100644 packages/nylas-dashboard/public/js/react.min.js diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index 365e53904..29c4ee2f0 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -7,6 +7,8 @@ const Package = require('./package'); const fs = require('fs'); const path = require('path'); +global.Promise = require('bluebird'); + const server = new Hapi.Server(); server.connection({ port: process.env.PORT || 5100 }); diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index 11782518a..8a047f830 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -1,6 +1,5 @@ const Imap = require('imap'); const EventEmitter = require('events'); -const Promise = require('bluebird'); const Capabilities = { Gmail: 'X-GM-EXT-1', diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index 5365e0289..59733f444 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -1,7 +1,10 @@ +global.Promise = require('bluebird'); + module.exports = { DatabaseConnector: require('./database-connector'), PubsubConnector: require('./pubsub-connector'), IMAPConnection: require('./imap-connection'), SyncPolicy: require('./sync-policy'), + SchedulerUtils: require('./scheduler-utils'), Config: require(`./config/${process.env.ENV || 'development'}`), } diff --git a/packages/nylas-core/pubsub-connector.js b/packages/nylas-core/pubsub-connector.js index c13d01ccf..2022e0dca 100644 --- a/packages/nylas-core/pubsub-connector.js +++ b/packages/nylas-core/pubsub-connector.js @@ -1,11 +1,10 @@ const Rx = require('rx') -const bluebird = require('bluebird') const redis = require("redis"); const SyncPolicy = require('./sync-policy'); -bluebird.promisifyAll(redis.RedisClient.prototype); -bluebird.promisifyAll(redis.Multi.prototype); +Promise.promisifyAll(redis.RedisClient.prototype); +Promise.promisifyAll(redis.Multi.prototype); class PubsubConnector { constructor() { @@ -32,7 +31,7 @@ class PubsubConnector { } channelForAccountDeltas(accountId) { - return `a-${accountId}-deltas`; + return `deltas-${accountId}`; } // Shared channel diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js new file mode 100644 index 000000000..f6595326d --- /dev/null +++ b/packages/nylas-core/scheduler-utils.js @@ -0,0 +1,29 @@ +const ACCOUNTS_UNCLAIMED = 'accounts:unclaimed'; +const ACCOUNTS_CLAIMED_PREFIX = 'accounts:id-'; +const ACCOUNTS_FOR = (id) => `${ACCOUNTS_CLAIMED_PREFIX}${id}`; +const HEARTBEAT_FOR = (id) => `heartbeat:${id}`; +const HEARTBEAT_EXPIRES = 30; // 2 min in prod? +const CLAIM_DURATION = 10 * 60 * 1000; // 2 hours on prod? + +const PubsubConnector = require('./pubsub-connector') + +const forEachAccountList = (forEachCallback) => { + const client = PubsubConnector.broadcastClient(); + return Promise.each(client.keysAsync(`accounts:*`), (key) => { + const processId = key.replace('accounts:', ''); + return client.lrangeAsync(key, 0, 20000).then((foundIds) => + forEachCallback(processId, foundIds) + ) + }); +} + +module.exports = { + ACCOUNTS_UNCLAIMED, + ACCOUNTS_CLAIMED_PREFIX, + ACCOUNTS_FOR, + HEARTBEAT_FOR, + HEARTBEAT_EXPIRES, + CLAIM_DURATION, + + forEachAccountList, +} diff --git a/packages/nylas-core/sync-policy.js b/packages/nylas-core/sync-policy.js index 5d2d59e0d..5443f3d28 100644 --- a/packages/nylas-core/sync-policy.js +++ b/packages/nylas-core/sync-policy.js @@ -1,7 +1,7 @@ class SyncPolicy { static defaultPolicy() { return { - afterSync: 'close', + afterSync: 'idle', interval: 120 * 1000, folderSyncOptions: { deepFolderScan: 10 * 60 * 1000, diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js new file mode 100644 index 000000000..cb0b6c909 --- /dev/null +++ b/packages/nylas-dashboard/app.js @@ -0,0 +1,75 @@ +const Hapi = require('hapi'); +const HapiWebSocket = require('hapi-plugin-websocket'); +const Inert = require('inert'); +const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`); +const {forEachAccountList} = SchedulerUtils; + +global.Promise = require('bluebird'); + +const server = new Hapi.Server(); +server.connection({ port: process.env.PORT || 5101 }); + +DatabaseConnector.forShared().then(({Account}) => { + server.register([HapiWebSocket, Inert], () => { + server.route({ + method: "POST", + path: "/accounts", + config: { + plugins: { + websocket: { + only: true, + connect: (wss, ws) => { + Account.findAll().then((accts) => { + accts.forEach((acct) => { + ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct })); + }); + }); + this.redis = PubsubConnector.buildClient(); + this.redis.on('pmessage', (pattern, channel) => { + Account.find({where: {id: channel.replace('a-', '')}}).then((acct) => { + ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct })); + }); + }); + this.redis.psubscribe(PubsubConnector.channelForAccount('*')); + this.assignmentsInterval = setInterval(() => { + const assignments = {}; + forEachAccountList((identity, accountIds) => { + for (const accountId of accountIds) { + assignments[accountId] = identity; + } + }).then(() => + ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments})) + ) + }, 1000); + }, + disconnect: () => { + clearInterval(this.assignmentsInterval); + this.redis.quit(); + }, + }, + }, + }, + handler: (request, reply) => { + if (request.payload.cmd === "PING") { + reply(JSON.stringify({ result: "PONG" })); + return; + } + }, + }); + + server.route({ + method: 'GET', + path: '/{param*}', + handler: { + directory: { + path: 'public', + }, + }, + }); + + server.start((startErr) => { + if (startErr) { throw startErr; } + console.log('Server running at:', server.info.uri); + }); + }); +}); diff --git a/packages/nylas-dashboard/package.json b/packages/nylas-dashboard/package.json new file mode 100644 index 000000000..a05f0fc24 --- /dev/null +++ b/packages/nylas-dashboard/package.json @@ -0,0 +1,17 @@ +{ + "name": "nylas-dashboard", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "hapi": "^13.4.1", + "hapi-plugin-websocket": "^0.9.2", + "inert": "^4.0.0", + "nylas-core": "0.x.x" + } +} diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css new file mode 100644 index 000000000..056b94d01 --- /dev/null +++ b/packages/nylas-dashboard/public/css/app.css @@ -0,0 +1,17 @@ +body { + background-image: -webkit-linear-gradient(top, rgba(232, 244, 250, 0.6), rgba(231, 231, 233, 1)), url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg); + background-size: cover; + font-family: sans-serif; +} + +.account { + display:inline-block; + width: 300px; + height: 100px; + background-color: white; + padding:15px; +} + +.account h3 { + margin: 0; padding: 0; +} diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html new file mode 100644 index 000000000..dac87884b --- /dev/null +++ b/packages/nylas-dashboard/public/index.html @@ -0,0 +1,13 @@ + + + + + + + + + +

Dashboard

+
+ + diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx new file mode 100644 index 000000000..dfc9e28ba --- /dev/null +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -0,0 +1,91 @@ +/* eslint react/react-in-jsx-scope: 0*/ + +class Account extends React.Component { + propTypes: { + account: React.PropTypes.object, + assignment: React.PropTypes.string, + } + + render() { + const {account, assignment} = this.props; + return ( +
+

{account.email_address}

+ {assignment} +
Sync Interval: {account.sync_policy.interval}ms
+
Sync Idle Behavior: {account.sync_policy.afterSync}
+
+ ); + } +} + +class Root extends React.Component { + + constructor() { + super(); + this.state = { + accounts: {}, + assignments: {}, + }; + } + + componentDidMount() { + let url = null; + if (window.location.protocol === "https:") { + url = `wss://${window.location.host}/accounts`; + } else { + url = `ws://${window.location.host}/accounts`; + } + this.websocket = new WebSocket(url); + this.websocket.onopen = () => { + this.websocket.send("Message to send"); + }; + this.websocket.onmessage = (evt) => { + try { + const msg = JSON.parse(evt.data); + if (msg.cmd === 'ACCOUNT') { + this.onReceivedAccount(msg.payload); + } + if (msg.cmd === 'ASSIGNMENTS') { + this.onReceivedAssignments(msg.payload); + } + } catch (err) { + console.error(err); + } + }; + this.websocket.onclose = () => { + window.location.reload(); + }; + } + + onReceivedAssignments(assignments) { + this.setState({assignments}) + } + + onReceivedAccount(account) { + const accounts = Object.assign({}, this.state.accounts); + accounts[account.id] = account; + this.setState({accounts}); + } + + render() { + return ( +
+ { + Object.keys(this.state.accounts).sort((a, b) => a.compare(b)).map((key) => + + ) + } +
+ ) + } +} + +ReactDOM.render( + , + document.getElementById('root') +); diff --git a/packages/nylas-dashboard/public/js/react-dom.js b/packages/nylas-dashboard/public/js/react-dom.js new file mode 100644 index 000000000..1cf5496b5 --- /dev/null +++ b/packages/nylas-dashboard/public/js/react-dom.js @@ -0,0 +1,42 @@ +/** + * ReactDOM v15.1.0 + * + * Copyright 2013-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +// Based off https://github.com/ForbesLindesay/umd/blob/master/template.js +;(function(f) { + // CommonJS + if (typeof exports === "object" && typeof module !== "undefined") { + module.exports = f(require('react')); + + // RequireJS + } else if (typeof define === "function" && define.amd) { + define(['react'], f); + + // + + K2 Dashboard -

Dashboard

+

K2 Dashboard

diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 296d9f287..3598037a5 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -32,11 +32,19 @@ class Account extends React.Component { render() { const {account, assignment, active} = this.props; const errorClass = account.sync_error ? ' errored' : '' + const syncCompletions = [] + for (const time of account.lastSyncCompletions) { + syncCompletions.push( +
{new Date(time).toString()}
+ ) + } return (

{account.email_address} {active ? '🌕' : '🌑'}

{assignment} 
{JSON.stringify(account.sync_policy, null, 2)}
+
Last Sync Completions
+ 
{syncCompletions}
{this.renderError()}
); From a99ffbce3c25641458df25b68bb02011f2329158 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Thu, 30 Jun 2016 17:31:52 -0700 Subject: [PATCH 151/800] Add favicon :snow_capped_mountain: --- packages/nylas-dashboard/public/favicon.png | Bin 0 -> 2439 bytes packages/nylas-dashboard/public/index.html | 1 + 2 files changed, 1 insertion(+) create mode 100644 packages/nylas-dashboard/public/favicon.png diff --git a/packages/nylas-dashboard/public/favicon.png b/packages/nylas-dashboard/public/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..d595d6d01db04ba06c163cffce983309cfdffb85 GIT binary patch literal 2439 zcmY*bdpr|rA0KVl+RQB^vJHo*jiE9#m#|sxp^Zh1Wz2@naVg}UYs&rFMUiW{=Sbu- z(mCCn4z*I0;q(?IA*C)xm7?yE0~H{fs*l?woDSKKlQKzXIA7(iuuPmut#aN(zAaG)TD~v%Qen5Dk7AT67E8-9* zG6-RcHbEPr2oMAUfsYJi;M{HP|4tY0EKtEbUN{beiH?p&M;oI#ktZ-lW@cs>Lo5c1 z-6uxuf|vQ|5ajh z|87g%AZF_XV}v%u{KG9y#cxG%E|JV2vGNu_!3h5Y`G2v$b?}%i@&762uTFnN#jO$` zc+5Z7Mu0dD>QVs!=}`w;YYz%|#LI`Heqv{Wu*2AVx>Hg5K*Zzo9ZqTn5s%divo4db zD&&&Sy-KbigV^2?_goe|uV>}U6>gWCK`Dgaxzup_p?+l8zoIHv`58{ECBCm9bSWY) zewxFWeCuKJ97s4RtSGBmimzUag}+4P(P7`~kW;m_%n!{z-FZ%qjU(>~b+iv|M?M2P*wnyAAJ5XJ z7x%pu#7q8JUMEv)-{qk@%Tg#(C#43ugKR-&7as+atuNX!%us}*+UtAYbgE0d8&_-H zr<$S_VG)0MqEVih5xUFdj{@IY^QQ2PV|W#MpJ-vHB50WB2txU0i!R|PluJz$My$8v z_MF^bGr+Dwbw9OPB(q)#Q7%VPA~`Jk+N$OxK@LICK!xx!Q-w1gG9P))WAMBo;5t2T zy0=UhtgSy5;4~2Am6*9lqHgIE?a0RhBJQw4mSmgr(Jq}N=}|?X2~_?Tap1V{^Q{u< zX&q?fwL8?7IPf*b2Z9bl-&TqOa!>at?%E7JqMQNFuL^bOa+fID=AbY>>no$gBikA& z90GbG^Qi-|?wNW!tfYzqu^MuMU(lT0IYYpObEnOfnWO%H_U*kq85v&1r0O5I5>2Wn zNA7u$i3IBjL3re)bAUqw_=hIj^7QgQT_B`LNVeuXf`d)$uTRKnKZQ?~LA^@6TwhL) zym{h4$x9hnWBy(UMHs%VhV89I+MOnoFovG856udI=e-vd7wS@Dke52^t8xDry8p;Hg{uWdVD3#pV6+Q2Mrw%0^t>Shc^VTi5VzxcGsR zpBN0-N>rZ&zOl5=F;eFlsm|V6z67xke@*SwRJEJ-&~5l`P!j}r_+~WW!^?rns469 zTCD$2b1*_A|PiC0l+@mzk^ef#db#)2bbb@av=`Y$f|NqD@H zM9xF51YM2n9q=K#<>9;hX z`?Dp6dXPn9eK04+&J@V0BCBWLOH|0~vlon~8s$V!BLZ2{z9q-1s5%?S*6F4vuHN4K zke-|uU(i|b;zuE-5yFil>m$U(ffoVENNP+0B#!UCtZwxpZCzb@!E|K;YI!n18wc4K2K-RyO5(H_1z8Y)7>xc zJRp1n9O&QdYLJ~HtR9728Iy)AOQ%c|9jp05(w0(F zbEuGTc)7UY#p%@xQLKdaI6dX<4F^ccjt9bO%DBPht;ONrM5KE^<8_eNY!VUk}5irJu2}01ADKj4?bZ!qcBD%sSDgQ=c3FD z_anf&Dg+urn;hAq$5T7!BrmStZ>+WuXsXri$M~kP6_w34k|u!GFtq zD6g#k5x-|B)!I+OUl6qaWxYs!q|*-19UtM&pAB&Cg@twG4g_MqthupVYG9h3_B&v` z$;YN&lvfS9(+0>LBoJ;{>+B1~V15+uvIWajUnaK;h3F}UvgOq4C7)ZZr?7g5dP>5D zX+|BhL_L;^mL*Pd;{B^{?=Q4YQ$}1*Rr@*au=RpD@iz;~Hg6=4jXQrPfPZ&U2{i*Il4tz23l$MSwF-)ky!tA zkB@PGX|m40qQiJY%7kwdkdS_7=-2LfM!Vgq&+|MzNzwZmq5Lb+V)nyHLXNQ$@uZPv zE5E0Cf6pUtd33L3z+ioVo99e4ofcEI#?dYwHZYB=jcxpyKzNg3#QR*zsM+aPUKywZ z3TZ4^+aW=Iu+ML3wZ!ichZgVInf2z1<8It)1n`>l&kMQVV`}cMB}4BNQspT$S|zHn x;!`q#@YVXn>fGGio-`BHvvu%zRqGr5z)_w3ufpjqySIKv4tCDAEjGuK{|)}$Fa!Vq literal 0 HcmV?d00001 diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html index 076c209bd..30486bada 100644 --- a/packages/nylas-dashboard/public/index.html +++ b/packages/nylas-dashboard/public/index.html @@ -5,6 +5,7 @@ + K2 Dashboard From 80f9ff38e30462024345ec684091769e9c52db6a Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Tue, 28 Jun 2016 00:00:31 -0700 Subject: [PATCH 152/800] Remove dead code --- packages/nylas-dashboard/public/js/app.jsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 3598037a5..c09831c2a 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -2,12 +2,6 @@ const React = window.React; const ReactDOM = window.ReactDOM; -class ErrorsRoot extends React.Component { - render() { - return
- } -} - class Account extends React.Component { renderError() { const {account} = this.props; From 05091a4447f85c8279da016a02f0318642525a0d Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Fri, 1 Jul 2016 10:24:39 -0700 Subject: [PATCH 153/800] Update thrown errors to use serializable NylasError --- packages/nylas-core/imap-connection.js | 32 +++++++++---------- .../imap/fetch-messages-in-category.js | 2 +- packages/nylas-sync/sync-worker.js | 4 +-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index 0626e7d4e..c69ecb18f 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -36,10 +36,10 @@ class IMAPBox { if (range.length === 0) { return Rx.Observable.empty() } - if (!options) { - throw new Error("IMAPBox.fetch now requires an options object.") - } return Rx.Observable.create((observer) => { + if (!options) { + return observer.onError(new NylasError("IMAPBox.fetch now requires an options object.")) + } const f = this._imap.fetch(range, options); f.on('message', (imapMessage) => { const parts = {}; @@ -75,10 +75,10 @@ class IMAPBox { fetchStream({uid, options}) { if (!uid) { - throw new Error("IMAPConnection.fetchStream requires a message uid.") + throw new NylasError("IMAPConnection.fetchStream requires a message uid.") } if (!options) { - throw new Error("IMAPConnection.fetchStream requires an options object.") + throw new NylasError("IMAPConnection.fetchStream requires an options object.") } return new Promise((resolve, reject) => { const f = this._imap.fetch(uid, options); @@ -123,28 +123,28 @@ class IMAPBox { addFlags(range, flags) { if (!this._imap) { - throw new Error(`IMAPBox::addFlags - You need to call connect() first.`) + throw new NylasError(`IMAPBox::addFlags - You need to call connect() first.`) } return this._imap.addFlagsAsync(range, flags) } delFlags(range, flags) { if (!this._imap) { - throw new Error(`IMAPBox::delFlags - You need to call connect() first.`) + throw new NylasError(`IMAPBox::delFlags - You need to call connect() first.`) } return this._imap.delFlagsAsync(range, flags) } moveFromBox(range, folderName) { if (!this._imap) { - throw new Error(`IMAPBox::moveFromBox - You need to call connect() first.`) + throw new NylasError(`IMAPBox::moveFromBox - You need to call connect() first.`) } return this._imap.moveAsync(range, folderName) } closeBox({expunge = true} = {}) { if (!this._imap) { - throw new Error(`IMAPBox::closeBox - You need to call connect() first.`) + throw new NylasError(`IMAPBox::closeBox - You need to call connect() first.`) } return this._imap.closeBoxAsync(expunge) } @@ -259,7 +259,7 @@ class IMAPConnection extends EventEmitter { serverSupports(capability) { if (!this._imap) { - throw new Error(`IMAPConnection::serverSupports - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::serverSupports - You need to call connect() first.`) } this._imap.serverSupports(capability); } @@ -269,7 +269,7 @@ class IMAPConnection extends EventEmitter { */ openBox(folderName, {readOnly = false} = {}) { if (!this._imap) { - throw new Error(`IMAPConnection::openBox - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::openBox - You need to call connect() first.`) } return this._imap.openBoxAsync(folderName, readOnly).then((box) => new IMAPBox(this._imap, box) @@ -278,35 +278,35 @@ class IMAPConnection extends EventEmitter { getBoxes() { if (!this._imap) { - throw new Error(`IMAPConnection::getBoxes - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::getBoxes - You need to call connect() first.`) } return this._imap.getBoxesAsync() } addBox(folderName) { if (!this._imap) { - throw new Error(`IMAPConnection::addBox - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::addBox - You need to call connect() first.`) } return this._imap.addBoxAsync(folderName) } renameBox(oldFolderName, newFolderName) { if (!this._imap) { - throw new Error(`IMAPConnection::renameBox - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::renameBox - You need to call connect() first.`) } return this._imap.renameBoxAsync(oldFolderName, newFolderName) } delBox(folderName) { if (!this._imap) { - throw new Error(`IMAPConnection::delBox - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::delBox - You need to call connect() first.`) } return this._imap.delBoxAsync(folderName) } runOperation(operation) { if (!this._imap) { - throw new Error(`IMAPConnection::runOperation - You need to call connect() first.`) + throw new NylasError(`IMAPConnection::runOperation - You need to call connect() first.`) } return new Promise((resolve, reject) => { this._queue.push({operation, resolve, reject}); diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-category.js index cf70dcef0..4b79042cf 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-category.js +++ b/packages/nylas-sync/imap/fetch-messages-in-category.js @@ -329,7 +329,7 @@ class FetchMessagesInFolder { _runScan() { const {fetchedmin, fetchedmax} = this._category.syncState; if (!fetchedmin || !fetchedmax) { - throw new Error("Unseen messages must be fetched at least once before the first update/delete scan.") + throw new NylasError("Unseen messages must be fetched at least once before the first update/delete scan.") } return this._shouldRunDeepScan() ? this._runDeepScan() : this._runShallowScan() } diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 915c10e25..d893ce764 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -47,7 +47,7 @@ class SyncWorker { case MessageTypes.SYNCBACK_REQUESTED: this.syncNow(); break; default: - throw new Error(`Invalid message: ${msg}`) + throw new NylasError(`Invalid message: ${msg}`) } } @@ -197,7 +197,7 @@ class SyncWorker { return Promise.resolve() } - throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) + throw new NylasError(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) } scheduleNextSync() { From fd0c4d734c3f00e2c8297b1cf2c17a33117084bf Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Fri, 1 Jul 2016 11:27:20 -0700 Subject: [PATCH 154/800] Fix delta sync to properly includes sub models --- packages/nylas-api/routes/delta.js | 10 ++++++++-- packages/nylas-core/models/account/thread.js | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js index d3648359f..401568a9a 100644 --- a/packages/nylas-api/routes/delta.js +++ b/packages/nylas-api/routes/delta.js @@ -14,8 +14,14 @@ function inflateTransactions(db, transactionModels = []) { return Promise.all(Object.keys(byModel).map((modelName) => { const ids = _.pluck(byModel[modelName], "objectId"); - const ModelKlass = db[modelName] - return ModelKlass.findAll({id: ids}).then((models = []) => { + const modelConstructorName = modelName.charAt(0).toUpperCase() + modelName.slice(1); + const ModelKlass = db[modelConstructorName] + let includes = []; + if (ModelKlass.requiredAssociationsForJSON) { + includes = ModelKlass.requiredAssociationsForJSON() + } + return ModelKlass.findAll({where: {id: ids}, include: includes}) + .then((models = []) => { for (const model of models) { const tsForId = byObjectIds[model.id]; if (!tsForId || tsForId.length === 0) { continue; } diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js index 6e9be1cca..2181fae68 100644 --- a/packages/nylas-core/models/account/thread.js +++ b/packages/nylas-core/models/account/thread.js @@ -20,6 +20,16 @@ module.exports = (sequelize, Sequelize) => { { fields: ['threadId'] }, ], classMethods: { + requiredAssociationsForJSON: () => { + return [ + {model: sequelize.models.folder}, + {model: sequelize.models.label}, + { + model: sequelize.models.message, + attributes: ['id'], + }, + ] + }, associate: ({Folder, Label, Message}) => { Thread.belongsToMany(Folder, {through: 'thread_folders'}) Thread.belongsToMany(Label, {through: 'thread_labels'}) From 3ccb7f164b1086aa223a721e698d3027cedf53b8 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 1 Jul 2016 11:36:02 -0700 Subject: [PATCH 155/800] Fix closing connection twice --- packages/nylas-sync/sync-worker.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index d893ce764..ff23e4077 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -35,7 +35,9 @@ class SyncWorker { } closeConnection() { - this._conn.end(); + if (this._conn) { + this._conn.end(); + } this._conn = null } From 0af1a4379449960389bae291567a40453141d100 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Fri, 1 Jul 2016 13:15:49 -0700 Subject: [PATCH 156/800] Add first sync completion time --- packages/nylas-api/serialization.js | 2 ++ packages/nylas-core/models/shared/account.js | 4 +++- packages/nylas-dashboard/public/css/app.css | 11 ++++++++++- packages/nylas-dashboard/public/js/app.jsx | 18 ++++++++++++------ packages/nylas-sync/sync-worker.js | 4 ++++ 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 68c77ddc8..77175001b 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -20,6 +20,8 @@ function jsonSchema(modelName) { connection_settings: Joi.object(), sync_policy: Joi.object(), sync_error: Joi.object().allow(null), + first_sync_completed_at: Joi.number().allow(null), + last_sync_completions: Joi.array(), }) } if (modelName === 'Folder') { diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index be0014d7d..eb61379cf 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -12,6 +12,7 @@ module.exports = (sequelize, Sequelize) => { connectionCredentials: Sequelize.STRING, syncPolicy: JSONType('syncPolicy'), syncError: JSONType('syncError', {defaultValue: null}), + firstSyncCompletedAt: Sequelize.INTEGER, lastSyncCompletions: JSONARRAYType('lastSyncCompletions'), }, { classMethods: { @@ -30,7 +31,8 @@ module.exports = (sequelize, Sequelize) => { connection_settings: this.connectionSettings, sync_policy: this.syncPolicy, sync_error: this.syncError, - lastSyncCompletions: this.lastSyncCompletions, + first_sync_completed_at: this.firstSyncCompletedAt, + last_sync_completions: this.lastSyncCompletions, } }, diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index 7a6ea18f7..7b3aefae4 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -2,6 +2,7 @@ body { background-image: -webkit-linear-gradient(top, rgba(232, 244, 250, 0.2), rgba(231, 231, 233, 1)), url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg); background-size: cover; font-family: Roboto, sans-serif; + font-size: 12px; } h2 { @@ -9,6 +10,10 @@ h2 { text-align: center; } +pre { + margin: 0; +} + .account { display: inline-block; border-radius: 5px; @@ -20,10 +25,14 @@ h2 { } .account h3 { - margin: 0; padding: 0; + font-size: 16px; + margin: 0; + padding: 0; } .account .section { + font-size: 14px; + padding: 10px 0; text-align: center; } diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index c09831c2a..1cd49bc4b 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -13,7 +13,6 @@ class Account extends React.Component { } return (
- Sync Errored: 
             {JSON.stringify(error, null, 2)}
@@ -26,9 +25,9 @@ class Account extends React.Component { render() { const {account, assignment, active} = this.props; const errorClass = account.sync_error ? ' errored' : '' - const syncCompletions = [] - for (const time of account.lastSyncCompletions) { - syncCompletions.push( + const lastSyncCompletions = [] + for (const time of account.last_sync_completions) { + lastSyncCompletions.push(
{new Date(time).toString()}
) } @@ -36,9 +35,16 @@ class Account extends React.Component {

{account.email_address} {active ? '🌕' : '🌑'}

{assignment} +
Sync Policy
{JSON.stringify(account.sync_policy, null, 2)}
-
Last Sync Completions
- 
{syncCompletions}
+
Sync Cycles
+
+ First Sync Completion: + 
{new Date(account.first_sync_completed_at).toString()}
+
+
Last Sync Completions:
+ 
{lastSyncCompletions}
+
Error
{this.renderError()}
); diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index ff23e4077..3a0e3bba5 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -178,6 +178,10 @@ class SyncWorker { onSyncDidComplete() { const {afterSync} = this._account.syncPolicy; + if (!this._account.firstSyncCompletedAt) { + this._account.firstSyncCompletedAt = Date.now() + } + let lastSyncCompletions = [...this._account.lastSyncCompletions] lastSyncCompletions = [Date.now(), ...lastSyncCompletions] if (lastSyncCompletions.length > 10) { From 81145eae32c422e7fbd149939281c80437c25230 Mon Sep 17 00:00:00 2001 From: Jackie Luo Date: Fri, 1 Jul 2016 13:24:08 -0700 Subject: [PATCH 157/800] Change error styling --- packages/nylas-dashboard/public/css/app.css | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index 7b3aefae4..335e92347 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -37,13 +37,9 @@ pre { } .account.errored { - background-image: linear-gradient(to bottom,#f2dede 0,#e7c3c3 100%); - background-repeat: repeat-x; - border-color: #dca7a7; - border: 1px solid; color: #a94442; border-radius: 4px; - background-color: #f2dede; + background-color: rgba(231, 195, 195, 0.6); } .account .error pre { From 8453985b1fdb98f276223ebcba084d656ca95787 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 1 Jul 2016 15:41:22 -0700 Subject: [PATCH 158/800] Change worker removal so it waits for sync to complete --- .../imap/fetch-messages-in-category.js | 10 +++++-- packages/nylas-sync/sync-process-manager.js | 30 ++++++++++--------- packages/nylas-sync/sync-worker.js | 13 +++++++- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-category.js index 4b79042cf..a44ecc3c3 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-category.js +++ b/packages/nylas-sync/imap/fetch-messages-in-category.js @@ -6,6 +6,10 @@ const {Capabilities} = IMAPConnection; const MessageFlagAttributes = ['id', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels'] +const SHALLOW_SCAN_UID_COUNT = 1000; +const FETCH_MESSAGES_FIRST_COUNT = 100; +const FETCH_MESSAGES_COUNT = 200; + class FetchMessagesInFolder { constructor(category, options) { this._imap = null @@ -293,7 +297,7 @@ class FetchMessagesInFolder { // sync based on number of messages / age of messages. if (isFirstSync) { - const lowerbound = Math.max(1, boxUidnext - 150); + const lowerbound = Math.max(1, boxUidnext - FETCH_MESSAGES_FIRST_COUNT); desiredRanges.push({min: lowerbound, max: boxUidnext}) } else { if (savedSyncState.fetchedmax < boxUidnext) { @@ -302,7 +306,7 @@ class FetchMessagesInFolder { console.log(" --- fetchedmax == uidnext, nothing more recent to fetch.") } if (savedSyncState.fetchedmin > 1) { - const lowerbound = Math.max(1, savedSyncState.fetchedmin - 1000); + const lowerbound = Math.max(1, savedSyncState.fetchedmin - FETCH_MESSAGES_COUNT); desiredRanges.push({min: lowerbound, max: savedSyncState.fetchedmin}) } else { console.log(" --- fetchedmin == 1, nothing older to fetch.") @@ -353,7 +357,7 @@ class FetchMessagesInFolder { } shallowFetch = this._box.fetchUIDAttributes(`1:*`, {changedsince: highestmodseq}); } else { - const range = `${this._getLowerBoundUID(1000)}:*`; + const range = `${this._getLowerBoundUID(SHALLOW_SCAN_UID_COUNT)}:*`; console.log(` - Shallow attribute scan (using range: ${range})`) shallowFetch = this._box.fetchUIDAttributes(range); } diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index c084c9e8a..513bd1664 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -10,7 +10,6 @@ const { ACCOUNTS_CLAIMED_PREFIX, HEARTBEAT_FOR, HEARTBEAT_EXPIRES, - CLAIM_DURATION, forEachAccountList, } = SchedulerUtils; @@ -150,9 +149,14 @@ class SyncProcessManager { const dst = ACCOUNTS_FOR(IDENTITY); return this._waitForAccountClient.brpoplpushAsync(src, dst, 10000).then((accountId) => { - if (!accountId) { return } + if (!accountId) { + return Promise.resolve(); + } this.addWorkerForAccountId(accountId); - setTimeout(() => this.removeWorker(), CLAIM_DURATION); + + // If we've added an account, wait a second before asking for another one. + // Spacing them out is probably healthy. + return Promise.delay(2000); }); } @@ -167,25 +171,23 @@ class SyncProcessManager { return; } console.log(`ProcessManager: Starting worker for Account ${accountId}`) - this._workers[account.id] = new SyncWorker(account, db); + this._workers[account.id] = new SyncWorker(account, db, () => { + this.removeWorkerForAccountId(accountId) + }); }); }); }); } - removeWorker() { + removeWorkerForAccountId(accountId) { const src = ACCOUNTS_FOR(IDENTITY); const dst = ACCOUNTS_UNCLAIMED; - return PubsubConnector.broadcastClient().rpoplpushAsync(src, dst).then((accountId) => { - if (!accountId) { - return; - } - - console.log(`ProcessManager: Returning account ${accountId} to unclaimed pool.`) - - if (this._workers[accountId]) { - this._workers[accountId].cleanup(); + return PubsubConnector.broadcastClient().lremAsync(src, 1, accountId).then((didRemove) => { + if (didRemove) { + PubsubConnector.broadcastClient().rpushAsync(dst, accountId) + } else { + throw new Error("Wanted to return item to pool, but didn't have claim on it.") } this._workers[accountId] = null; }); diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index ff23e4077..bf0fca5ef 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -6,17 +6,21 @@ const { MessageTypes, } = require('nylas-core'); +const {CLAIM_DURATION} = SchedulerUtils; + const FetchFolderList = require('./imap/fetch-category-list') const FetchMessagesInFolder = require('./imap/fetch-messages-in-category') const SyncbackTaskFactory = require('./syncback-task-factory') class SyncWorker { - constructor(account, db) { + constructor(account, db, onExpired) { this._db = db; this._conn = null; this._account = account; + this._startTime = Date.now(); this._lastSyncTime = null; + this._onExpired = onExpired; this._syncTimer = null; this._expirationTimer = null; @@ -203,6 +207,13 @@ class SyncWorker { } scheduleNextSync() { + if (Date.now() - this._startTime > CLAIM_DURATION) { + console.log("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool."); + this.cleanup(); + this._onExpired(); + return; + } + SchedulerUtils.checkIfAccountIsActive(this._account.id).then((active) => { const {intervals} = this._account.syncPolicy; const interval = active ? intervals.active : intervals.inactive; From 8b1f012a3c8ad76de98b6db7432c1d7546c1bd3f Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Fri, 1 Jul 2016 15:46:15 -0700 Subject: [PATCH 159/800] Fix error serialization - Remove cumbersome NylasError - Add helper to serialize error when saving to account --- packages/nylas-api/app.js | 1 - packages/nylas-core/database-connector.js | 2 +- packages/nylas-core/imap-connection.js | 38 +++++++++---------- packages/nylas-core/index.js | 2 - packages/nylas-core/models/account/file.js | 3 +- packages/nylas-core/models/account/message.js | 3 +- packages/nylas-core/models/shared/account.js | 2 +- packages/nylas-core/nylas-error.js | 18 --------- packages/nylas-dashboard/app.js | 3 +- packages/nylas-dashboard/public/js/app.jsx | 5 ++- packages/nylas-message-processor/app.js | 3 +- packages/nylas-sync/app.js | 3 +- .../imap/fetch-messages-in-category.js | 6 +-- packages/nylas-sync/sync-utils.js | 11 ++++++ packages/nylas-sync/sync-worker.js | 13 ++++--- 15 files changed, 51 insertions(+), 62 deletions(-) delete mode 100644 packages/nylas-core/nylas-error.js create mode 100644 packages/nylas-sync/sync-utils.js diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index 8c21774c1..8bcb3359f 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -9,7 +9,6 @@ const fs = require('fs'); const path = require('path'); global.Promise = require('bluebird'); -global.NylasError = require('nylas-core').NylasError; const server = new Hapi.Server({ connections: { diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index 4c0e7e6ea..5d95b16e1 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -36,7 +36,7 @@ class DatabaseConnector { _sequelizeForAccount(accountId) { if (!accountId) { - return Promise.reject(new NylasError(`You need to pass an accountId to init the database!`)) + return Promise.reject(new Error(`You need to pass an accountId to init the database!`)) } const sequelize = new Sequelize(accountId, '', '', { storage: path.join(STORAGE_DIR, `a-${accountId}.sqlite`), diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index c69ecb18f..cda68a70a 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -18,7 +18,7 @@ class IMAPBox { } if (_.isFunction(prop) && target._imap._box.name !== target._box.name) { return () => Promise.reject( - new NylasError(`IMAPBox::${name} - Can't operate on a mailbox that is no longer open on the current IMAPConnection.`) + new Error(`IMAPBox::${name} - Can't operate on a mailbox that is no longer open on the current IMAPConnection.`) ) } return prop @@ -33,13 +33,13 @@ class IMAPBox { * @return {Observable} that will feed each message as it becomes ready */ fetch(range, options) { + if (!options) { + throw new Error("IMAPBox.fetch now requires an options object.") + } if (range.length === 0) { return Rx.Observable.empty() } return Rx.Observable.create((observer) => { - if (!options) { - return observer.onError(new NylasError("IMAPBox.fetch now requires an options object.")) - } const f = this._imap.fetch(range, options); f.on('message', (imapMessage) => { const parts = {}; @@ -75,10 +75,10 @@ class IMAPBox { fetchStream({uid, options}) { if (!uid) { - throw new NylasError("IMAPConnection.fetchStream requires a message uid.") + throw new Error("IMAPConnection.fetchStream requires a message uid.") } if (!options) { - throw new NylasError("IMAPConnection.fetchStream requires an options object.") + throw new Error("IMAPConnection.fetchStream requires an options object.") } return new Promise((resolve, reject) => { const f = this._imap.fetch(uid, options); @@ -123,28 +123,28 @@ class IMAPBox { addFlags(range, flags) { if (!this._imap) { - throw new NylasError(`IMAPBox::addFlags - You need to call connect() first.`) + throw new Error(`IMAPBox::addFlags - You need to call connect() first.`) } return this._imap.addFlagsAsync(range, flags) } delFlags(range, flags) { if (!this._imap) { - throw new NylasError(`IMAPBox::delFlags - You need to call connect() first.`) + throw new Error(`IMAPBox::delFlags - You need to call connect() first.`) } return this._imap.delFlagsAsync(range, flags) } moveFromBox(range, folderName) { if (!this._imap) { - throw new NylasError(`IMAPBox::moveFromBox - You need to call connect() first.`) + throw new Error(`IMAPBox::moveFromBox - You need to call connect() first.`) } return this._imap.moveAsync(range, folderName) } closeBox({expunge = true} = {}) { if (!this._imap) { - throw new NylasError(`IMAPBox::closeBox - You need to call connect() first.`) + throw new Error(`IMAPBox::closeBox - You need to call connect() first.`) } return this._imap.closeBoxAsync(expunge) } @@ -197,7 +197,7 @@ class IMAPConnection extends EventEmitter { if (this._settings.refresh_token) { const xoauthFields = ['client_id', 'client_secret', 'imap_username', 'refresh_token']; if (Object.keys(_.pick(this._settings, xoauthFields)).length !== 4) { - return Promise.reject(new NylasError(`IMAPConnection: Expected ${xoauthFields.join(',')} when given refresh_token`)) + return Promise.reject(new Error(`IMAPConnection: Expected ${xoauthFields.join(',')} when given refresh_token`)) } return new Promise((resolve, reject) => { xoauth2.createXOAuth2Generator({ @@ -259,7 +259,7 @@ class IMAPConnection extends EventEmitter { serverSupports(capability) { if (!this._imap) { - throw new NylasError(`IMAPConnection::serverSupports - You need to call connect() first.`) + throw new Error(`IMAPConnection::serverSupports - You need to call connect() first.`) } this._imap.serverSupports(capability); } @@ -269,7 +269,7 @@ class IMAPConnection extends EventEmitter { */ openBox(folderName, {readOnly = false} = {}) { if (!this._imap) { - throw new NylasError(`IMAPConnection::openBox - You need to call connect() first.`) + throw new Error(`IMAPConnection::openBox - You need to call connect() first.`) } return this._imap.openBoxAsync(folderName, readOnly).then((box) => new IMAPBox(this._imap, box) @@ -278,35 +278,35 @@ class IMAPConnection extends EventEmitter { getBoxes() { if (!this._imap) { - throw new NylasError(`IMAPConnection::getBoxes - You need to call connect() first.`) + throw new Error(`IMAPConnection::getBoxes - You need to call connect() first.`) } return this._imap.getBoxesAsync() } addBox(folderName) { if (!this._imap) { - throw new NylasError(`IMAPConnection::addBox - You need to call connect() first.`) + throw new Error(`IMAPConnection::addBox - You need to call connect() first.`) } return this._imap.addBoxAsync(folderName) } renameBox(oldFolderName, newFolderName) { if (!this._imap) { - throw new NylasError(`IMAPConnection::renameBox - You need to call connect() first.`) + throw new Error(`IMAPConnection::renameBox - You need to call connect() first.`) } return this._imap.renameBoxAsync(oldFolderName, newFolderName) } delBox(folderName) { if (!this._imap) { - throw new NylasError(`IMAPConnection::delBox - You need to call connect() first.`) + throw new Error(`IMAPConnection::delBox - You need to call connect() first.`) } return this._imap.delBoxAsync(folderName) } runOperation(operation) { if (!this._imap) { - throw new NylasError(`IMAPConnection::runOperation - You need to call connect() first.`) + throw new Error(`IMAPConnection::runOperation - You need to call connect() first.`) } return new Promise((resolve, reject) => { this._queue.push({operation, resolve, reject}); @@ -327,7 +327,7 @@ class IMAPConnection extends EventEmitter { const {operation, resolve, reject} = this._currentOperation; const result = operation.run(this._db, this); if (result instanceof Promise === false) { - reject(new NylasError(`Expected ${operation.constructor.name} to return promise.`)) + reject(new Error(`Expected ${operation.constructor.name} to return promise.`)) } result .then(() => { diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index b8bb708f2..d8ac0b59a 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -1,5 +1,4 @@ global.Promise = require('bluebird'); -global.NylasError = require('./nylas-error'); module.exports = { Provider: { @@ -12,5 +11,4 @@ module.exports = { SyncPolicy: require('./sync-policy'), SchedulerUtils: require('./scheduler-utils'), MessageTypes: require('./message-types'), - NylasError, } diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js index a73ed0844..8e121ca7c 100644 --- a/packages/nylas-core/models/account/file.js +++ b/packages/nylas-core/models/account/file.js @@ -1,5 +1,4 @@ const IMAPConnection = require('../../imap-connection') -const NylasError = require('../../nylas-error') module.exports = (sequelize, Sequelize) => { const File = sequelize.define('file', { @@ -36,7 +35,7 @@ module.exports = (sequelize, Sequelize) => { if (stream) { return Promise.resolve(stream) } - return Promise.reject(new NylasError(`Unable to fetch binary data for File ${this.id}`)) + return Promise.reject(new Error(`Unable to fetch binary data for File ${this.id}`)) }) .finally(() => connection.end()) }) diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js index da092fa9a..0f39e7673 100644 --- a/packages/nylas-core/models/account/message.js +++ b/packages/nylas-core/models/account/message.js @@ -1,6 +1,5 @@ const crypto = require('crypto'); const IMAPConnection = require('../../imap-connection') -const NylasError = require('../../nylas-error') const {JSONType, JSONARRAYType} = require('../../database-types'); @@ -83,7 +82,7 @@ module.exports = (sequelize, Sequelize) => { if (message) { return Promise.resolve(`${message.headers}${message.body}`) } - return Promise.reject(new NylasError(`Unable to fetch raw message for Message ${this.id}`)) + return Promise.reject(new Error(`Unable to fetch raw message for Message ${this.id}`)) }) .finally(() => connection.end()) }) diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index eb61379cf..83039d542 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -42,7 +42,7 @@ module.exports = (sequelize, Sequelize) => { setCredentials: function setCredentials(json) { if (!(json instanceof Object)) { - throw new NylasError("Call setCredentials with JSON!") + throw new Error("Call setCredentials with JSON!") } const cipher = crypto.createCipher(DB_ENCRYPTION_ALGORITHM, DB_ENCRYPTION_PASSWORD) let crypted = cipher.update(JSON.stringify(json), 'utf8', 'hex') diff --git a/packages/nylas-core/nylas-error.js b/packages/nylas-core/nylas-error.js deleted file mode 100644 index 7b5d4d7cd..000000000 --- a/packages/nylas-core/nylas-error.js +++ /dev/null @@ -1,18 +0,0 @@ -class NylasError extends Error { - constructor(message) { - super(message); - this.name = this.constructor.name; - this.message = message; - Error.captureStackTrace(this, this.constructor); - } - - toJSON() { - const obj = {} - Object.getOwnPropertyNames(this).forEach((key) => { - obj[key] = this[key]; - }); - return obj - } -} - -module.exports = NylasError diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js index 801ecc72e..98247ae16 100644 --- a/packages/nylas-dashboard/app.js +++ b/packages/nylas-dashboard/app.js @@ -1,10 +1,9 @@ const Hapi = require('hapi'); const HapiWebSocket = require('hapi-plugin-websocket'); const Inert = require('inert'); -const {DatabaseConnector, PubsubConnector, SchedulerUtils, NylasError} = require(`nylas-core`); +const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`); global.Promise = require('bluebird'); -global.NylasError = NylasError; const server = new Hapi.Server(); server.connection({ port: process.env.PORT / 1 + 1 || 5101 }); diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 1cd49bc4b..ee753bfd6 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -7,9 +7,10 @@ class Account extends React.Component { const {account} = this.props; if (account.sync_error != null) { + const {message, stack} = account.sync_error const error = { - message: account.sync_error.message, - stack: account.sync_error.stack ? account.sync_error.stack.split('\n').slice(0, 4) : [], + message, + stack: stack.slice(0, 4), } return (
diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js index fd33379b4..458a074e3 100644 --- a/packages/nylas-message-processor/app.js +++ b/packages/nylas-message-processor/app.js @@ -1,8 +1,7 @@ -const {PubsubConnector, DatabaseConnector, NylasError} = require(`nylas-core`) +const {PubsubConnector, DatabaseConnector} = require(`nylas-core`) const {processors} = require('./processors') global.Promise = require('bluebird'); -global.NylasError = NylasError; // List of the attributes of Message that the processor should be allowed to change. // The message may move between folders, get starred, etc. while it's being diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 5a86cd830..5d7319245 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -1,6 +1,6 @@ global.Promise = require('bluebird'); -const {DatabaseConnector, NylasError} = require(`nylas-core`) +const {DatabaseConnector} = require(`nylas-core`) const SyncProcessManager = require('./sync-process-manager'); const manager = new SyncProcessManager(); @@ -18,5 +18,4 @@ DatabaseConnector.forShared().then((db) => { }); }); -global.NylasError = NylasError; global.manager = manager; diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-category.js index a44ecc3c3..554f09e71 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-category.js +++ b/packages/nylas-sync/imap/fetch-messages-in-category.js @@ -18,7 +18,7 @@ class FetchMessagesInFolder { this._category = category; this._options = options; if (!this._category) { - throw new NylasError("FetchMessagesInFolder requires a category") + throw new Error("FetchMessagesInFolder requires a category") } } @@ -273,7 +273,7 @@ class FetchMessagesInFolder { return this._imap.openBox(this._category.name) .then((box) => { if (box.persistentUIDs === false) { - return Promise.reject(new NylasError("Mailbox does not support persistentUIDs.")) + return Promise.reject(new Error("Mailbox does not support persistentUIDs.")) } if (box.uidvalidity !== this._category.syncState.uidvalidity) { return this._recoverFromUIDInvalidity() @@ -333,7 +333,7 @@ class FetchMessagesInFolder { _runScan() { const {fetchedmin, fetchedmax} = this._category.syncState; if (!fetchedmin || !fetchedmax) { - throw new NylasError("Unseen messages must be fetched at least once before the first update/delete scan.") + throw new Error("Unseen messages must be fetched at least once before the first update/delete scan.") } return this._shouldRunDeepScan() ? this._runDeepScan() : this._runShallowScan() } diff --git a/packages/nylas-sync/sync-utils.js b/packages/nylas-sync/sync-utils.js new file mode 100644 index 000000000..82d5b6cdf --- /dev/null +++ b/packages/nylas-sync/sync-utils.js @@ -0,0 +1,11 @@ + +function jsonError(error) { + return { + message: error.message, + stack: error.stack ? error.stack.split('\n') : [], + } +} + +module.exports = { + jsonError, +} diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 9cae14fe8..41815610f 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -5,6 +5,9 @@ const { DatabaseConnector, MessageTypes, } = require('nylas-core'); +const { + jsonError, +} = require('./sync-utils') const {CLAIM_DURATION} = SchedulerUtils; @@ -53,7 +56,7 @@ class SyncWorker { case MessageTypes.SYNCBACK_REQUESTED: this.syncNow(); break; default: - throw new NylasError(`Invalid message: ${msg}`) + throw new Error(`Invalid message: ${msg}`) } } @@ -87,10 +90,10 @@ class SyncWorker { const credentials = this._account.decryptedCredentials(); if (!settings || !settings.imap_host) { - return Promise.reject(new NylasError("ensureConnection: There are no IMAP connection settings for this account.")) + return Promise.reject(new Error("ensureConnection: There are no IMAP connection settings for this account.")) } if (!credentials) { - return Promise.reject(new NylasError("ensureConnection: There are no IMAP connection credentials for this account.")) + return Promise.reject(new Error("ensureConnection: There are no IMAP connection credentials for this account.")) } const conn = new IMAPConnection(this._db, Object.assign({}, settings, credentials)); @@ -175,7 +178,7 @@ class SyncWorker { // Continue to retry if it was a network error return Promise.resolve() } - this._account.syncError = error + this._account.syncError = jsonError(error) return this._account.save() } @@ -207,7 +210,7 @@ class SyncWorker { return Promise.resolve() } - throw new NylasError(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) + throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) } scheduleNextSync() { From ed2e184728ffe48541d16cca721ef4d23b361fd1 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 5 Jul 2016 13:21:27 -0700 Subject: [PATCH 160/800] Add Elastic Beanstalk to gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index e4a337dbf..c5eadbbb1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ node_modules dump.rdb *npm-debug.log storage/ + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml From e2e827297cfa1656a1b805364520196f08c73219 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 5 Jul 2016 14:09:44 -0700 Subject: [PATCH 161/800] Add authless /ping endpoint that returns "pong" --- packages/nylas-api/routes/ping.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 packages/nylas-api/routes/ping.js diff --git a/packages/nylas-api/routes/ping.js b/packages/nylas-api/routes/ping.js new file mode 100644 index 000000000..3e1d10f09 --- /dev/null +++ b/packages/nylas-api/routes/ping.js @@ -0,0 +1,12 @@ +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/ping', + config: { + auth: false, + }, + handler: (request, reply) => { + reply("pong") + }, + }); +}; From 9c342c0c7756f6d1a36cac78eda5cd3215cd5977 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Tue, 5 Jul 2016 15:41:56 -0700 Subject: [PATCH 162/800] Add ability to edit sync policy from dashboard --- packages/nylas-core/scheduler-utils.js | 2 +- packages/nylas-dashboard/app.js | 15 +++++ packages/nylas-dashboard/public/css/app.css | 13 ++++ packages/nylas-dashboard/public/index.html | 1 + packages/nylas-dashboard/public/js/app.jsx | 7 +- .../nylas-dashboard/public/js/sync-policy.jsx | 66 +++++++++++++++++++ .../nylas-dashboard/routes/sync-policy.js | 30 +++++++++ 7 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 packages/nylas-dashboard/public/js/sync-policy.jsx create mode 100644 packages/nylas-dashboard/routes/sync-policy.js diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js index da89aecf5..e6a4fa824 100644 --- a/packages/nylas-core/scheduler-utils.js +++ b/packages/nylas-core/scheduler-utils.js @@ -24,7 +24,7 @@ const forEachAccountList = (forEachCallback) => { const assignPolicy = (accountId, policy) => { console.log(`Changing policy for ${accountId} to ${JSON.stringify(policy)}`) const DatabaseConnector = require('./database-connector'); - DatabaseConnector.forShared().then(({Account}) => { + return DatabaseConnector.forShared().then(({Account}) => { Account.find({where: {id: accountId}}).then((account) => { account.syncPolicy = policy; account.save() diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js index 98247ae16..d0912330c 100644 --- a/packages/nylas-dashboard/app.js +++ b/packages/nylas-dashboard/app.js @@ -2,14 +2,29 @@ const Hapi = require('hapi'); const HapiWebSocket = require('hapi-plugin-websocket'); const Inert = require('inert'); const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`); +const fs = require('fs'); +const path = require('path'); global.Promise = require('bluebird'); const server = new Hapi.Server(); server.connection({ port: process.env.PORT / 1 + 1 || 5101 }); + +const attach = (directory) => { + const routesDir = path.join(__dirname, directory) + fs.readdirSync(routesDir).forEach((filename) => { + if (filename.endsWith('.js')) { + const routeFactory = require(path.join(routesDir, filename)); + routeFactory(server); + } + }); +} + DatabaseConnector.forShared().then(({Account}) => { server.register([HapiWebSocket, Inert], () => { + attach('./routes/') + server.route({ method: "POST", path: "/accounts", diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index 335e92347..78c729cf5 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -47,3 +47,16 @@ pre { width: inherit; overflow: hidden; } + +.sync-policy .action-link { + display: inline-block; + margin: 5px; + color: rgba(16, 83, 161, 0.88); + text-decoration: underline; + cursor: pointer; +} + +.sync-policy textarea { + width: 100%; + height: 200px; +} diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html index 30486bada..58c0d9ac3 100644 --- a/packages/nylas-dashboard/public/index.html +++ b/packages/nylas-dashboard/public/index.html @@ -3,6 +3,7 @@ + diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index ee753bfd6..ce8561fa1 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -1,6 +1,7 @@ /* eslint react/react-in-jsx-scope: 0*/ const React = window.React; const ReactDOM = window.ReactDOM; +const SyncPolicy = window.SyncPolicy; class Account extends React.Component { renderError() { @@ -36,8 +37,10 @@ class Account extends React.Component {

{account.email_address} {active ? '🌕' : '🌑'}

{assignment} -
Sync Policy
- 
{JSON.stringify(account.sync_policy, null, 2)}
+
Sync Cycles
First Sync Completion: diff --git a/packages/nylas-dashboard/public/js/sync-policy.jsx b/packages/nylas-dashboard/public/js/sync-policy.jsx new file mode 100644 index 000000000..bfbee8ffb --- /dev/null +++ b/packages/nylas-dashboard/public/js/sync-policy.jsx @@ -0,0 +1,66 @@ +const React = window.React; + +class SyncPolicy extends React.Component { + constructor(props) { + super(props); + this.state = {editMode: false}; + this.accountId = props.accountId; + } + + edit() { + this.setState({editMode: true}) + } + + save() { + const req = new XMLHttpRequest(); + const url = `${window.location.protocol}/sync-policy/${this.accountId}`; + req.open("POST", url, true); + req.setRequestHeader("Content-type", "application/json"); + req.onreadystatechange = () => { + if (req.readyState === XMLHttpRequest.DONE) { + console.log(req.responseText); + if (req.status === 200) { + this.setState({editMode: false}); + } + } + } + + const newPolicy = document.getElementById(`sync-policy-${this.accountId}`).value; + req.send(JSON.stringify({sync_policy: newPolicy})); + } + + cancel() { + this.setState({editMode: false}); + } + + render() { + if (this.state.editMode) { + const id = `sync-policy-${this.props.accountId}`; + return ( +
+
Sync Policy
+ + + this.cancel.call(this)}> Cancel +
+ + ) + } + return ( +
+
Sync Policy
+ 
{this.props.stringifiedSyncPolicy}
+ this.edit.call(this)}> Edit +
+ ) + } +} + +SyncPolicy.propTypes = { + accountId: React.PropTypes.number, + stringifiedSyncPolicy: React.PropTypes.string, +} + +window.SyncPolicy = SyncPolicy; diff --git a/packages/nylas-dashboard/routes/sync-policy.js b/packages/nylas-dashboard/routes/sync-policy.js new file mode 100644 index 000000000..2822849f7 --- /dev/null +++ b/packages/nylas-dashboard/routes/sync-policy.js @@ -0,0 +1,30 @@ +const Joi = require('joi'); +const {SchedulerUtils} = require(`nylas-core`); + +module.exports = (server) => { + server.route({ + method: 'POST', + path: '/sync-policy/{account_id}', + config: { + description: 'Set the sync policy', + notes: 'Notes go here', + tags: ['sync-policy'], + validate: { + params: { + account_id: Joi.number().integer(), + }, + payload: { + sync_policy: Joi.string(), + }, + }, + response: { + schema: Joi.string(), + }, + }, + handler: (request, reply) => { + const newPolicy = JSON.parse(request.payload.sync_policy); + SchedulerUtils.assignPolicy(request.params.account_id, newPolicy) + .then(() => reply("Success")); + }, + }); +}; From 5d2c52df41308e9a2a00cddbc382c650fe981c37 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Tue, 5 Jul 2016 15:20:37 -0700 Subject: [PATCH 163/800] Start just nylas-api --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 2b2aa647c..52189ef05 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,8 @@ "lerna": "2.0.0-beta.20" }, "scripts": { - "start": "foreman start -f Procfile.dev", - "postinstall": "lerna bootstrap" + "start": "node packages/nylas-api/app.js", + "postinstall": "" }, "repository": { "type": "git", From 3068fbbf206e8fd88f5a86cf2aa02fb32b731dab Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 6 Jul 2016 13:50:22 -0700 Subject: [PATCH 164/800] Upgrade lerna --- .gitignore | 1 + lerna.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c5eadbbb1..ce1018e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules dump.rdb *npm-debug.log storage/ +lerna-debug.log # Elastic Beanstalk Files .elasticbeanstalk/* diff --git a/lerna.json b/lerna.json index 56dbae74a..7a61c20d8 100644 --- a/lerna.json +++ b/lerna.json @@ -1,4 +1,4 @@ { - "lerna": "2.0.0-beta.20", + "lerna": "2.0.0-beta.23", "version": "0.0.1" } From af9d5da17c36aa5ecd129334b8e1477b23804db6 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 6 Jul 2016 13:51:05 -0700 Subject: [PATCH 165/800] Revert "Start just nylas-api" This reverts commit 91d8cfa6c0b8fe4c6df007e095ca4af07e3fd3ef. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 52189ef05..2b2aa647c 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,8 @@ "lerna": "2.0.0-beta.20" }, "scripts": { - "start": "node packages/nylas-api/app.js", - "postinstall": "" + "start": "foreman start -f Procfile.dev", + "postinstall": "lerna bootstrap" }, "repository": { "type": "git", From 2e4427d96e21c3c6a511b45f3c492549c702cf4e Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 6 Jul 2016 14:59:56 -0700 Subject: [PATCH 166/800] Add console to ping --- packages/nylas-api/routes/ping.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nylas-api/routes/ping.js b/packages/nylas-api/routes/ping.js index 3e1d10f09..a2c9e82e4 100644 --- a/packages/nylas-api/routes/ping.js +++ b/packages/nylas-api/routes/ping.js @@ -6,6 +6,7 @@ module.exports = (server) => { auth: false, }, handler: (request, reply) => { + console.log("---> Ping!") reply("pong") }, }); From be277b80048ae4dd9ca459f89c8ca783172762b8 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 6 Jul 2016 15:30:35 -0700 Subject: [PATCH 167/800] Add dockerfile --- Dockerfile | 9 +++++++++ package.json | 5 +++-- start-aws.js | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 start-aws.js diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..be4deaf43 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# https://github.com/nodejs/docker-node +FROM node:6 +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +COPY package.json /usr/src/app/ +RUN npm install +COPY . /usr/src/app +EXPOSE 8080 +CMD [ "npm", "run", "start-aws"] diff --git a/package.json b/package.json index 2b2aa647c..672cbaadf 100644 --- a/package.json +++ b/package.json @@ -19,11 +19,12 @@ "eslint-plugin-jsx-a11y": "1.x", "eslint-plugin-react": "5.x", "eslint_d": "3.x", - "lerna": "2.0.0-beta.20" + "lerna": "2.0.0-beta.23" }, "scripts": { "start": "foreman start -f Procfile.dev", - "postinstall": "lerna bootstrap" + "start-aws": "node start-aws.js", + "postinstall": "node_modules/.bin/lerna bootstrap" }, "repository": { "type": "git", diff --git a/start-aws.js b/start-aws.js new file mode 100644 index 000000000..b08e1d0f0 --- /dev/null +++ b/start-aws.js @@ -0,0 +1,6 @@ +const proc = require('child_process'); + +// TODO: Swtich on env variables +proc.spawn('node', ['packages/nylas-api/app.js'], { + stdio: 'inherit' +}) From 12560ab71173b5276606d9a8bb9d0f21e3eb9f8e Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Wed, 6 Jul 2016 15:48:25 -0700 Subject: [PATCH 168/800] Add ability to apply a sync policy to all accounts from dashboard --- packages/nylas-core/scheduler-utils.js | 14 ++++ packages/nylas-dashboard/public/css/app.css | 23 +++++++ packages/nylas-dashboard/public/index.html | 1 + packages/nylas-dashboard/public/js/app.jsx | 6 +- .../public/js/set-all-sync-policies.jsx | 64 +++++++++++++++++++ .../nylas-dashboard/routes/sync-policy.js | 24 +++++++ 6 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 packages/nylas-dashboard/public/js/set-all-sync-policies.jsx diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js index e6a4fa824..55f8a4959 100644 --- a/packages/nylas-core/scheduler-utils.js +++ b/packages/nylas-core/scheduler-utils.js @@ -32,6 +32,19 @@ const assignPolicy = (accountId, policy) => { }); } +const assignPolicyToAcounts = (accountIds, policy) => { + console.log(`Changing policy for ${accountIds} to ${JSON.stringify(policy)}`) + const DatabaseConnector = require('./database-connector'); + return DatabaseConnector.forShared().then(({Account}) => { + Account.findAll({where: {id: {$or: accountIds}}}).then((accounts) => { + for (const account of accounts) { + account.syncPolicy = policy; + account.save() + } + }) + }); +} + const checkIfAccountIsActive = (accountId) => { const client = PubsubConnector.broadcastClient(); const key = ACTIVE_KEY_FOR(accountId); @@ -69,6 +82,7 @@ module.exports = { CLAIM_DURATION, assignPolicy, + assignPolicyToAcounts, forEachAccountList, listActiveAccounts, markAccountIsActive, diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index 78c729cf5..6871c472f 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -48,6 +48,11 @@ pre { overflow: hidden; } +#set-all-sync { + display: block; + margin-bottom: 10px; +} + .sync-policy .action-link { display: inline-block; margin: 5px; @@ -59,4 +64,22 @@ pre { .sync-policy textarea { width: 100%; height: 200px; + white-space: pre; +} + +.modal { + background-color: white; + width: 50%; + margin: auto; + padding: 20px; +} + +.modal-bg { + position: fixed; + width: 100%; + height: 100%; + left: 0; + top: 0; + background-color: rgba(0, 0, 0, 0.3); + padding-top: 10%; } diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html index 58c0d9ac3..5f69bb9e0 100644 --- a/packages/nylas-dashboard/public/index.html +++ b/packages/nylas-dashboard/public/index.html @@ -4,6 +4,7 @@ + diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index ce8561fa1..9f9d69b49 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -1,7 +1,7 @@ /* eslint react/react-in-jsx-scope: 0*/ const React = window.React; const ReactDOM = window.ReactDOM; -const SyncPolicy = window.SyncPolicy; +const {SyncPolicy, SetAllSyncPolicies} = window; class Account extends React.Component { renderError() { @@ -119,10 +119,12 @@ class Root extends React.Component { } render() { + const ids = Object.keys(this.state.accounts); return (
+ parseInt(id, 10))} /> { - Object.keys(this.state.accounts).sort((a, b) => a.localeCompare(b)).map((id) => + ids.sort((a, b) => a.localeCompare(b)).map((id) => { + if (req.readyState === XMLHttpRequest.DONE) { + console.log(req.responseText); + if (req.status === 200) { + this.setState({editMode: false}); + } + } + } + + const newPolicy = document.getElementById(`sync-policy-all`).value; + req.send(JSON.stringify({ + sync_policy: newPolicy, + account_ids: accountIds, + })); + } + + cancel() { + this.setState({editMode: false}); + } + + render() { + if (this.state.editMode) { + return ( +
+
+
Sync Policy
+ + + this.cancel.call(this)}> Cancel +
+
+ ) + } + return ( + + ) + } +} + +SetAllSyncPolicies.propTypes = { + accountIds: React.PropTypes.arrayOf(React.PropTypes.number), +} + +window.SetAllSyncPolicies = SetAllSyncPolicies; diff --git a/packages/nylas-dashboard/routes/sync-policy.js b/packages/nylas-dashboard/routes/sync-policy.js index 2822849f7..ac327af4b 100644 --- a/packages/nylas-dashboard/routes/sync-policy.js +++ b/packages/nylas-dashboard/routes/sync-policy.js @@ -27,4 +27,28 @@ module.exports = (server) => { .then(() => reply("Success")); }, }); + + server.route({ + method: 'POST', + path: '/sync-policy', + config: { + description: 'Set the sync policy for several accounts', + notes: 'Notes go here', + tags: ['sync-policy'], + validate: { + payload: { + sync_policy: Joi.string(), + account_ids: Joi.array().items(Joi.number().integer().min(0)), + }, + }, + response: { + schema: Joi.string(), + }, + }, + handler: (request, reply) => { + const newPolicy = JSON.parse(request.payload.sync_policy); + SchedulerUtils.assignPolicyToAcounts(request.payload.account_ids, newPolicy) + .then(() => reply("Success")); + }, + }) }; From 40ab07cfdf23c122c2c9c99813633001cdea43c3 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Wed, 6 Jul 2016 16:49:16 -0700 Subject: [PATCH 169/800] Add account filtering (by error status) to dashboard --- packages/nylas-dashboard/public/css/app.css | 2 +- packages/nylas-dashboard/public/index.html | 1 + .../public/js/account-filter.jsx | 26 +++++++++++++++++++ packages/nylas-dashboard/public/js/app.jsx | 22 ++++++++++++++-- .../public/js/set-all-sync-policies.jsx | 6 +++-- 5 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 packages/nylas-dashboard/public/js/account-filter.jsx diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index 6871c472f..0809a0541 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -53,7 +53,7 @@ pre { margin-bottom: 10px; } -.sync-policy .action-link { +.action-link { display: inline-block; margin: 5px; color: rgba(16, 83, 161, 0.88); diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html index 5f69bb9e0..cf8b10b05 100644 --- a/packages/nylas-dashboard/public/index.html +++ b/packages/nylas-dashboard/public/index.html @@ -5,6 +5,7 @@ + diff --git a/packages/nylas-dashboard/public/js/account-filter.jsx b/packages/nylas-dashboard/public/js/account-filter.jsx new file mode 100644 index 000000000..2d7a37b67 --- /dev/null +++ b/packages/nylas-dashboard/public/js/account-filter.jsx @@ -0,0 +1,26 @@ +const React = window.React; + +function AccountFilter(props) { + return ( +
+ Display: +
+ ) +} + +AccountFilter.propTypes = { + onChange: React.PropTypes.func, + id: React.PropTypes.string, +} + +AccountFilter.states = { + all: "all", + errored: "errored", + notErrored: "not-errored", +}; + +window.AccountFilter = AccountFilter; diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 9f9d69b49..64fd80519 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -1,7 +1,7 @@ /* eslint react/react-in-jsx-scope: 0*/ const React = window.React; const ReactDOM = window.ReactDOM; -const {SyncPolicy, SetAllSyncPolicies} = window; +const {SyncPolicy, SetAllSyncPolicies, AccountFilter} = window; class Account extends React.Component { renderError() { @@ -69,6 +69,7 @@ class Root extends React.Component { accounts: {}, assignments: {}, activeAccountIds: [], + visibleAccounts: AccountFilter.states.all, }; } @@ -118,10 +119,27 @@ class Root extends React.Component { this.setState({accounts}); } + onFilter() { + this.setState({visibleAccounts: document.getElementById('account-filter').value}); + } + render() { - const ids = Object.keys(this.state.accounts); + let ids = Object.keys(this.state.accounts); + + switch (this.state.visibleAccounts) { + case AccountFilter.states.errored: + ids = ids.filter((id) => this.state.accounts[id].sync_error) + break; + case AccountFilter.states.notErrored: + ids = ids.filter((id) => !this.state.accounts[id].sync_error) + break; + default: + break; + } + return (
+ this.onFilter.call(this)} /> parseInt(id, 10))} /> { ids.sort((a, b) => a.localeCompare(b)).map((id) => diff --git a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx index 00ad5fd44..f9346c1e6 100644 --- a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx +++ b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx @@ -44,7 +44,7 @@ class SetAllSyncPolicies extends React.Component { this.cancel.call(this)}> Cancel
@@ -52,7 +52,9 @@ class SetAllSyncPolicies extends React.Component { ) } return ( - + this.edit.call(this)}> + Set sync policies for currently displayed accounts + ) } } From 822b8e54e6b4dde8d43aec9a6dbeee34508d44ad Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 7 Jul 2016 11:37:55 -0700 Subject: [PATCH 170/800] Allow usage with mysql in addition to SQLite --- package.json | 1 + packages/nylas-api/routes/auth.js | 11 +++--- packages/nylas-core/database-connector.js | 46 +++++++++++++++++------ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 672cbaadf..4b3ab2a8b 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "", "dependencies": { "bluebird": "3.x.x", + "mysql": "^2.11.1", "redis": "2.x.x", "rx": "4.x.x", "sequelize": "3.x.x", diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index 35e771915..fd14f4fc5 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -9,8 +9,6 @@ const { DatabaseConnector, SyncPolicy, Provider, - PubsubConnector, - MessageTypes, } = require('nylas-core'); const {GMAIL_CLIENT_ID, GMAIL_CLIENT_SECRET, GMAIL_REDIRECT_URL} = process.env; @@ -56,10 +54,11 @@ const buildAccountWith = ({name, email, provider, settings, credentials}) => { account.setCredentials(credentials); return account.save().then((saved) => - AccountToken.create({ - accountId: saved.id, - }).then((token) => - Promise.resolve({account: saved, token: token}) + AccountToken.create({accountId: saved.id}).then((token) => + DatabaseConnector.prepareAccountDatabase(saved.id).thenReturn({ + account: saved, + token: token, + }) ) ); }); diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index 5d95b16e1..d7459458a 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -34,16 +34,27 @@ class DatabaseConnector { return db; } + _sequelizePoolForDatabase(dbname) { + if (process.env.DB_HOSTNAME) { + return new Sequelize(dbname, process.env.DB_USERNAME, process.env.DB_PASSWORD, { + host: process.env.DB_HOSTNAME, + dialect: "mysql", + logging: false, + }); + } + + return Sequelize(dbname, '', '', { + storage: path.join(STORAGE_DIR, `${dbname}.sqlite`), + dialect: "sqlite", + logging: false, + }) + } + _sequelizeForAccount(accountId) { if (!accountId) { return Promise.reject(new Error(`You need to pass an accountId to init the database!`)) } - const sequelize = new Sequelize(accountId, '', '', { - storage: path.join(STORAGE_DIR, `a-${accountId}.sqlite`), - dialect: "sqlite", - logging: false, - }); - + const sequelize = this._sequelizePoolForDatabase(`a-${accountId}`); const modelsPath = path.join(__dirname, 'models/account'); const db = this._readModelsInDirectory(sequelize, modelsPath) @@ -64,13 +75,24 @@ class DatabaseConnector { return this._pools[accountId]; } - _sequelizeForShared() { - const sequelize = new Sequelize('shared', '', '', { - storage: path.join(STORAGE_DIR, 'shared.sqlite'), - dialect: "sqlite", - logging: false, - }); + prepareAccountDatabase(accountId) { + const dbname = `a-${accountId}`; + if (process.env.DB_HOSTNAME) { + const sequelize = new Sequelize(null, process.env.DB_USERNAME, process.env.DB_PASSWORD, { + host: process.env.DB_HOSTNAME, + dialect: "mysql", + logging: false, + }) + return sequelize.authenticate().then(() => + sequelize.query(`CREATE DATABASE \`${dbname}\``) + ); + } + return Promise.resolve() + } + + _sequelizeForShared() { + const sequelize = this._sequelizePoolForDatabase(`shared`); const modelsPath = path.join(__dirname, 'models/shared'); const db = this._readModelsInDirectory(sequelize, modelsPath) From f14443a83ea7a59f999667c1a3bf0d1ddd11cd33 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 6 Jul 2016 15:31:10 -0700 Subject: [PATCH 171/800] Remove sqlite --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 4b3ab2a8b..defa37b40 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ "redis": "2.x.x", "rx": "4.x.x", "sequelize": "3.x.x", - "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz", "underscore": "1.x.x" }, "devDependencies": { From 3820521d62fdb0aa466a95bba0189e66f448fe86 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Wed, 6 Jul 2016 16:22:32 -0700 Subject: [PATCH 172/800] Dockerfile successfully building ping endpoint --- .dockerignore | 6 +++++ Dockerfile | 21 +++++++++++++----- packages/nylas-api/app.js | 46 +++++++++++++++++++++++---------------- 3 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..d80f4ce50 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.gitignore +README.md +Procfile* +*node_modules* +docs diff --git a/Dockerfile b/Dockerfile index be4deaf43..fa5e964b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,20 @@ +# Use the latest Node 6 base docker image # https://github.com/nodejs/docker-node FROM node:6 -RUN mkdir -p /usr/src/app -WORKDIR /usr/src/app -COPY package.json /usr/src/app/ + +# Copy everything (excluding what's in .dockerignore) into an empty dir +COPY . /home +WORKDIR /home + RUN npm install -COPY . /usr/src/app -EXPOSE 8080 + +# This will do an `npm install` for each of our modules and then link them +# all together. See more about Lerna here: https://github.com/lerna/lerna +RUN node_modules/.bin/lerna bootstrap + +# External services run on port 5100. Expose it. +EXPOSE 5100 + +# We use a start-aws command that automatically spawns the correct process +# based on environment variables (which changes instance to instance) CMD [ "npm", "run", "start-aws"] diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index 8bcb3359f..68ebaa377 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -31,30 +31,38 @@ const plugins = [Inert, Vision, HapiBasicAuth, HapiBoom, { }]; let sharedDb = null; -const {DatabaseConnector, SchedulerUtils} = require(`nylas-core`) -DatabaseConnector.forShared().then((db) => { - sharedDb = db; -}); const validate = (request, username, password, callback) => { - const {AccountToken} = sharedDb; + const {DatabaseConnector, SchedulerUtils} = require(`nylas-core`); - AccountToken.find({ - where: { - value: username, - }, - }).then((token) => { - if (!token) { - callback(null, false, {}); - return - } - token.getAccount().then((account) => { - if (!account) { + let getSharedDb = null; + if (sharedDb) { + getSharedDb = Promise.resolve(sharedDb) + } else { + getSharedDb = DatabaseConnector.forShared() + } + + getSharedDb.then((db) => { + sharedDb = db; + const {AccountToken} = db; + + AccountToken.find({ + where: { + value: username, + }, + }).then((token) => { + if (!token) { callback(null, false, {}); - return; + return } - SchedulerUtils.markAccountIsActive(account.id) - callback(null, true, account); + token.getAccount().then((account) => { + if (!account) { + callback(null, false, {}); + return; + } + SchedulerUtils.markAccountIsActive(account.id) + callback(null, true, account); + }); }); }); }; From a97ed5a311669c0a0d213681f999e0a7e2e60223 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 12:08:48 -0700 Subject: [PATCH 173/800] Added Dockerrun.aws.json --- Dockerrun.aws.json | 16 ++++++++++++++++ README.md | 37 ++++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 13 deletions(-) create mode 100644 Dockerrun.aws.json diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json new file mode 100644 index 000000000..7cdb1b7b6 --- /dev/null +++ b/Dockerrun.aws.json @@ -0,0 +1,16 @@ +{ + "AWSEBDockerrunVersion": "1", + "Authentication": { + "Bucket": "elasticbeanstalk-us-east-1-925176737378", + "Key": "docker/.dockercfg" + }, + "Image": { + "Name": "nylas/k2", + "Update": "true" + }, + "Ports": [ + { + "ContainerPort": "5100" + } + ] +} diff --git a/README.md b/README.md index 47160fad6..21d9803c6 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,34 @@ # K2 - Sync Engine Experiment -# Initial Setup +# Initial Setup: -1. Download https://toolbelt.heroku.com/ +## New Computer (Mac): -``` -brew install redis -nvm install 6 -npm install -``` +1. Install [Homebrew](http://brew.sh/) +1. Install [VirtualBox 5+](https://www.virtualbox.org/wiki/Downloads) +1. Install [Docker for Mac](https://docs.docker.com/docker-for-mac/) +1. Install [NVM](https://github.com/creationix/nvm) `brew install nvm` +1. Install Node 6+ via NVM: `nvm install 6` -# Running locally +## New to AWS: + +1. Install [Elastic Beanstalk CLI](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html#eb-cli3-install-osx): `brew install awsebcli` +1. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli` + 1. Add your AWS IAM Security Credentials to `aws configure`. + 1. These are at Console Home -> IAM -> Users -> {{Your Name}} -> Security + Credentials. Note that your private key was only shown unpon creation. If + you've lost your private key you have to deactivate your old key and + create a new one. +1. Get the K2 team private SSH key. (Ignore this when we have a Bastion Host). Ask someone on K2 for a copy of the private SSH key. Copy it to your ~/.ssh folder. + 1. `chmod 400 ~/.ssh/k2-keypair.pem` + 1. `ssh i ~/.ssh/k2-keypair.pem some-ec2-box-we-own.amazonaws.com` +1. Connect to Elastic Beanstalk instances: `eb init`. Select correct region. Select correct application. + + +# Developing Locally: ``` npm start ``` -## Auth an account - -``` -curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" -``` +# Deploying From 05de9f6cffc65ddae8ddadf80245d73f8023cce8 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 12:09:46 -0700 Subject: [PATCH 174/800] Remove .elasticbeanstalk from gitignore --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index ce1018e2e..64172ae2d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,3 @@ dump.rdb *npm-debug.log storage/ lerna-debug.log - -# Elastic Beanstalk Files -.elasticbeanstalk/* -!.elasticbeanstalk/*.cfg.yml -!.elasticbeanstalk/*.global.yml From ecb97fa4378a4f8166bd00343795c04cde05e8ee Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 12:10:17 -0700 Subject: [PATCH 175/800] Revert "Remove sqlite" This reverts commit f14443a83ea7a59f999667c1a3bf0d1ddd11cd33. --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index defa37b40..4b3ab2a8b 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "redis": "2.x.x", "rx": "4.x.x", "sequelize": "3.x.x", + "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz", "underscore": "1.x.x" }, "devDependencies": { From ac5fee1c39d6997ffd7c6836cd270a53b50ee258 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 12:13:27 -0700 Subject: [PATCH 176/800] Re-add .elasticbeanstalk to gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 64172ae2d..ce1018e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,8 @@ dump.rdb *npm-debug.log storage/ lerna-debug.log + +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml From 189cf21d2ad168ab8123766b903ac0abdd7f353b Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 12:39:31 -0700 Subject: [PATCH 177/800] Commit to test Docker automatic builds --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 21d9803c6..0e42d475d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,6 @@ 1. `ssh i ~/.ssh/k2-keypair.pem some-ec2-box-we-own.amazonaws.com` 1. Connect to Elastic Beanstalk instances: `eb init`. Select correct region. Select correct application. - # Developing Locally: ``` From dfb3935969b8016721da3ab0fb462e6f67d4367a Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 13:32:54 -0700 Subject: [PATCH 178/800] Update package.json. Remove Procfile --- Dockerfile | 2 +- Procfile | 2 -- Procfile.dev | 5 ----- package.json | 4 ++-- 4 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 Procfile delete mode 100644 Procfile.dev diff --git a/Dockerfile b/Dockerfile index fa5e964b9..9f56102cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:6 # Copy everything (excluding what's in .dockerignore) into an empty dir -COPY . /home +COPY ../ /home WORKDIR /home RUN npm install diff --git a/Procfile b/Procfile deleted file mode 100644 index bc443adf9..000000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: node packages/nylas-api/app.js -worker: node packages/nylas-sync/app.js diff --git a/Procfile.dev b/Procfile.dev deleted file mode 100644 index 03ec16455..000000000 --- a/Procfile.dev +++ /dev/null @@ -1,5 +0,0 @@ -redis: redis-server -api: node packages/nylas-api/app.js -sync: node packages/nylas-sync/app.js -processor: node packages/nylas-message-processor/app.js -dashboard: node packages/nylas-dashboard/app.js diff --git a/package.json b/package.json index 4b3ab2a8b..ff375096e 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ "redis": "2.x.x", "rx": "4.x.x", "sequelize": "3.x.x", - "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz", "underscore": "1.x.x" }, "devDependencies": { @@ -20,7 +19,8 @@ "eslint-plugin-jsx-a11y": "1.x", "eslint-plugin-react": "5.x", "eslint_d": "3.x", - "lerna": "2.0.0-beta.23" + "lerna": "2.0.0-beta.23", + "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { "start": "foreman start -f Procfile.dev", From 21a89ee45542ed6947a91196fb7ddb64bb918c9c Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 13:33:27 -0700 Subject: [PATCH 179/800] Remove Dockerrun.aws.json --- Dockerrun.aws.json | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 Dockerrun.aws.json diff --git a/Dockerrun.aws.json b/Dockerrun.aws.json deleted file mode 100644 index 7cdb1b7b6..000000000 --- a/Dockerrun.aws.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "AWSEBDockerrunVersion": "1", - "Authentication": { - "Bucket": "elasticbeanstalk-us-east-1-925176737378", - "Key": "docker/.dockercfg" - }, - "Image": { - "Name": "nylas/k2", - "Update": "true" - }, - "Ports": [ - { - "ContainerPort": "5100" - } - ] -} From 83e7d149a3f064efa82f8f2ac587a2a313b73c7d Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 13:36:38 -0700 Subject: [PATCH 180/800] Fix Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f56102cd..fa5e964b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:6 # Copy everything (excluding what's in .dockerignore) into an empty dir -COPY ../ /home +COPY . /home WORKDIR /home RUN npm install From 1ed55a6ee6085a8d4b0a23737c2af0b2d6a6abbf Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 13:37:33 -0700 Subject: [PATCH 181/800] Add production flag to Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa5e964b9..5a8fbcf45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ FROM node:6 COPY . /home WORKDIR /home -RUN npm install +RUN npm install --production # This will do an `npm install` for each of our modules and then link them # all together. See more about Lerna here: https://github.com/lerna/lerna From 7a4550627176178de2174beb6a9e44ce5f371877 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 13:42:44 -0700 Subject: [PATCH 182/800] Update package.json to include lerna in main dependencies --- Dockerfile | 9 +++++++-- package.json | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5a8fbcf45..00f32fa29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,7 @@ +# This Dockerfile builds a production-ready image of K2 to be used across all +# services. See the Dockerfile documentation here: +# https://docs.docker.com/engine/reference/builder/ + # Use the latest Node 6 base docker image # https://github.com/nodejs/docker-node FROM node:6 @@ -8,8 +12,9 @@ WORKDIR /home RUN npm install --production -# This will do an `npm install` for each of our modules and then link them -# all together. See more about Lerna here: https://github.com/lerna/lerna +# This will do an `npm install` for each of our modules and then link them all +# together. See more about Lerna here: https://github.com/lerna/lerna We have +# to run this separately from npm postinstall due to permission issues. RUN node_modules/.bin/lerna bootstrap # External services run on port 5100. Expose it. diff --git a/package.json b/package.json index ff375096e..ffda11a18 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "main": "", "dependencies": { "bluebird": "3.x.x", + "lerna": "2.0.0-beta.23", "mysql": "^2.11.1", "redis": "2.x.x", "rx": "4.x.x", @@ -19,7 +20,6 @@ "eslint-plugin-jsx-a11y": "1.x", "eslint-plugin-react": "5.x", "eslint_d": "3.x", - "lerna": "2.0.0-beta.23", "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { From 2db11d7dec494629be7c8363ed409aa8a9648dd7 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Thu, 7 Jul 2016 14:30:51 -0700 Subject: [PATCH 183/800] Enhance sync stats on dashboard --- packages/nylas-core/models/shared/account.js | 1 + packages/nylas-dashboard/public/css/app.css | 16 +++- packages/nylas-dashboard/public/index.html | 1 + packages/nylas-dashboard/public/js/app.jsx | 49 ++++++---- .../nylas-dashboard/public/js/sync-graph.jsx | 92 +++++++++++++++++++ packages/nylas-sync/sync-worker.js | 9 +- 6 files changed, 148 insertions(+), 20 deletions(-) create mode 100644 packages/nylas-dashboard/public/js/sync-graph.jsx diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index 83039d542..29a845ac9 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -33,6 +33,7 @@ module.exports = (sequelize, Sequelize) => { sync_error: this.syncError, first_sync_completed_at: this.firstSyncCompletedAt, last_sync_completions: this.lastSyncCompletions, + created_at: this.createdAt, } }, diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index 0809a0541..ceb513342 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -1,6 +1,9 @@ body { + background-image: url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg); + background-image: -moz-linear-gradient(top, rgba(232, 244, 250, 0.2), rgba(231, 231, 233, 1)), url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg); background-image: -webkit-linear-gradient(top, rgba(232, 244, 250, 0.2), rgba(231, 231, 233, 1)), url(http://news.nationalgeographic.com/content/dam/news/2015/12/13/BookTalk%20K2/01BookTalkK2.jpg); - background-size: cover; + background-size: 100vw auto; + background-attachment: fixed; font-family: Roboto, sans-serif; font-size: 12px; } @@ -51,6 +54,7 @@ pre { #set-all-sync { display: block; margin-bottom: 10px; + color: #ffffff; } .action-link { @@ -83,3 +87,13 @@ pre { background-color: rgba(0, 0, 0, 0.3); padding-top: 10%; } + +.sync-graph { + margin-top: 3px; +} + +.stats b { + display: inline-block; + margin-top: 5px; + margin-bottom: 1px; +} diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html index cf8b10b05..89227c3d8 100644 --- a/packages/nylas-dashboard/public/index.html +++ b/packages/nylas-dashboard/public/index.html @@ -6,6 +6,7 @@ + diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 64fd80519..5f740093b 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -1,7 +1,12 @@ /* eslint react/react-in-jsx-scope: 0*/ const React = window.React; const ReactDOM = window.ReactDOM; -const {SyncPolicy, SetAllSyncPolicies, AccountFilter} = window; +const { + SyncPolicy, + SetAllSyncPolicies, + AccountFilter, + SyncGraph, +} = window; class Account extends React.Component { renderError() { @@ -14,10 +19,13 @@ class Account extends React.Component { stack: stack.slice(0, 4), } return ( -
- 
-            {JSON.stringify(error, null, 2)}
-
+
+
Error
+
+ 
+              {JSON.stringify(error, null, 2)}
+
+
) } @@ -27,12 +35,18 @@ class Account extends React.Component { render() { const {account, assignment, active} = this.props; const errorClass = account.sync_error ? ' errored' : '' - const lastSyncCompletions = [] - for (const time of account.last_sync_completions) { - lastSyncCompletions.push( -
{new Date(time).toString()}
- ) + + const numStoredSyncs = account.last_sync_completions.length; + const oldestSync = account.last_sync_completions[numStoredSyncs - 1]; + const newestSync = account.last_sync_completions[0]; + const avgBetweenSyncs = (newestSync - oldestSync) / (1000 * numStoredSyncs); + const timeSinceLastSync = (Date.now() - newestSync) / 1000; + + let firstSyncDuration = "Incomplete"; + if (account.first_sync_completed_at) { + firstSyncDuration = (new Date(account.first_sync_completed_at) - new Date(account.created_at)) / 1000; } + return (

{account.email_address} {active ? '🌕' : '🌑'}

@@ -42,13 +56,16 @@ class Account extends React.Component { stringifiedSyncPolicy={JSON.stringify(account.sync_policy, null, 2)} />
Sync Cycles
-
- First Sync Completion: - 
{new Date(account.first_sync_completed_at).toString()}
+
+ First Sync Duration (seconds): + 
{firstSyncDuration}
+ Average Time Between Syncs (seconds): + 
{avgBetweenSyncs}
+ Time Since Last Sync (seconds): + 
{timeSinceLastSync}
+ Recent Syncs: +
-
Last Sync Completions:
- 
{lastSyncCompletions}
-
Error
{this.renderError()}
); diff --git a/packages/nylas-dashboard/public/js/sync-graph.jsx b/packages/nylas-dashboard/public/js/sync-graph.jsx new file mode 100644 index 000000000..7c1a1fc15 --- /dev/null +++ b/packages/nylas-dashboard/public/js/sync-graph.jsx @@ -0,0 +1,92 @@ +const React = window.React; +const ReactDOM = window.ReactDOM; + +class SyncGraph extends React.Component { + + componentDidMount() { + this.drawGraph(); + } + + componentDidUpdate() { + this.drawGraph(true); + } + + drawGraph(isUpdate) { + const now = Date.now(); + const config = SyncGraph.config; + const context = ReactDOM.findDOMNode(this).getContext('2d'); + + // Background + // (This hides any previous data points, so we don't have to clear the canvas) + context.fillStyle = config.backgroundColor; + context.fillRect(0, 0, config.width, config.height); + + // Data points + const pxPerSec = config.width / config.timeLength; + context.strokeStyle = config.dataColor; + context.beginPath(); + for (const syncTimeMs of this.props.syncTimestamps) { + const secsAgo = (now - syncTimeMs) / 1000; + const pxFromRight = secsAgo * pxPerSec; + const pxFromLeft = config.width - pxFromRight; + context.moveTo(pxFromLeft, 0); + context.lineTo(pxFromLeft, config.height); + } + context.stroke(); + + // Tick marks + const interval = config.width / config.numTicks; + context.strokeStyle = config.tickColor; + context.beginPath(); + for (let px = interval; px < config.width; px += interval) { + context.moveTo(px, config.height - config.tickHeight); + context.lineTo(px, config.height); + } + context.stroke(); + + // Axis labels + if (!isUpdate) { // only draw these on the initial render + context.fillStyle = config.labelColor; + context.font = `${config.labelFontSize}px sans-serif`; + const fontY = config.height + config.labelFontSize + config.labelTopMargin; + const nowText = "now"; + const nowWidth = context.measureText(nowText).width; + context.fillText(nowText, config.width - nowWidth - 1, fontY); + context.fillText("-30m", 1, fontY); + } + } + + render() { + return ( + + ) + } + +} + +SyncGraph.config = { + height: 50, // Doesn't include labels + width: 300, + // timeLength is 30 minutes in seconds. If you change this, be sure to update + // syncGraphTimeLength in sync-worker.js and the axis labels in drawGraph()! + timeLength: 60 * 30, + numTicks: 10, + tickHeight: 10, + tickColor: 'white', + labelFontSize: 8, + labelTopMargin: 2, + labelColor: 'black', + backgroundColor: 'black', + dataColor: 'blue', +} + +SyncGraph.propTypes = { + syncTimestamps: React.PropTypes.arrayOf(React.PropTypes.number), +} + +window.SyncGraph = SyncGraph; diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 41815610f..67c69a719 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -189,11 +189,14 @@ class SyncWorker { this._account.firstSyncCompletedAt = Date.now() } + const now = Date.now(); + const syncGraphTimeLength = 60 * 30; // 30 minutes, should be the same as SyncGraph.config.timeLength let lastSyncCompletions = [...this._account.lastSyncCompletions] - lastSyncCompletions = [Date.now(), ...lastSyncCompletions] - if (lastSyncCompletions.length > 10) { - lastSyncCompletions.pop() + lastSyncCompletions = [now, ...lastSyncCompletions] + while (now - lastSyncCompletions[lastSyncCompletions.length - 1] > 1000 * syncGraphTimeLength) { + lastSyncCompletions.pop(); } + this._account.lastSyncCompletions = lastSyncCompletions this._account.save() console.log('Syncworker: Completed sync cycle') From 63f66fb7e7e10bd944bff009c7c18a3dec1941b9 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 7 Jul 2016 15:25:45 -0700 Subject: [PATCH 184/800] Switch to PM2 for dev + prod --- Dockerfile | 2 +- Procfile | 2 -- Procfile.dev | 5 ----- package.json | 4 +++- packages/nylas-core/database-connector.js | 2 +- pm2-dev.yml | 24 +++++++++++++++++++++++ pm2-prod-api.yml | 9 +++++++++ pm2-prod-sync.yml | 9 +++++++++ start-aws.js | 6 ------ 9 files changed, 47 insertions(+), 16 deletions(-) delete mode 100644 Procfile delete mode 100644 Procfile.dev create mode 100644 pm2-dev.yml create mode 100644 pm2-prod-api.yml create mode 100644 pm2-prod-sync.yml delete mode 100644 start-aws.js diff --git a/Dockerfile b/Dockerfile index be4deaf43..831aafc2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,4 +6,4 @@ COPY package.json /usr/src/app/ RUN npm install COPY . /usr/src/app EXPOSE 8080 -CMD [ "npm", "run", "start-aws"] +CMD [ "./node_modules/pm2/bin/pm2", "start", "./pm2-prod-${AWS_SERVICE_NAME}.yml"] diff --git a/Procfile b/Procfile deleted file mode 100644 index bc443adf9..000000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: node packages/nylas-api/app.js -worker: node packages/nylas-sync/app.js diff --git a/Procfile.dev b/Procfile.dev deleted file mode 100644 index 03ec16455..000000000 --- a/Procfile.dev +++ /dev/null @@ -1,5 +0,0 @@ -redis: redis-server -api: node packages/nylas-api/app.js -sync: node packages/nylas-sync/app.js -processor: node packages/nylas-message-processor/app.js -dashboard: node packages/nylas-dashboard/app.js diff --git a/package.json b/package.json index 4b3ab2a8b..ad44dd6d0 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "dependencies": { "bluebird": "3.x.x", "mysql": "^2.11.1", + "newrelic": "^1.28.1", + "pm2": "^1.1.3", "redis": "2.x.x", "rx": "4.x.x", "sequelize": "3.x.x", @@ -23,7 +25,7 @@ "lerna": "2.0.0-beta.23" }, "scripts": { - "start": "foreman start -f Procfile.dev", + "start": "./node_modules/pm2/bin/pm2 start ./pm2-dev.yml --no-daemon", "start-aws": "node start-aws.js", "postinstall": "node_modules/.bin/lerna bootstrap" }, diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index d7459458a..b37fe81cf 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -43,7 +43,7 @@ class DatabaseConnector { }); } - return Sequelize(dbname, '', '', { + return new Sequelize(dbname, '', '', { storage: path.join(STORAGE_DIR, `${dbname}.sqlite`), dialect: "sqlite", logging: false, diff --git a/pm2-dev.yml b/pm2-dev.yml new file mode 100644 index 000000000..9c3d520ce --- /dev/null +++ b/pm2-dev.yml @@ -0,0 +1,24 @@ +apps: + - script : packages/nylas-api/app.js + name : api + env : + DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" + DB_ENCRYPTION_PASSWORD : "d6F3Efeq" + GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" + GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF" + GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" + - script : packages/nylas-sync/app.js + name : sync + env : + DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" + DB_ENCRYPTION_PASSWORD : "d6F3Efeq" + - script : packages/nylas-dashboard/app.js + name : dashboard + env : + DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" + DB_ENCRYPTION_PASSWORD : "d6F3Efeq" + - script : packages/nylas-message-processor/app.js + name : processor + env : + DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" + DB_ENCRYPTION_PASSWORD : "d6F3Efeq" diff --git a/pm2-prod-api.yml b/pm2-prod-api.yml new file mode 100644 index 000000000..d9bc01862 --- /dev/null +++ b/pm2-prod-api.yml @@ -0,0 +1,9 @@ +apps: + - script : packages/nylas-api/app.js + name : api + instances: 0 + exec_mode: cluster + - script : packages/nylas-dashboard/app.js + name : dashboard + instances: 1 + exec_mode: cluster diff --git a/pm2-prod-sync.yml b/pm2-prod-sync.yml new file mode 100644 index 000000000..1bcb4cfb4 --- /dev/null +++ b/pm2-prod-sync.yml @@ -0,0 +1,9 @@ +apps: + - script : packages/nylas-sync/app.js + name : sync + instances: 0 + exec_mode: fork + - script : packages/nylas-message-processor/app.js + name : processor + instances: 0 + exec_mode: fork diff --git a/start-aws.js b/start-aws.js deleted file mode 100644 index b08e1d0f0..000000000 --- a/start-aws.js +++ /dev/null @@ -1,6 +0,0 @@ -const proc = require('child_process'); - -// TODO: Swtich on env variables -proc.spawn('node', ['packages/nylas-api/app.js'], { - stdio: 'inherit' -}) From 0972592aa5729bb841de1f82061ab808e731ae73 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 16:05:44 -0700 Subject: [PATCH 185/800] Add ports to pm2-dev --- packages/nylas-api/app.js | 2 +- packages/nylas-dashboard/app.js | 3 +-- pm2-dev.yml | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index 68ebaa377..bca30c9af 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -18,7 +18,7 @@ const server = new Hapi.Server({ }, }); -server.connection({ port: process.env.PORT || 5100 }); +server.connection({ port: process.env.PORT }); const plugins = [Inert, Vision, HapiBasicAuth, HapiBoom, { register: HapiSwagger, diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js index d0912330c..f29e78121 100644 --- a/packages/nylas-dashboard/app.js +++ b/packages/nylas-dashboard/app.js @@ -8,8 +8,7 @@ const path = require('path'); global.Promise = require('bluebird'); const server = new Hapi.Server(); -server.connection({ port: process.env.PORT / 1 + 1 || 5101 }); - +server.connection({ port: process.env.PORT }); const attach = (directory) => { const routesDir = path.join(__dirname, directory) diff --git a/pm2-dev.yml b/pm2-dev.yml index 9c3d520ce..6a0e8768f 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -2,6 +2,7 @@ apps: - script : packages/nylas-api/app.js name : api env : + PORT: 5100 DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" @@ -15,6 +16,7 @@ apps: - script : packages/nylas-dashboard/app.js name : dashboard env : + PORT: 5101 DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - script : packages/nylas-message-processor/app.js From f2418cef5746dcba43733bc773707f5b8799bf4e Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 7 Jul 2016 16:06:04 -0700 Subject: [PATCH 186/800] Minor updates to readme --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0e42d475d..d4c47a0f3 100644 --- a/README.md +++ b/README.md @@ -5,24 +5,24 @@ ## New Computer (Mac): 1. Install [Homebrew](http://brew.sh/) -1. Install [VirtualBox 5+](https://www.virtualbox.org/wiki/Downloads) -1. Install [Docker for Mac](https://docs.docker.com/docker-for-mac/) -1. Install [NVM](https://github.com/creationix/nvm) `brew install nvm` -1. Install Node 6+ via NVM: `nvm install 6` +2. Install [VirtualBox 5+](https://www.virtualbox.org/wiki/Downloads) +3. Install [Docker for Mac](https://docs.docker.com/docker-for-mac/) +4. Install [NVM](https://github.com/creationix/nvm) `brew install nvm` +5. Install Node 6+ via NVM: `nvm install 6` ## New to AWS: -1. Install [Elastic Beanstalk CLI](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html#eb-cli3-install-osx): `brew install awsebcli` -1. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli` +1. Install [Elastic Beanstalk CLI](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html#eb-cli3-install-osx): `sudo pip install awsebcli` +2. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli` 1. Add your AWS IAM Security Credentials to `aws configure`. 1. These are at Console Home -> IAM -> Users -> {{Your Name}} -> Security Credentials. Note that your private key was only shown unpon creation. If you've lost your private key you have to deactivate your old key and create a new one. -1. Get the K2 team private SSH key. (Ignore this when we have a Bastion Host). Ask someone on K2 for a copy of the private SSH key. Copy it to your ~/.ssh folder. +3. Get the K2 team private SSH key. (Ignore this when we have a Bastion Host). Ask someone on K2 for a copy of the private SSH key. Copy it to your ~/.ssh folder. 1. `chmod 400 ~/.ssh/k2-keypair.pem` 1. `ssh i ~/.ssh/k2-keypair.pem some-ec2-box-we-own.amazonaws.com` -1. Connect to Elastic Beanstalk instances: `eb init`. Select correct region. Select correct application. +4. Connect to Elastic Beanstalk instances: `eb init`. Select correct region. Select correct application. # Developing Locally: From 1ff61beec93e02e24f81cb25ea9499fb15ab5c37 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 16:08:08 -0700 Subject: [PATCH 187/800] package json fixed --- package.json | 1 - packages/nylas-message-processor/package.json | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 89c480b5e..c73ac7f1a 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ }, "scripts": { "start": "./node_modules/pm2/bin/pm2 start ./pm2-dev.yml --no-daemon", - "start-aws": "node start-aws.js", "postinstall": "node_modules/.bin/lerna bootstrap" }, "repository": { diff --git a/packages/nylas-message-processor/package.json b/packages/nylas-message-processor/package.json index 38370e3f6..74ac74f2d 100644 --- a/packages/nylas-message-processor/package.json +++ b/packages/nylas-message-processor/package.json @@ -13,7 +13,7 @@ "babel-preset-es2015": "6.9.0", "jasmine": "2.4.1", "mailparser": "0.6.0", - "mimelib": "^0.2.19", + "mimelib": "0.2.19", "nylas-core": "0.x.x" }, "devDependencies": { From 31e4d25448e2cfc1260ec814c17534d047760420 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 16:10:35 -0700 Subject: [PATCH 188/800] Add API port to prod config --- pm2-prod-api.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pm2-prod-api.yml b/pm2-prod-api.yml index d9bc01862..5a6e886ba 100644 --- a/pm2-prod-api.yml +++ b/pm2-prod-api.yml @@ -3,6 +3,8 @@ apps: name : api instances: 0 exec_mode: cluster + env : + PORT: 5100 - script : packages/nylas-dashboard/app.js name : dashboard instances: 1 From aca24ef18a74ac738aa69576c629163ad7091fe8 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Thu, 7 Jul 2016 16:32:46 -0700 Subject: [PATCH 189/800] Update dockerfile to start with env variable --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 713bb6711..7386cd58e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,4 +22,4 @@ EXPOSE 5100 # We use a start-aws command that automatically spawns the correct process # based on environment variables (which changes instance to instance) -CMD [ "./node_modules/pm2/bin/pm2", "start", "./pm2-prod-${AWS_SERVICE_NAME}.yml"] +CMD ./node_modules/pm2/bin/pm2 start --no-daemon ./pm2-prod-${AWS_SERVICE_NAME}.yml From e7affd3f6517964fd235f262c0f176941645a94b Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Thu, 7 Jul 2016 17:07:19 -0700 Subject: [PATCH 190/800] Add dashboard pm2 config --- pm2-prod-api.yml | 4 ---- pm2-prod-dashboard.yml | 7 +++++++ 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 pm2-prod-dashboard.yml diff --git a/pm2-prod-api.yml b/pm2-prod-api.yml index 5a6e886ba..e6f679b56 100644 --- a/pm2-prod-api.yml +++ b/pm2-prod-api.yml @@ -5,7 +5,3 @@ apps: exec_mode: cluster env : PORT: 5100 - - script : packages/nylas-dashboard/app.js - name : dashboard - instances: 1 - exec_mode: cluster diff --git a/pm2-prod-dashboard.yml b/pm2-prod-dashboard.yml new file mode 100644 index 000000000..5f5e4234e --- /dev/null +++ b/pm2-prod-dashboard.yml @@ -0,0 +1,7 @@ +apps: + - script : packages/nylas-dashboard/app.js + name : dashboard + instances: 0 + exec_mode: cluster + env : + PORT: 5100 From 6fecf0e40d6ced73e0fbea96c0faafabd1dd893a Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Fri, 8 Jul 2016 12:00:57 -0400 Subject: [PATCH 191/800] Add redis to local pm2 --- README.md | 1 + pm2-dev.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index d4c47a0f3..0e0e1dc9b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ 3. Install [Docker for Mac](https://docs.docker.com/docker-for-mac/) 4. Install [NVM](https://github.com/creationix/nvm) `brew install nvm` 5. Install Node 6+ via NVM: `nvm install 6` +6. Install Redis locally `brew install redis` ## New to AWS: diff --git a/pm2-dev.yml b/pm2-dev.yml index 6a0e8768f..0031cd026 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -1,4 +1,6 @@ apps: + - script : redis-server + name : redis - script : packages/nylas-api/app.js name : api env : From ab3713830c8978778a89434809e5169c057e87e3 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Fri, 8 Jul 2016 12:12:11 -0400 Subject: [PATCH 192/800] Readme fixes --- README.md | 12 ++++++++++++ pm2-dev.yml | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/README.md b/README.md index 0e0e1dc9b..83e953c62 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,16 @@ npm start ``` +We use [pm2](http://pm2.keymetrics.io/) to launch a variety of processes +(sync, api, dashboard, processor, etc). + +You can see the scripts that are running and their arguments in +`/pm2-dev.yml` + +To test to see if the basic API is up go to: `http://lvh.me:5100/ping`. You +should see `pong`. + +`lvh.me` is a DNS hack that redirects back to 127.0.0.1 with the added +benefit of letting us use subdomains. + # Deploying diff --git a/pm2-dev.yml b/pm2-dev.yml index 0031cd026..313be4c0d 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -1,6 +1,8 @@ apps: - script : redis-server name : redis + + - script : packages/nylas-api/app.js name : api env : @@ -10,17 +12,23 @@ apps: GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF" GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" + + - script : packages/nylas-sync/app.js name : sync env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" + + - script : packages/nylas-dashboard/app.js name : dashboard env : PORT: 5101 DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" + + - script : packages/nylas-message-processor/app.js name : processor env : From 17df581105e60e1e3c92d73496f0dbf9780c54a1 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 8 Jul 2016 13:49:51 -0700 Subject: [PATCH 193/800] Use TEXT vs STRING, specify string column lengths --- packages/nylas-core/database-types.js | 4 +-- packages/nylas-core/models/account/file.js | 4 +-- packages/nylas-core/models/account/message.js | 10 +++--- .../models/account/syncback-request.js | 6 ++-- packages/nylas-core/models/account/thread.js | 4 +-- .../nylas-core/models/account/transaction.js | 12 ++----- packages/nylas-core/models/model-helpers.js | 31 ------------------- packages/nylas-core/models/shared/account.js | 2 +- 8 files changed, 18 insertions(+), 55 deletions(-) delete mode 100644 packages/nylas-core/models/model-helpers.js diff --git a/packages/nylas-core/database-types.js b/packages/nylas-core/database-types.js index 4bca93136..ea9235e04 100644 --- a/packages/nylas-core/database-types.js +++ b/packages/nylas-core/database-types.js @@ -2,7 +2,7 @@ const Sequelize = require('sequelize'); module.exports = { JSONType: (fieldName, {defaultValue = '{}'} = {}) => ({ - type: Sequelize.STRING, + type: Sequelize.TEXT, defaultValue, get: function get() { return JSON.parse(this.getDataValue(fieldName)) @@ -12,7 +12,7 @@ module.exports = { }, }), JSONARRAYType: (fieldName, {defaultValue = '[]'} = {}) => ({ - type: Sequelize.STRING, + type: Sequelize.TEXT, defaultValue, get: function get() { return JSON.parse(this.getDataValue(fieldName)) diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js index 8e121ca7c..5043318e8 100644 --- a/packages/nylas-core/models/account/file.js +++ b/packages/nylas-core/models/account/file.js @@ -4,9 +4,9 @@ module.exports = (sequelize, Sequelize) => { const File = sequelize.define('file', { accountId: { type: Sequelize.STRING, allowNull: false }, version: Sequelize.INTEGER, - filename: Sequelize.STRING, + filename: Sequelize.STRING(500), partId: Sequelize.STRING, - contentType: Sequelize.STRING, + contentType: Sequelize.STRING(500), size: Sequelize.INTEGER, }, { classMethods: { diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js index 0f39e7673..80bf1b5b5 100644 --- a/packages/nylas-core/models/account/message.js +++ b/packages/nylas-core/models/account/message.js @@ -8,11 +8,11 @@ module.exports = (sequelize, Sequelize) => { accountId: { type: Sequelize.STRING, allowNull: false }, version: Sequelize.INTEGER, headerMessageId: Sequelize.STRING, - body: Sequelize.STRING, + body: Sequelize.TEXT, headers: JSONType('headers'), - subject: Sequelize.STRING, - snippet: Sequelize.STRING, - hash: Sequelize.STRING, + subject: Sequelize.STRING(500), + snippet: Sequelize.STRING(255), + hash: Sequelize.STRING(65), date: Sequelize.DATE, unread: Sequelize.BOOLEAN, starred: Sequelize.BOOLEAN, @@ -23,7 +23,7 @@ module.exports = (sequelize, Sequelize) => { bcc: JSONARRAYType('bcc'), replyTo: JSONARRAYType('replyTo'), folderImapUID: { type: Sequelize.STRING, allowNull: true}, - folderImapXGMLabels: { type: Sequelize.STRING, allowNull: true}, + folderImapXGMLabels: { type: Sequelize.TEXT, allowNull: true}, }, { indexes: [ { diff --git a/packages/nylas-core/models/account/syncback-request.js b/packages/nylas-core/models/account/syncback-request.js index 5f1069344..a9c67afc2 100644 --- a/packages/nylas-core/models/account/syncback-request.js +++ b/packages/nylas-core/models/account/syncback-request.js @@ -1,4 +1,4 @@ -const {typeJSON} = require('../model-helpers') +const {JSONType} = require('../../database-types'); module.exports = (sequelize, Sequelize) => { const SyncbackRequest = sequelize.define('syncbackRequest', { @@ -8,8 +8,8 @@ module.exports = (sequelize, Sequelize) => { defaultValue: "NEW", allowNull: false, }, - error: typeJSON('error'), - props: typeJSON('props'), + error: JSONType('error'), + props: JSONType('props'), }); return SyncbackRequest; diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js index 2181fae68..0b77cbd1f 100644 --- a/packages/nylas-core/models/account/thread.js +++ b/packages/nylas-core/models/account/thread.js @@ -5,8 +5,8 @@ module.exports = (sequelize, Sequelize) => { accountId: { type: Sequelize.STRING, allowNull: false }, version: Sequelize.INTEGER, threadId: Sequelize.STRING, - subject: Sequelize.STRING, - snippet: Sequelize.STRING, + subject: Sequelize.STRING(500), + snippet: Sequelize.STRING(255), unreadCount: Sequelize.INTEGER, starredCount: Sequelize.INTEGER, firstMessageDate: Sequelize.DATE, diff --git a/packages/nylas-core/models/account/transaction.js b/packages/nylas-core/models/account/transaction.js index 044659b6d..b089c7cdb 100644 --- a/packages/nylas-core/models/account/transaction.js +++ b/packages/nylas-core/models/account/transaction.js @@ -1,17 +1,11 @@ +const {JSONARRAYType} = require('../../database-types'); + module.exports = (sequelize, Sequelize) => { const Transaction = sequelize.define('transaction', { type: Sequelize.STRING, objectId: Sequelize.STRING, modelName: Sequelize.STRING, - changedFields: { - type: Sequelize.STRING, - get: function get() { - return JSON.parse(this.getDataValue('changedFields')) - }, - set: function set(val) { - this.setDataValue('changedFields', JSON.stringify(val)); - }, - }, + changedFields: JSONARRAYType('changedFields'), }); return Transaction; diff --git a/packages/nylas-core/models/model-helpers.js b/packages/nylas-core/models/model-helpers.js deleted file mode 100644 index 911e87023..000000000 --- a/packages/nylas-core/models/model-helpers.js +++ /dev/null @@ -1,31 +0,0 @@ -const Sequelize = require('sequelize'); - -module.exports = { - typeJSON: function typeJSON(key) { - return { - type: Sequelize.STRING, - get: function get() { - const val = this.getDataValue(key); - if (typeof val === 'string') { - try { - return JSON.parse(val) - } catch (e) { - return val - } - } - return val - }, - set: function set(val) { - let valToSet = val - if (typeof val !== 'string') { - try { - valToSet = JSON.stringify(val) - } catch (e) { - valToSet = val; - } - } - return this.setDataValue(key, valToSet) - }, - } - }, -} diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index 29a845ac9..e5b203666 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -9,7 +9,7 @@ module.exports = (sequelize, Sequelize) => { provider: Sequelize.STRING, emailAddress: Sequelize.STRING, connectionSettings: JSONType('connectionSettings'), - connectionCredentials: Sequelize.STRING, + connectionCredentials: Sequelize.TEXT, syncPolicy: JSONType('syncPolicy'), syncError: JSONType('syncError', {defaultValue: null}), firstSyncCompletedAt: Sequelize.INTEGER, From 3e348252ab53295a58eea71e0189d7ff55b42d6f Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 8 Jul 2016 14:41:53 -0700 Subject: [PATCH 194/800] =?UTF-8?q?Fix=20issue=20with=20default=20values?= =?UTF-8?q?=20=E2=80=94=20just=20implement=20in=20getters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/nylas-core/database-types.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/nylas-core/database-types.js b/packages/nylas-core/database-types.js index ea9235e04..8216e8d0f 100644 --- a/packages/nylas-core/database-types.js +++ b/packages/nylas-core/database-types.js @@ -1,21 +1,23 @@ const Sequelize = require('sequelize'); module.exports = { - JSONType: (fieldName, {defaultValue = '{}'} = {}) => ({ + JSONType: (fieldName, {defaultValue = {}} = {}) => ({ type: Sequelize.TEXT, - defaultValue, get: function get() { - return JSON.parse(this.getDataValue(fieldName)) + const val = this.getDataValue(fieldName); + if (!val) { return defaultValue } + return JSON.parse(val); }, set: function set(val) { this.setDataValue(fieldName, JSON.stringify(val)); }, }), - JSONARRAYType: (fieldName, {defaultValue = '[]'} = {}) => ({ + JSONARRAYType: (fieldName, {defaultValue = []} = {}) => ({ type: Sequelize.TEXT, - defaultValue, get: function get() { - return JSON.parse(this.getDataValue(fieldName)) + const val = this.getDataValue(fieldName); + if (!val) { return defaultValue } + return JSON.parse(val); }, set: function set(val) { this.setDataValue(fieldName, JSON.stringify(val)); From 53a2e7e56b8b27478b3dab96aa1d838ad0516a53 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 8 Jul 2016 14:55:12 -0700 Subject: [PATCH 195/800] Add ping to dashboard, wait to get shared db --- packages/nylas-dashboard/app.js | 84 +++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js index f29e78121..2ee772b4a 100644 --- a/packages/nylas-dashboard/app.js +++ b/packages/nylas-dashboard/app.js @@ -20,18 +20,18 @@ const attach = (directory) => { }); } -DatabaseConnector.forShared().then(({Account}) => { - server.register([HapiWebSocket, Inert], () => { - attach('./routes/') +server.register([HapiWebSocket, Inert], () => { + attach('./routes/') - server.route({ - method: "POST", - path: "/accounts", - config: { - plugins: { - websocket: { - only: true, - connect: (wss, ws) => { + server.route({ + method: "POST", + path: "/accounts", + config: { + plugins: { + websocket: { + only: true, + connect: (wss, ws) => { + DatabaseConnector.forShared().then(({Account}) => { Account.findAll().then((accounts) => { accounts.forEach((acct) => { ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct })); @@ -57,35 +57,47 @@ DatabaseConnector.forShared().then(({Account}) => { ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments})) ) }, 1000); - }, - disconnect: () => { - clearInterval(this.pollInterval); - this.observable.dispose(); - }, + }); + }, + disconnect: () => { + clearInterval(this.pollInterval); + this.observable.dispose(); }, }, }, - handler: (request, reply) => { - if (request.payload.cmd === "PING") { - reply(JSON.stringify({ result: "PONG" })); - return; - } - }, - }); + }, + handler: (request, reply) => { + if (request.payload.cmd === "PING") { + reply(JSON.stringify({ result: "PONG" })); + return; + } + }, + }); - server.route({ - method: 'GET', - path: '/{param*}', - handler: { - directory: { - path: require('path').join(__dirname, 'public'), - }, - }, - }); + server.route({ + method: 'GET', + path: '/ping', + config: { + auth: false, + }, + handler: (request, reply) => { + console.log("---> Ping!") + reply("pong") + }, + }); - server.start((startErr) => { - if (startErr) { throw startErr; } - console.log('Dashboard running at:', server.info.uri); - }); + server.route({ + method: 'GET', + path: '/{param*}', + handler: { + directory: { + path: require('path').join(__dirname, 'public'), + }, + }, + }); + + server.start((startErr) => { + if (startErr) { throw startErr; } + console.log('Dashboard running at:', server.info.uri); }); }); From ab8f1b4fbf50188dbaa6dd6d05be530b0e0b7c5d Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Fri, 8 Jul 2016 18:33:37 -0400 Subject: [PATCH 196/800] Silent transactions if only syncState changes --- packages/nylas-core/hook-transaction-log.js | 24 ++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/nylas-core/hook-transaction-log.js b/packages/nylas-core/hook-transaction-log.js index e6157999c..537ce51ab 100644 --- a/packages/nylas-core/hook-transaction-log.js +++ b/packages/nylas-core/hook-transaction-log.js @@ -9,13 +9,31 @@ module.exports = (db, sequelize) => { } } - const isTransaction = ({$modelOptions}) => { - return $modelOptions.name.singular === "transaction" + const isSilent = (data) => { + data._previousDataValues + data._changed + + if (data.$modelOptions.name.singular === "transaction") { + return true + } + + if (data._changed && data._changed.syncState) { + for (const key of Object.keys(data._changed)) { + if (key === "syncState") { continue } + if (data._changed[key] !== data._previousDataValues[key]) { + // SyncState changed, but so did something else + return false; + } + } + // Be silent due to nothing but sync state changing + return true; + } } const transactionLogger = (type) => { return (sequelizeHookData) => { - if (isTransaction(sequelizeHookData)) return; + if (isSilent(sequelizeHookData)) return; + const transactionData = Object.assign({type: type}, parseHookData(sequelizeHookData) ); From 8c122a57a32fe17937fe615ff13db513ea9281af Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 8 Jul 2016 15:36:50 -0700 Subject: [PATCH 197/800] Stop syncing in a hard loop --- packages/nylas-sync/sync-worker.js | 31 ++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 67c69a719..c5162be59 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -29,7 +29,7 @@ class SyncWorker { this._expirationTimer = null; this._destroyed = false; - this.syncNow(); + this.syncNow({reason: 'Initial'}); this._onMessage = this._onMessage.bind(this); this._listener = PubsubConnector.observeAccount(account.id).subscribe(this._onMessage) @@ -54,22 +54,23 @@ class SyncWorker { case MessageTypes.ACCOUNT_UPDATED: this._onAccountUpdated(); break; case MessageTypes.SYNCBACK_REQUESTED: - this.syncNow(); break; + this.syncNow({reason: 'Syncback Action Queued'}); break; default: throw new Error(`Invalid message: ${msg}`) } } _onAccountUpdated() { - console.log("SyncWorker: Detected change to account. Reloading and syncing now."); - this._getAccount().then((account) => { - this._account = account; - this.syncNow(); - }) + if (this.isNextSyncScheduled()) { + this._getAccount().then((account) => { + this._account = account; + this.syncNow({reason: 'Account Modification'}); + }); + } } _onConnectionIdleUpdate() { - this.syncNow(); + this.syncNow({reason: 'IMAP IDLE Fired'}); } _getAccount() { @@ -151,13 +152,15 @@ class SyncWorker { .then(() => this.syncAllCategories()) } - syncNow() { + syncNow({reason} = {}) { clearTimeout(this._syncTimer); + this._syncTimer = null; if (!process.env.SYNC_AFTER_ERRORS && this._account.errored()) { - console.log(`SyncWorker: Account ${this._account.emailAddress} is in error state - Skipping sync`) + console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) is in error state - Skipping sync`) return } + console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) sync started (${reason})`) this.ensureConnection() .then(() => this._account.update({syncError: null})) @@ -171,7 +174,7 @@ class SyncWorker { } onSyncError(error) { - console.error(`SyncWorker: Error while syncing account ${this._account.emailAddress} `, error) + console.error(`SyncWorker: Error while syncing account ${this._account.emailAddress} (${this._account.id})`, error) this.closeConnection() if (error.source === 'socket') { @@ -216,6 +219,10 @@ class SyncWorker { throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) } + isNextSyncScheduled() { + return this._syncTimer != null; + } + scheduleNextSync() { if (Date.now() - this._startTime > CLAIM_DURATION) { console.log("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool."); @@ -232,7 +239,7 @@ class SyncWorker { const target = this._lastSyncTime + interval; console.log(`SyncWorker: Account ${active ? 'active' : 'inactive'}. Next sync scheduled for ${new Date(target).toLocaleString()}`); this._syncTimer = setTimeout(() => { - this.syncNow(); + this.syncNow({reason: 'Scheduled'}); }, target - Date.now()); } }); From 016bad67b949cd3628409c675fd5e3f960b6f41d Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 8 Jul 2016 16:53:01 -0700 Subject: [PATCH 198/800] Temporarily patch /account to not check schema --- packages/nylas-api/serialization.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 77175001b..5d6a1c07c 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -6,23 +6,25 @@ function replacer(key, value) { } function jsonSchema(modelName) { - const models = ['Message', 'Thread', 'File', 'Error', 'SyncbackRequest'] + const models = ['Message', 'Thread', 'File', 'Error', 'SyncbackRequest', 'Account'] if (models.includes(modelName)) { return Joi.object(); } if (modelName === 'Account') { - return Joi.object().keys({ - id: Joi.number(), - object: Joi.string(), - email_address: Joi.string(), - provider: Joi.string(), - organization_unit: Joi.string(), - connection_settings: Joi.object(), - sync_policy: Joi.object(), - sync_error: Joi.object().allow(null), - first_sync_completed_at: Joi.number().allow(null), - last_sync_completions: Joi.array(), - }) + // Ben: Disabled temporarily because folks keep changing the keys and it's hard + // to keep in sync. Might need to consider another approach to these. + // return Joi.object().keys({ + // id: Joi.number(), + // object: Joi.string(), + // email_address: Joi.string(), + // provider: Joi.string(), + // organization_unit: Joi.string(), + // connection_settings: Joi.object(), + // sync_policy: Joi.object(), + // sync_error: Joi.object().allow(null), + // first_sync_completed_at: Joi.number().allow(null), + // last_sync_completions: Joi.array(), + // }) } if (modelName === 'Folder') { return Joi.object().keys({ From 14b5bef0a74ebc78a11f58e9cb87b25a66c7007f Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Fri, 8 Jul 2016 16:59:00 -0700 Subject: [PATCH 199/800] Fix error handling on connection close --- packages/nylas-core/imap-connection.js | 50 ++++++++++++++++---------- packages/nylas-sync/sync-worker.js | 33 ++++++++--------- 2 files changed, 49 insertions(+), 34 deletions(-) diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index cda68a70a..ac1eedb73 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -4,6 +4,14 @@ const _ = require('underscore'); const xoauth2 = require('xoauth2'); const EventEmitter = require('events'); +class IMAPConnectionNotReadyError extends Error { + constructor(funcName) { + super(`${funcName} - You must call connect() first.`); + + // hack so that the error matches the ones used by node-imap + this.source = 'socket'; + } +} class IMAPBox { @@ -123,28 +131,28 @@ class IMAPBox { addFlags(range, flags) { if (!this._imap) { - throw new Error(`IMAPBox::addFlags - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPBox::addFlags`) } return this._imap.addFlagsAsync(range, flags) } delFlags(range, flags) { if (!this._imap) { - throw new Error(`IMAPBox::delFlags - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPBox::delFlags`) } return this._imap.delFlagsAsync(range, flags) } moveFromBox(range, folderName) { if (!this._imap) { - throw new Error(`IMAPBox::moveFromBox - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPBox::moveFromBox`) } return this._imap.moveAsync(range, folderName) } closeBox({expunge = true} = {}) { if (!this._imap) { - throw new Error(`IMAPBox::closeBox - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPBox::closeBox`) } return this._imap.closeBoxAsync(expunge) } @@ -174,7 +182,8 @@ class IMAPConnection extends EventEmitter { this._queue = []; this._currentOperation = null; this._settings = settings; - this._imap = null + this._imap = null; + this._connectPromise = null; } connect() { @@ -222,7 +231,9 @@ class IMAPConnection extends EventEmitter { this._imap = Promise.promisifyAll(new Imap(settings)); this._imap.once('end', () => { - console.log('Connection ended'); + console.log('Underlying IMAP Connection ended'); + this._connectPromise = null; + this._imap = null; }); this._imap.on('alert', (msg) => { @@ -255,11 +266,12 @@ class IMAPConnection extends EventEmitter { this._queue = []; this._imap.end(); this._imap = null; + this._connectPromise = null; } serverSupports(capability) { if (!this._imap) { - throw new Error(`IMAPConnection::serverSupports - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::serverSupports`) } this._imap.serverSupports(capability); } @@ -269,7 +281,7 @@ class IMAPConnection extends EventEmitter { */ openBox(folderName, {readOnly = false} = {}) { if (!this._imap) { - throw new Error(`IMAPConnection::openBox - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::openBox`) } return this._imap.openBoxAsync(folderName, readOnly).then((box) => new IMAPBox(this._imap, box) @@ -278,35 +290,35 @@ class IMAPConnection extends EventEmitter { getBoxes() { if (!this._imap) { - throw new Error(`IMAPConnection::getBoxes - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::getBoxes`) } return this._imap.getBoxesAsync() } addBox(folderName) { if (!this._imap) { - throw new Error(`IMAPConnection::addBox - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::addBox`) } return this._imap.addBoxAsync(folderName) } renameBox(oldFolderName, newFolderName) { if (!this._imap) { - throw new Error(`IMAPConnection::renameBox - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::renameBox`) } return this._imap.renameBoxAsync(oldFolderName, newFolderName) } delBox(folderName) { if (!this._imap) { - throw new Error(`IMAPConnection::delBox - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::delBox`) } return this._imap.delBoxAsync(folderName) } runOperation(operation) { if (!this._imap) { - throw new Error(`IMAPConnection::runOperation - You need to call connect() first.`) + throw new IMAPConnectionNotReadyError(`IMAPConnection::runOperation`) } return new Promise((resolve, reject) => { this._queue.push({operation, resolve, reject}); @@ -317,11 +329,13 @@ class IMAPConnection extends EventEmitter { } processNextOperation() { - if (this._currentOperation) { return } + if (this._currentOperation) { + return; + } this._currentOperation = this._queue.shift(); if (!this._currentOperation) { this.emit('queue-empty'); - return + return; } const {operation, resolve, reject} = this._currentOperation; @@ -329,8 +343,8 @@ class IMAPConnection extends EventEmitter { if (result instanceof Promise === false) { reject(new Error(`Expected ${operation.constructor.name} to return promise.`)) } - result - .then(() => { + + result.then(() => { this._currentOperation = null; console.log(`Finished task: ${operation.description()}`) resolve(); @@ -344,6 +358,6 @@ class IMAPConnection extends EventEmitter { }) } } -IMAPConnection.Capabilities = Capabilities; +IMAPConnection.Capabilities = Capabilities; module.exports = IMAPConnection diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index c5162be59..6ebb34b8d 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -45,7 +45,6 @@ class SyncWorker { if (this._conn) { this._conn.end(); } - this._conn = null } _onMessage(msg) { @@ -61,15 +60,19 @@ class SyncWorker { } _onAccountUpdated() { - if (this.isNextSyncScheduled()) { - this._getAccount().then((account) => { - this._account = account; - this.syncNow({reason: 'Account Modification'}); - }); + if (!this.isWaitingForNextSync()) { + return; } + this._getAccount().then((account) => { + this._account = account; + this.syncNow({reason: 'Account Modification'}); + }); } _onConnectionIdleUpdate() { + if (!this.isWaitingForNextSync()) { + return; + } this.syncNow({reason: 'IMAP IDLE Fired'}); } @@ -127,7 +130,8 @@ class SyncWorker { .catch((error) => { syncbackRequest.error = error syncbackRequest.status = "FAILED" - }).finally(() => syncbackRequest.save()) + }) + .finally(() => syncbackRequest.save()) } syncAllCategories() { @@ -146,12 +150,6 @@ class SyncWorker { }); } - performSync() { - return this.syncbackMessageActions() - .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider))) - .then(() => this.syncAllCategories()) - } - syncNow({reason} = {}) { clearTimeout(this._syncTimer); this._syncTimer = null; @@ -164,7 +162,9 @@ class SyncWorker { this.ensureConnection() .then(() => this._account.update({syncError: null})) - .then(() => this.performSync()) + .then(() => this.syncbackMessageActions()) + .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider))) + .then(() => this.syncAllCategories()) .then(() => this.onSyncDidComplete()) .catch((error) => this.onSyncError(error)) .finally(() => { @@ -177,10 +177,11 @@ class SyncWorker { console.error(`SyncWorker: Error while syncing account ${this._account.emailAddress} (${this._account.id})`, error) this.closeConnection() - if (error.source === 'socket') { + if (error.source.includes('socket') || error.source.includes('timeout')) { // Continue to retry if it was a network error return Promise.resolve() } + this._account.syncError = jsonError(error) return this._account.save() } @@ -219,7 +220,7 @@ class SyncWorker { throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) } - isNextSyncScheduled() { + isWaitingForNextSync() { return this._syncTimer != null; } From 2a776e6cffe4fcd52b727f8c99f97a1a127ca93c Mon Sep 17 00:00:00 2001 From: Annie Date: Fri, 8 Jul 2016 17:06:21 -0700 Subject: [PATCH 200/800] Build(contacts): built contact model, added routes and processor --- packages/nylas-api/routes/contacts.js | 83 +++++++++++++++++++ packages/nylas-api/serialization.js | 2 +- packages/nylas-core/models/account/contact.js | 22 +++++ .../processors/contact.js | 53 ++++++++++++ 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 packages/nylas-api/routes/contacts.js create mode 100644 packages/nylas-core/models/account/contact.js create mode 100644 packages/nylas-message-processor/processors/contact.js diff --git a/packages/nylas-api/routes/contacts.js b/packages/nylas-api/routes/contacts.js new file mode 100644 index 000000000..778f348cf --- /dev/null +++ b/packages/nylas-api/routes/contacts.js @@ -0,0 +1,83 @@ +const Joi = require('joi'); +const Serialization = require('../serialization'); + +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/contacts', + config: { + description: 'Returns an array of contacts', + notes: 'Notes go here', + tags: ['contacts'], + validate: { + query: { + name: Joi.string(), + email: Joi.string().email(), + limit: Joi.number().integer().min(1).max(2000).default(100), + offset: Joi.number().integer().min(0).default(0), + }, + }, + response: { + schema: Joi.array().items( + Serialization.jsonSchema('Contact') + ), + }, + }, + handler: (request, reply) => { + request.getAccountDatabase().then((db) => { + const {Contact} = db; + const query = request.query; + const where = {}; + + if (query.name) { + where.name = {like: query.name}; + } + if (query.email) { + where.email = query.email; + } + + Contact.findAll({ + where: where, + limit: request.query.limit, + offset: request.query.offset, + }).then((contacts) => { + reply(Serialization.jsonStringify(contacts)) + }) + }) + }, + }) + + server.route({ + method: 'GET', + path: '/contacts/{id}', + config: { + description: 'Returns a contact with specified id.', + notes: 'Notes go here', + tags: ['contacts'], + validate: { + params: { + id: Joi.string(), + }, + }, + response: { + schema: Serialization.jsonSchema('Contact'), + }, + }, + handler: (request, reply) => { + request.getAccountDatabase().then(({Contact}) => { + const {params: {id}} = request + + Contact.findOne({where: {id}}).then((contact) => { + if (!contact) { + return reply.notFound(`Contact ${id} not found`) + } + return reply(Serialization.jsonStringify(contact)) + }) + .catch((error) => { + console.log('Error fetching contacts: ', error) + reply(error) + }) + }) + }, + }) +} diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 77175001b..11b9c9a4b 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -6,7 +6,7 @@ function replacer(key, value) { } function jsonSchema(modelName) { - const models = ['Message', 'Thread', 'File', 'Error', 'SyncbackRequest'] + const models = ['Message', 'Thread', 'File', 'Error', 'SyncbackRequest', 'Contact'] if (models.includes(modelName)) { return Joi.object(); } diff --git a/packages/nylas-core/models/account/contact.js b/packages/nylas-core/models/account/contact.js new file mode 100644 index 000000000..243787a64 --- /dev/null +++ b/packages/nylas-core/models/account/contact.js @@ -0,0 +1,22 @@ +module.exports = (sequelize, Sequelize) => { + const Contact = sequelize.define('contact', { + accountId: { type: Sequelize.STRING, allowNull: false }, + version: Sequelize.INTEGER, + name: Sequelize.STRING, + email: Sequelize.STRING, + }, { + instanceMethods: { + toJSON: function toJSON() { + return { + id: this.id, + account_id: this.accountId, + object: 'contact', + email: this.email, + name: this.name, + } + }, + }, + }) + + return Contact; +} diff --git a/packages/nylas-message-processor/processors/contact.js b/packages/nylas-message-processor/processors/contact.js new file mode 100644 index 000000000..11d1423e1 --- /dev/null +++ b/packages/nylas-message-processor/processors/contact.js @@ -0,0 +1,53 @@ + +class ContactProcessor { + + verified(contact) { + // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages + const regex = new RegExp(/^(noreply|no-reply|donotreply|mailer|support|webmaster|news(letter)?@)/ig) + + if (regex.test(contact.email) || contact.email.length > 60) { + console.log('Email address doesn\'t seem to be areal person') + return false + } + return true + } + + emptyContact(Contact, options = {}, accountId) { + options.accountId = accountId + return Contact.create(options) + } + + findOrCreateByContactId(Contact, contact, accountId) { + return Contact.find({where: {email: contact.email}}) + .then((contactDb) => { + return contactDb || this.emptyContact(Contact, contact, accountId) + }) + } + + + processMessage({db, message}) { + const {Contact} = db; + + let allContacts = [] + const fields = ['to', 'from', 'bcc', 'cc'] + fields.forEach((field) => { + allContacts = allContacts.concat(message[field]) + }) + const filtered = allContacts.filter(this.verified) + const contactPromises = filtered.map((contact) => { + return this.findOrCreateByContactId(Contact, contact, message.accountId) + }) + + return Promise.all(contactPromises) + .then(() => { + return message + }) + } +} + +const processor = new ContactProcessor() + +module.exports = { + order: 3, + processMessage: processor.processMessage.bind(processor), +} From dce872fac8166f8102d197ef86bafb67395f1797 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Fri, 8 Jul 2016 17:13:30 -0700 Subject: [PATCH 201/800] Adds bunyan for json logging on every package! - Bunyan logs json output, and added a stream to send our logs to cloudwatch - Replaces /all/ instances of console.log. Turned eslint rule back on, so we don't use console.log ever again. - Added npm scripts to view pretty logs --- .eslintrc | 1 - README.md | 4 +- package.json | 8 ++- packages/nylas-api/app.js | 6 +- packages/nylas-api/decorators/connections.js | 3 + packages/nylas-api/routes/auth.js | 4 +- packages/nylas-api/routes/files.js | 17 +++--- packages/nylas-api/routes/messages.js | 9 +-- packages/nylas-api/routes/ping.js | 4 +- packages/nylas-core/imap-connection.js | 19 ++++-- packages/nylas-core/index.js | 1 + packages/nylas-core/logger.js | 52 ++++++++++++++++ packages/nylas-core/models/account/file.js | 4 +- packages/nylas-core/models/account/message.js | 4 +- packages/nylas-core/package.json | 1 + packages/nylas-core/pubsub-connector.js | 4 +- packages/nylas-core/scheduler-utils.js | 8 ++- packages/nylas-dashboard/app.js | 5 +- packages/nylas-message-processor/app.js | 29 ++++----- .../processors/parsing.js | 4 +- .../processors/threading.js | 11 +++- .../spec/threading-spec.js | 1 - packages/nylas-sync/app.js | 9 +-- ...-category-list.js => fetch-folder-list.js} | 3 +- ...ategory.js => fetch-messages-in-folder.js} | 60 +++++++++++++------ packages/nylas-sync/sync-process-manager.js | 18 +++--- packages/nylas-sync/sync-worker.js | 28 +++++---- pm2-dev.yml | 10 ++-- 28 files changed, 216 insertions(+), 111 deletions(-) create mode 100644 packages/nylas-core/logger.js rename packages/nylas-sync/imap/{fetch-category-list.js => fetch-folder-list.js} (97%) rename packages/nylas-sync/imap/{fetch-messages-in-category.js => fetch-messages-in-folder.js} (84%) diff --git a/.eslintrc b/.eslintrc index 1128872ff..d712529ce 100644 --- a/.eslintrc +++ b/.eslintrc @@ -17,7 +17,6 @@ "object-curly-spacing": "off", "max-len": "off", "new-cap": ["error", {"capIsNew": false}], - "no-console": "off", "no-constant-condition": "off", "no-loop-func": "off", "no-shadow": "error", diff --git a/README.md b/README.md index 83e953c62..76f21b673 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,9 @@ # Developing Locally: ``` -npm start +npm run start +npm run logs +npm run stop ``` We use [pm2](http://pm2.keymetrics.io/) to launch a variety of processes diff --git a/package.json b/package.json index c73ac7f1a..411c58799 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,8 @@ "main": "", "dependencies": { "bluebird": "3.x.x", + "bunyan": "^1.8.1", + "bunyan-cloudwatch": "^2.0.0", "lerna": "2.0.0-beta.23", "mysql": "^2.11.1", "newrelic": "^1.28.1", @@ -25,8 +27,10 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "./node_modules/pm2/bin/pm2 start ./pm2-dev.yml --no-daemon", - "postinstall": "node_modules/.bin/lerna bootstrap" + "start": "pm2 start ./pm2-dev.yml", + "logs": "pm2 logs --raw | bunyan -o short", + "stop": "pm2 delete all", + "postinstall": "lerna bootstrap" }, "repository": { "type": "git", diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index bca30c9af..d6fab7358 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -7,8 +7,10 @@ const Vision = require('vision'); const Package = require('./package'); const fs = require('fs'); const path = require('path'); +const {DatabaseConnector, SchedulerUtils, Logger} = require(`nylas-core`); global.Promise = require('bluebird'); +global.Logger = Logger.createLogger('nylas-k2-api') const server = new Hapi.Server({ connections: { @@ -33,8 +35,6 @@ const plugins = [Inert, Vision, HapiBasicAuth, HapiBoom, { let sharedDb = null; const validate = (request, username, password, callback) => { - const {DatabaseConnector, SchedulerUtils} = require(`nylas-core`); - let getSharedDb = null; if (sharedDb) { getSharedDb = Promise.resolve(sharedDb) @@ -88,6 +88,6 @@ server.register(plugins, (err) => { server.start((startErr) => { if (startErr) { throw startErr; } - console.log('API running at:', server.info.uri); + global.Logger.info({url: server.info.uri}, 'API running'); }); }); diff --git a/packages/nylas-api/decorators/connections.js b/packages/nylas-api/decorators/connections.js index 2003696cf..f0fe1d437 100644 --- a/packages/nylas-api/decorators/connections.js +++ b/packages/nylas-api/decorators/connections.js @@ -7,4 +7,7 @@ module.exports = (server) => { const account = this.auth.credentials; return DatabaseConnector.forAccount(account.id); }); + server.decorate('request', 'logger', (request) => { + return global.Logger.forAccount(request.auth.credentials) + }, {apply: true}); } diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index fd14f4fc5..65c776927 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -97,7 +97,7 @@ module.exports = (server) => { const {settings, email, provider, name} = request.payload; if (provider === 'imap') { - connectionChecks.push(IMAPConnection.connect(dbStub, settings)) + connectionChecks.push(IMAPConnection.connect({db: dbStub, settings})) } Promise.all(connectionChecks).then(() => { @@ -188,7 +188,7 @@ module.exports = (server) => { } Promise.all([ - IMAPConnection.connect({}, Object.assign({}, settings, credentials)), + IMAPConnection.connect({db: {}, settings: Object.assign({}, settings, credentials)}), ]) .then(() => buildAccountWith({ diff --git a/packages/nylas-api/routes/files.js b/packages/nylas-api/routes/files.js index e22975daf..46f7ee980 100644 --- a/packages/nylas-api/routes/files.js +++ b/packages/nylas-api/routes/files.js @@ -73,9 +73,7 @@ module.exports = (server) => { }, handler: (request, reply) => { request.getAccountDatabase().then(({File}) => { - const {headers: {accept}} = request const {params: {id}} = request - const account = request.auth.credentials File.findOne({where: {id}}).then((file) => { if (!file) { @@ -83,9 +81,9 @@ module.exports = (server) => { } return reply(Serialization.jsonStringify(file)); }) - .catch((error) => { - console.log('Error fetching file: ', error) - reply(error) + .catch((err) => { + request.logger.error(err, 'Error fetching file') + reply(err) }) }) }, @@ -107,7 +105,6 @@ module.exports = (server) => { handler: (request, reply) => { request.getAccountDatabase() .then((db) => { - const {headers: {accept}} = request const {params: {id}} = request const account = request.auth.credentials @@ -116,12 +113,12 @@ module.exports = (server) => { if (!file) { return reply.notFound(`File ${id} not found`) } - return file.fetch({account, db}) + return file.fetch({account, db, logger: request.logger}) .then((stream) => reply(stream)) }) - .catch((error) => { - console.log('Error fetching file: ', error) - reply(error) + .catch((err) => { + request.logger.error(err, 'Error downloading file') + reply(err) }) }) }, diff --git a/packages/nylas-api/routes/messages.js b/packages/nylas-api/routes/messages.js index c5efda964..474f773fe 100644 --- a/packages/nylas-api/routes/messages.js +++ b/packages/nylas-api/routes/messages.js @@ -123,15 +123,16 @@ module.exports = (server) => { return reply.notFound(`Message ${id} not found`) } if (accept === 'message/rfc822') { - return message.fetchRaw({account, db}).then((rawMessage) => + return message.fetchRaw({account, db, logger: request.logger}) + .then((rawMessage) => reply(rawMessage) ) } return reply(Serialization.jsonStringify(message)); }) - .catch((error) => { - console.log('Error fetching message: ', error) - reply(error) + .catch((err) => { + request.logger.error(err, 'Error fetching message') + reply(err) }) }) }, diff --git a/packages/nylas-api/routes/ping.js b/packages/nylas-api/routes/ping.js index a2c9e82e4..0e7072f78 100644 --- a/packages/nylas-api/routes/ping.js +++ b/packages/nylas-api/routes/ping.js @@ -6,8 +6,8 @@ module.exports = (server) => { auth: false, }, handler: (request, reply) => { - console.log("---> Ping!") - reply("pong") + request.logger.info('----> Pong!') + reply("Pong") }, }); }; diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index ac1eedb73..05e703b49 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -176,8 +176,9 @@ class IMAPConnection extends EventEmitter { return new IMAPConnection(...args).connect() } - constructor(db, settings) { + constructor({db, settings, logger = console} = {}) { super(); + this._logger = logger; this._db = db; this._queue = []; this._currentOperation = null; @@ -231,13 +232,13 @@ class IMAPConnection extends EventEmitter { this._imap = Promise.promisifyAll(new Imap(settings)); this._imap.once('end', () => { - console.log('Underlying IMAP Connection ended'); + this._logger.info('Underlying IMAP Connection ended'); this._connectPromise = null; this._imap = null; }); this._imap.on('alert', (msg) => { - console.log(`IMAP SERVER SAYS: ${msg}`) + this._logger.info({imap_server_msg: msg}, `IMAP server message`) }) // Emitted when new mail arrives in the currently open mailbox. @@ -346,14 +347,20 @@ class IMAPConnection extends EventEmitter { result.then(() => { this._currentOperation = null; - console.log(`Finished task: ${operation.description()}`) + this._logger.info({ + operation_type: operation.constructor.name, + operation_description: operation.description(), + }, `Finished sync operation`) resolve(); this.processNextOperation(); }) .catch((err) => { this._currentOperation = null; - console.log(`Task errored: ${operation.description()}`) - console.error(err) + this._logger.error({ + err, + operation_type: operation.constructor.name, + operation_description: operation.description(), + }, `Sync operation errored`) reject(err); }) } diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js index d8ac0b59a..3282ce9a3 100644 --- a/packages/nylas-core/index.js +++ b/packages/nylas-core/index.js @@ -11,4 +11,5 @@ module.exports = { SyncPolicy: require('./sync-policy'), SchedulerUtils: require('./scheduler-utils'), MessageTypes: require('./message-types'), + Logger: require('./logger'), } diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js new file mode 100644 index 000000000..aaa885fb4 --- /dev/null +++ b/packages/nylas-core/logger.js @@ -0,0 +1,52 @@ +const bunyan = require('bunyan') +const createCWStream = require('bunyan-cloudwatch') +const NODE_ENV = process.env.NODE_ENV || 'unknown' + + +function getLogStreams(name, env) { + const stdoutStream = { + stream: process.stdout, + level: 'info', + } + if (env === 'development') { + return [stdoutStream] + } + + const cloudwatchStream = { + stream: createCWStream({ + logGroup: `k2-${env}`, + logStream: `${name}-${env}`, + cloudWatchLogsOptions: { + region: 'us-east-1', + }, + }), + type: 'raw', + } + return [stdoutStream, cloudwatchStream] +} + +function createLogger(name, env = NODE_ENV) { + const childLogs = new Map() + const logger = bunyan.createLogger({ + name, + serializers: bunyan.stdSerializers, + streams: getLogStreams(name, env), + }) + + return Object.assign(logger, { + forAccount(account = {}) { + if (!childLogs.has(account.id)) { + const childLog = logger.child({ + account_id: account.id, + account_email: account.emailAddress, + }) + childLogs.set(account.id, childLog) + } + return childLogs.get(account.id) + }, + }) +} + +module.exports = { + createLogger, +} diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js index 5043318e8..655e8b13d 100644 --- a/packages/nylas-core/models/account/file.js +++ b/packages/nylas-core/models/account/file.js @@ -15,11 +15,11 @@ module.exports = (sequelize, Sequelize) => { }, }, instanceMethods: { - fetch: function fetch({account, db}) { + fetch: function fetch({account, db, logger}) { const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials()) return Promise.props({ message: this.getMessage(), - connection: IMAPConnection.connect(db, settings), + connection: IMAPConnection.connect({db, settings, logger}), }) .then(({message, connection}) => { return message.getFolder() diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js index 80bf1b5b5..e5fffdacc 100644 --- a/packages/nylas-core/models/account/message.js +++ b/packages/nylas-core/models/account/message.js @@ -69,11 +69,11 @@ module.exports = (sequelize, Sequelize) => { ) }, - fetchRaw: function fetchRaw({account, db}) { + fetchRaw: function fetchRaw({account, db, logger}) { const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials()) return Promise.props({ folder: this.getFolder(), - connection: IMAPConnection.connect(db, settings), + connection: IMAPConnection.connect({db, settings, logger}), }) .then(({folder, connection}) => { return connection.openBox(folder.name) diff --git a/packages/nylas-core/package.json b/packages/nylas-core/package.json index fa5d9cc27..3c151ff87 100644 --- a/packages/nylas-core/package.json +++ b/packages/nylas-core/package.json @@ -4,6 +4,7 @@ "description": "Core shared packages", "main": "index.js", "dependencies": { + "bunyan": "^1.8.1", "imap": "0.8.x", "xoauth2": "1.x.x" }, diff --git a/packages/nylas-core/pubsub-connector.js b/packages/nylas-core/pubsub-connector.js index 76df1df5d..36121688a 100644 --- a/packages/nylas-core/pubsub-connector.js +++ b/packages/nylas-core/pubsub-connector.js @@ -1,9 +1,11 @@ const Rx = require('rx') const redis = require("redis"); +const log = global.Logger || console Promise.promisifyAll(redis.RedisClient.prototype); Promise.promisifyAll(redis.Multi.prototype); + class PubsubConnector { constructor() { this._broadcastClient = null; @@ -13,7 +15,7 @@ class PubsubConnector { buildClient() { const client = redis.createClient(process.env.REDIS_URL || null); - client.on("error", console.error); + client.on("error", log.error); return client; } diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js index 55f8a4959..9b719f0ba 100644 --- a/packages/nylas-core/scheduler-utils.js +++ b/packages/nylas-core/scheduler-utils.js @@ -22,7 +22,9 @@ const forEachAccountList = (forEachCallback) => { } const assignPolicy = (accountId, policy) => { - console.log(`Changing policy for ${accountId} to ${JSON.stringify(policy)}`) + const log = global.Logger || console + log.info({policy, account_id: accountId}, `Changing single policy`) + const DatabaseConnector = require('./database-connector'); return DatabaseConnector.forShared().then(({Account}) => { Account.find({where: {id: accountId}}).then((account) => { @@ -33,7 +35,9 @@ const assignPolicy = (accountId, policy) => { } const assignPolicyToAcounts = (accountIds, policy) => { - console.log(`Changing policy for ${accountIds} to ${JSON.stringify(policy)}`) + const log = global.Logger || console + log.info({policy, account_ids: accountIds}, `Changing multiple policies`) + const DatabaseConnector = require('./database-connector'); return DatabaseConnector.forShared().then(({Account}) => { Account.findAll({where: {id: {$or: accountIds}}}).then((accounts) => { diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js index 2ee772b4a..dbd02d606 100644 --- a/packages/nylas-dashboard/app.js +++ b/packages/nylas-dashboard/app.js @@ -1,11 +1,12 @@ const Hapi = require('hapi'); const HapiWebSocket = require('hapi-plugin-websocket'); const Inert = require('inert'); -const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`); +const {DatabaseConnector, PubsubConnector, SchedulerUtils, Logger} = require(`nylas-core`); const fs = require('fs'); const path = require('path'); global.Promise = require('bluebird'); +global.Logger = Logger.createLogger('nylas-k2-dashboard') const server = new Hapi.Server(); server.connection({ port: process.env.PORT }); @@ -98,6 +99,6 @@ server.register([HapiWebSocket, Inert], () => { server.start((startErr) => { if (startErr) { throw startErr; } - console.log('Dashboard running at:', server.info.uri); + global.Logger.info({uri: server.info.uri}, 'Dashboard running'); }); }); diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js index 458a074e3..4ad7aee14 100644 --- a/packages/nylas-message-processor/app.js +++ b/packages/nylas-message-processor/app.js @@ -1,7 +1,8 @@ -const {PubsubConnector, DatabaseConnector} = require(`nylas-core`) +const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`) const {processors} = require('./processors') global.Promise = require('bluebird'); +global.Logger = Logger.createLogger('nylas-k2-message-processor') // List of the attributes of Message that the processor should be allowed to change. // The message may move between folders, get starred, etc. while it's being @@ -11,15 +12,13 @@ const MessageProcessorVersion = 1; const redis = PubsubConnector.buildClient(); -function runPipeline({db, accountId, message}) { - console.log(`Processing message ${message.id}`) +function runPipeline({db, accountId, message, logger}) { + logger.info(`MessageProcessor: Processing message`) return processors.reduce((prevPromise, processor) => ( prevPromise.then((prevMessage) => { - const processed = processor({message: prevMessage, accountId, db}); - if (!(processed instanceof Promise)) { - throw new Error(`processor ${processor} did not return a promise.`) - } - return processed.then((nextMessage) => { + const processed = processor({message: prevMessage, accountId, db, logger}); + return Promise.resolve(processed) + .then((nextMessage) => { if (!nextMessage.body) { throw new Error("processor did not resolve with a valid message object.") } @@ -46,26 +45,28 @@ function dequeueJob() { try { json = JSON.parse(item[1]); } catch (error) { - console.error(`MessageProcessor Failed: Found invalid JSON item in queue: ${item}`) + global.Logger.error({item}, `MessageProcessor: Found invalid JSON item in queue`) return dequeueJob(); } const {messageId, accountId} = json; + const logger = global.Logger.forAccount({id: accountId}).child({message_id: messageId}) - DatabaseConnector.forAccount(accountId).then((db) => - db.Message.find({ + DatabaseConnector.forAccount(accountId).then((db) => { + return db.Message.find({ where: {id: messageId}, include: [{model: db.Folder}, {model: db.Label}], }).then((message) => { if (!message) { return Promise.reject(new Error(`Message not found (${messageId}). Maybe account was deleted?`)) } - return runPipeline({db, accountId, message}).then((processedMessage) => + return runPipeline({db, accountId, message, logger}).then((processedMessage) => saveMessage(processedMessage) ).catch((err) => - console.error(`MessageProcessor Failed: ${err} ${err.stack}`) + logger.error(err, `MessageProcessor: Failed`) ) }) - ).finally(() => { + }) + .finally(() => { dequeueJob() }); diff --git a/packages/nylas-message-processor/processors/parsing.js b/packages/nylas-message-processor/processors/parsing.js index 6763437e0..a144199ab 100644 --- a/packages/nylas-message-processor/processors/parsing.js +++ b/packages/nylas-message-processor/processors/parsing.js @@ -11,7 +11,7 @@ function Contact({name, address} = {}) { const extractContacts = (values) => (values || []).map(v => Contact(mimelib.parseAddresses(v).pop())) -function processMessage({message}) { +function processMessage({message, logger}) { if (message.snippet) { // trim and clean snippet which is alreay present (from message plaintext) message.snippet = message.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ') @@ -24,7 +24,7 @@ function processMessage({message}) { // TODO: Fanciness message.snippet = message.body.substr(0, Math.min(message.body.length, SNIPPET_SIZE)); } else { - console.log("Received message has no body or snippet.") + logger.info("MessageProcessor: Parsing - Received message has no body or snippet.") } message.to = extractContacts(message.headers.to); diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js index 57e896ec2..2f9c847fd 100644 --- a/packages/nylas-message-processor/processors/threading.js +++ b/packages/nylas-message-processor/processors/threading.js @@ -9,11 +9,14 @@ class ThreadingProcessor { // conversation. Put it back soonish. // const messageEmails = _.uniq([].concat(message.to, message.cc, message.from).map(p => p.email)); - // console.log(`Found ${threads.length} candidate threads for message with subject: ${message.subject}`) + // this.logger.info({ + // num_candidate_threads: threads.length, + // message_subject: message.subject, + // }, `Found candidate threads for message`) // // for (const thread of threads) { // const threadEmails = _.uniq([].concat(thread.participants).map(p => p.email)); - // console.log(`Intersection: ${_.intersection(threadEmails, messageEmails).join(',')}`) + // this.logger.info(`Intersection: ${_.intersection(threadEmails, messageEmails).join(',')}`) // // if (_.intersection(threadEmails, messageEmails) >= threadEmails.length * 0.9) { // return thread; @@ -66,7 +69,7 @@ class ThreadingProcessor { }) } - processMessage({db, message}) { + processMessage({db, message, logger}) { if (!(message.labels instanceof Array)) { throw new Error("Threading processMessage expects labels to be an inflated array."); } @@ -74,6 +77,8 @@ class ThreadingProcessor { throw new Error("Threading processMessage expects folder value to be present."); } + this.logger = logger + const {Folder, Label} = db; let findOrCreateThread = null; if (message.headers['x-gm-thrid']) { diff --git a/packages/nylas-message-processor/spec/threading-spec.js b/packages/nylas-message-processor/spec/threading-spec.js index b1df5d815..5e0aaa3ba 100644 --- a/packages/nylas-message-processor/spec/threading-spec.js +++ b/packages/nylas-message-processor/spec/threading-spec.js @@ -43,7 +43,6 @@ it('adds the message to the thread', (done) => { }, create: (message) => { message.setThread = (thread) => { - console.log("setting") message.thread = thread.id } return Promise.resolve(message) diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 5d7319245..af70a66cb 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -1,16 +1,17 @@ global.Promise = require('bluebird'); - -const {DatabaseConnector} = require(`nylas-core`) +const {DatabaseConnector, Logger} = require(`nylas-core`) const SyncProcessManager = require('./sync-process-manager'); +global.Logger = Logger.createLogger('nylas-k2-sync') + const manager = new SyncProcessManager(); DatabaseConnector.forShared().then((db) => { const {Account} = db; Account.findAll().then((accounts) => { if (accounts.length === 0) { - console.log(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) - console.log(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) + global.Logger.info(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) + global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) } manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => { manager.start(); diff --git a/packages/nylas-sync/imap/fetch-category-list.js b/packages/nylas-sync/imap/fetch-folder-list.js similarity index 97% rename from packages/nylas-sync/imap/fetch-category-list.js rename to packages/nylas-sync/imap/fetch-folder-list.js index b82f33876..d5c5a0000 100644 --- a/packages/nylas-sync/imap/fetch-category-list.js +++ b/packages/nylas-sync/imap/fetch-folder-list.js @@ -3,8 +3,9 @@ const {Provider} = require('nylas-core'); const GMAIL_FOLDERS = ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam']; class FetchFolderList { - constructor(provider) { + constructor(provider, logger = console) { this._provider = provider; + this._logger = logger; } description() { diff --git a/packages/nylas-sync/imap/fetch-messages-in-category.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js similarity index 84% rename from packages/nylas-sync/imap/fetch-messages-in-category.js rename to packages/nylas-sync/imap/fetch-messages-in-folder.js index 554f09e71..0bf994f61 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-category.js +++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js @@ -11,12 +11,13 @@ const FETCH_MESSAGES_FIRST_COUNT = 100; const FETCH_MESSAGES_COUNT = 200; class FetchMessagesInFolder { - constructor(category, options) { + constructor(category, options, logger = console) { this._imap = null this._box = null this._db = null this._category = category; this._options = options; + this._logger = logger; if (!this._category) { throw new Error("FetchMessagesInFolder requires a category") } @@ -86,9 +87,13 @@ class FetchMessagesInFolder { } }) - console.log(` --- found ${flagChangeMessages.length || 'no'} flag changes`) + this._logger.info({ + flag_changes: flagChangeMessages.length, + }, `FetchMessagesInFolder: found flag changes`) if (createdUIDs.length > 0) { - console.log(` --- found ${createdUIDs.length} new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`) + this._logger.info({ + new_messages: createdUIDs.length, + }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`) } if (flagChangeMessages.length === 0) { @@ -111,7 +116,9 @@ class FetchMessagesInFolder { .filter(msg => !remoteUIDAttributes[msg.folderImapUID]) .map(msg => msg.folderImapUID) - console.log(` --- found ${removedUIDs.length} messages no longer in the folder`) + this._logger.info({ + removed_messages: removedUIDs.length, + }, `FetchMessagesInFolder: found messages no longer in the folder`) if (removedUIDs.length === 0) { return Promise.resolve(); @@ -148,7 +155,9 @@ class FetchMessagesInFolder { } if (desired.length === 0) { - console.warn(`Could not find good part. Options are: ${available.join(', ')}`) + this._logger.warn({ + available_options: available.join(', '), + }, `FetchMessagesInFolder: Could not find good part`) } return desired; @@ -173,7 +182,10 @@ class FetchMessagesInFolder { const uids = uidsByPart[key]; const desiredParts = JSON.parse(key); const bodies = ['HEADER'].concat(desiredParts.map(p => p.id)); - console.log(`Fetching parts ${key} for ${uids.length} messages`) + this._logger.info({ + key, + num_messages: uids.length, + }`FetchMessagesInFolder: Fetching parts for messages`) // note: the order of UIDs in the array doesn't matter, Gmail always // returns them in ascending (oldest => newest) order. @@ -258,11 +270,17 @@ class FetchMessagesInFolder { ) .then((message) => { if (created) { - console.log(`Created message ID: ${message.id}, UID: ${attributes.uid}`) + this._logger.info({ + message_id: message.id, + uid: attributes.uid, + }, `FetchMessagesInFolder: Created message`) this._createFilesFromStruct({message, struct: attributes.struct}) PubsubConnector.queueProcessMessage({accountId, messageId: message.id}); } else { - console.log(`Updated message ID: ${message.id}, UID: ${attributes.uid}`) + this._logger.info({ + message_id: message.id, + uid: attributes.uid, + }, `FetchMessagesInFolder: Updated message`) } }) @@ -291,7 +309,9 @@ class FetchMessagesInFolder { const desiredRanges = []; - console.log(` - Fetching messages. Currently have range: ${savedSyncState.fetchedmin}:${savedSyncState.fetchedmax}`) + this._logger.info({ + range: `${savedSyncState.fetchedmin}:${savedSyncState.fetchedmax}`, + }, `FetchMessagesInFolder: Fetching messages.`) // Todo: In the future, this is where logic should go that limits // sync based on number of messages / age of messages. @@ -303,18 +323,20 @@ class FetchMessagesInFolder { if (savedSyncState.fetchedmax < boxUidnext) { desiredRanges.push({min: savedSyncState.fetchedmax, max: boxUidnext}) } else { - console.log(" --- fetchedmax == uidnext, nothing more recent to fetch.") + this._logger.info('FetchMessagesInFolder: fetchedmax == uidnext, nothing more recent to fetch.') } if (savedSyncState.fetchedmin > 1) { const lowerbound = Math.max(1, savedSyncState.fetchedmin - FETCH_MESSAGES_COUNT); desiredRanges.push({min: lowerbound, max: savedSyncState.fetchedmin}) } else { - console.log(" --- fetchedmin == 1, nothing older to fetch.") + this._logger.info("FetchMessagesInFolder: fetchedmin == 1, nothing older to fetch.") } } return Promise.each(desiredRanges, ({min, max}) => { - console.log(` --- fetching range: ${min}:${max}`); + this._logger.info({ + range: `${min}:${max}`, + }, `FetchMessagesInFolder: Fetching range`); return this._fetchMessagesAndQueueForProcessing(`${min}:${max}`).then(() => { const {fetchedmin, fetchedmax} = this._category.syncState; @@ -326,7 +348,7 @@ class FetchMessagesInFolder { }); }) }).then(() => { - console.log(` - Fetching messages finished`); + this._logger.info(`FetchMessagesInFolder: Fetching messages finished`); }); } @@ -350,15 +372,15 @@ class FetchMessagesInFolder { let shallowFetch = null; if (this._imap.serverSupports(Capabilities.Condstore)) { - console.log(` - Shallow attribute scan (using CONDSTORE)`) + this._logger.info(`FetchMessagesInFolder: Shallow attribute scan (using CONDSTORE)`) if (nextHighestmodseq === highestmodseq) { - console.log(" --- highestmodseq matches, nothing more to fetch") + this._logger.info('FetchMessagesInFolder: highestmodseq matches, nothing more to fetch') return Promise.resolve(); } shallowFetch = this._box.fetchUIDAttributes(`1:*`, {changedsince: highestmodseq}); } else { const range = `${this._getLowerBoundUID(SHALLOW_SCAN_UID_COUNT)}:*`; - console.log(` - Shallow attribute scan (using range: ${range})`) + this._logger.info({range}, `FetchMessagesInFolder: Shallow attribute scan`) shallowFetch = this._box.fetchUIDAttributes(range); } @@ -372,7 +394,7 @@ class FetchMessagesInFolder { this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) )) .then(() => { - console.log(` - finished fetching changes to messages`); + this._logger.info(`FetchMessagesInFolder: finished fetching changes to messages`); return this.updateFolderSyncState({ highestmodseq: nextHighestmodseq, timeShallowScan: Date.now(), @@ -386,7 +408,7 @@ class FetchMessagesInFolder { const {fetchedmin, fetchedmax} = this._category.syncState; const range = `${fetchedmin}:${fetchedmax}`; - console.log(` - Deep attribute scan: fetching attributes in range: ${range}`) + this._logger.info({range}, `FetchMessagesInFolder: Deep attribute scan: fetching attributes in range`) return this._box.fetchUIDAttributes(range) .then((remoteUIDAttributes) => { @@ -401,7 +423,7 @@ class FetchMessagesInFolder { }) )) .then(() => { - console.log(` - Deep scan finished.`); + this._logger.info(`FetchMessagesInFolder: Deep scan finished.`); return this.updateFolderSyncState({ highestmodseq: this._box.highestmodseq, timeDeepScan: Date.now(), diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index 513bd1664..d9cab0829 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -43,7 +43,7 @@ class SyncProcessManager { } start() { - console.log(`ProcessManager: Starting with ID ${IDENTITY}`) + global.Logger.info(`ProcessManager: Starting with ID ${IDENTITY}`) this.unassignAccountsAssignedTo(IDENTITY).then(() => { this.unassignAccountsMissingHeartbeats(); @@ -63,12 +63,12 @@ class SyncProcessManager { client.setAsync(key, Date.now()).then(() => client.expireAsync(key, HEARTBEAT_EXPIRES) ).then(() => - console.log("ProcessManager: 💘") + global.Logger.info("ProcessManager: 💘") ) } onSigInt() { - console.log(`ProcessManager: Exiting...`) + global.Logger.info(`ProcessManager: Exiting...`) this._exiting = true; this.unassignAccountsAssignedTo(IDENTITY).then(() => @@ -85,7 +85,7 @@ class SyncProcessManager { let unseenIds = [].concat(accountIds); - console.log("ProcessManager: Starting scan for accountIds in database that are not present in Redis.") + global.Logger.info("ProcessManager: Starting scan for accountIds in database that are not present in Redis.") return forEachAccountList((foundProcessIdentity, foundIds) => { unseenIds = unseenIds.filter((a) => !foundIds.includes(`${a}`)) @@ -94,7 +94,7 @@ class SyncProcessManager { if (unseenIds.length === 0) { return; } - console.log(`ProcessManager: Adding account IDs ${unseenIds.join(',')} to ${ACCOUNTS_UNCLAIMED}.`) + global.Logger.info(`ProcessManager: Adding account IDs ${unseenIds.join(',')} to ${ACCOUNTS_UNCLAIMED}.`) unseenIds.map((id) => client.lpushAsync(ACCOUNTS_UNCLAIMED, id)); }); } @@ -102,7 +102,7 @@ class SyncProcessManager { unassignAccountsMissingHeartbeats() { const client = PubsubConnector.broadcastClient(); - console.log("ProcessManager: Starting unassignment for processes missing heartbeats.") + global.Logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.") Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => { const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, ''); @@ -125,12 +125,12 @@ class SyncProcessManager { ) return unassignOne(0).then((returned) => { - console.log(`ProcessManager: Returned ${returned} accounts assigned to ${identity}.`) + global.Logger.info(`ProcessManager: Returned ${returned} accounts assigned to ${identity}.`) }); } update() { - console.log(`ProcessManager: Searching for an unclaimed account to sync.`) + global.Logger.info(`ProcessManager: Searching for an unclaimed account to sync.`) this.acceptUnclaimedAccount().finally(() => { if (this._exiting) { @@ -170,7 +170,7 @@ class SyncProcessManager { if (this._exiting || this._workers[account.id]) { return; } - console.log(`ProcessManager: Starting worker for Account ${accountId}`) + global.Logger.info(`ProcessManager: Starting worker for Account ${accountId}`) this._workers[account.id] = new SyncWorker(account, db, () => { this.removeWorkerForAccountId(accountId) }); diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 6ebb34b8d..d0788ca78 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -11,8 +11,8 @@ const { const {CLAIM_DURATION} = SchedulerUtils; -const FetchFolderList = require('./imap/fetch-category-list') -const FetchMessagesInFolder = require('./imap/fetch-messages-in-category') +const FetchFolderList = require('./imap/fetch-folder-list') +const FetchMessagesInFolder = require('./imap/fetch-messages-in-folder') const SyncbackTaskFactory = require('./syncback-task-factory') @@ -24,6 +24,7 @@ class SyncWorker { this._startTime = Date.now(); this._lastSyncTime = null; this._onExpired = onExpired; + this._logger = global.Logger.forAccount(account) this._syncTimer = null; this._expirationTimer = null; @@ -100,7 +101,7 @@ class SyncWorker { return Promise.reject(new Error("ensureConnection: There are no IMAP connection credentials for this account.")) } - const conn = new IMAPConnection(this._db, Object.assign({}, settings, credentials)); + const conn = new IMAPConnection({db: this._db, settings: Object.assign({}, settings, credentials), logger: this._logger}); conn.on('mail', () => { this._onConnectionIdleUpdate(); }) @@ -145,7 +146,7 @@ class SyncWorker { ) return Promise.all(categoriesToSync.map((cat) => - this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions)) + this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions, this._logger)) )) }); } @@ -155,10 +156,10 @@ class SyncWorker { this._syncTimer = null; if (!process.env.SYNC_AFTER_ERRORS && this._account.errored()) { - console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) is in error state - Skipping sync`) + this._logger.info(`SyncWorker: Account is in error state - Skipping sync`) return } - console.log(`SyncWorker: Account ${this._account.emailAddress} (${this._account.id}) sync started (${reason})`) + this._logger.info({reason}, `SyncWorker: Account sync started`) this.ensureConnection() .then(() => this._account.update({syncError: null})) @@ -174,7 +175,7 @@ class SyncWorker { } onSyncError(error) { - console.error(`SyncWorker: Error while syncing account ${this._account.emailAddress} (${this._account.id})`, error) + this._logger.error(error, `SyncWorker: Error while syncing account`) this.closeConnection() if (error.source.includes('socket') || error.source.includes('timeout')) { @@ -203,16 +204,16 @@ class SyncWorker { this._account.lastSyncCompletions = lastSyncCompletions this._account.save() - console.log('Syncworker: Completed sync cycle') + this._logger.info('Syncworker: Completed sync cycle') if (afterSync === 'idle') { return this._getIdleFolder() .then((idleFolder) => this._conn.openBox(idleFolder.name)) - .then(() => console.log('SyncWorker: - Idling on inbox category')) + .then(() => this._logger.info('SyncWorker: - Idling on inbox category')) } if (afterSync === 'close') { - console.log('SyncWorker: - Closing connection'); + this._logger.info('SyncWorker: - Closing connection'); this.closeConnection() return Promise.resolve() } @@ -226,7 +227,7 @@ class SyncWorker { scheduleNextSync() { if (Date.now() - this._startTime > CLAIM_DURATION) { - console.log("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool."); + this._logger.info("SyncWorker: - Has held account for more than CLAIM_DURATION, returning to pool."); this.cleanup(); this._onExpired(); return; @@ -238,7 +239,10 @@ class SyncWorker { if (interval) { const target = this._lastSyncTime + interval; - console.log(`SyncWorker: Account ${active ? 'active' : 'inactive'}. Next sync scheduled for ${new Date(target).toLocaleString()}`); + this._logger.info({ + is_active: active, + next_sync: new Date(target).toLocaleString(), + }, `SyncWorker: Next sync scheduled`); this._syncTimer = setTimeout(() => { this.syncNow({reason: 'Scheduled'}); }, target - Date.now()); diff --git a/pm2-dev.yml b/pm2-dev.yml index 313be4c0d..a88d4e8c2 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -12,25 +12,23 @@ apps: GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com" GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF" GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" - - + NODE_ENV: 'development' - script : packages/nylas-sync/app.js name : sync env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - - + NODE_ENV: 'development' - script : packages/nylas-dashboard/app.js name : dashboard env : PORT: 5101 DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" - - + NODE_ENV: 'development' - script : packages/nylas-message-processor/app.js name : processor env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" + NODE_ENV: 'development' From 93189b0f22f49d897d55b20858d41da8e4727e62 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 10:56:08 -0700 Subject: [PATCH 202/800] Try adding newrelic logging --- packages/nylas-api/app.js | 2 ++ packages/nylas-api/newrelic.js | 24 ++++++++++++++++++++++++ packages/nylas-sync/app.js | 1 + packages/nylas-sync/newrelic.js | 24 ++++++++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 packages/nylas-api/newrelic.js create mode 100644 packages/nylas-sync/newrelic.js diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index bca30c9af..8fab0cd2e 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -1,3 +1,5 @@ +require('newrelic'); + const Hapi = require('hapi'); const HapiSwagger = require('hapi-swagger'); const HapiBoom = require('hapi-boom-decorators') diff --git a/packages/nylas-api/newrelic.js b/packages/nylas-api/newrelic.js new file mode 100644 index 000000000..00882097e --- /dev/null +++ b/packages/nylas-api/newrelic.js @@ -0,0 +1,24 @@ +/** + * New Relic agent configuration. + * + * See lib/config.defaults.js in the agent distribution for a more complete + * description of configuration variables and their potential values. + */ +exports.config = { + /** + * Array of application names. + */ + app_name: ['Nylas K2 API'], + /** + * Your New Relic license key. + */ + license_key: 'e232d6ccc786bd87aa72b86782439710162e3739', + logging: { + /** + * Level at which to log. 'trace' is most useful to New Relic when diagnosing + * issues with the agent, 'info' and higher will impose the least overhead on + * production applications. + */ + level: 'info', + }, +} diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 5d7319245..227e7a365 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -1,3 +1,4 @@ +require('newrelic'); global.Promise = require('bluebird'); const {DatabaseConnector} = require(`nylas-core`) diff --git a/packages/nylas-sync/newrelic.js b/packages/nylas-sync/newrelic.js new file mode 100644 index 000000000..a85120e87 --- /dev/null +++ b/packages/nylas-sync/newrelic.js @@ -0,0 +1,24 @@ +/** + * New Relic agent configuration. + * + * See lib/config.defaults.js in the agent distribution for a more complete + * description of configuration variables and their potential values. + */ +exports.config = { + /** + * Array of application names. + */ + app_name: ['Nylas K2 Sync'], + /** + * Your New Relic license key. + */ + license_key: 'e232d6ccc786bd87aa72b86782439710162e3739', + logging: { + /** + * Level at which to log. 'trace' is most useful to New Relic when diagnosing + * issues with the agent, 'info' and higher will impose the least overhead on + * production applications. + */ + level: 'info', + }, +} From 600245ebcb930f8b0e161b285c933f33e8b18e13 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Fri, 8 Jul 2016 18:48:52 -0700 Subject: [PATCH 203/800] Fix cloudwatch config --- packages/nylas-core/logger.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js index aaa885fb4..da3bba76f 100644 --- a/packages/nylas-core/logger.js +++ b/packages/nylas-core/logger.js @@ -14,8 +14,8 @@ function getLogStreams(name, env) { const cloudwatchStream = { stream: createCWStream({ - logGroup: `k2-${env}`, - logStream: `${name}-${env}`, + logGroupName: `k2-${env}`, + logStreamName: `${name}-${env}`, cloudWatchLogsOptions: { region: 'us-east-1', }, From f648016c621463357a7fffb89299ca83ea1eb530 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 11:27:35 -0700 Subject: [PATCH 204/800] Lock logger versions --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 411c58799..c7d2fea4f 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "main": "", "dependencies": { "bluebird": "3.x.x", - "bunyan": "^1.8.1", - "bunyan-cloudwatch": "^2.0.0", + "bunyan": "1.8.0", + "bunyan-cloudwatch": "2.0.0", "lerna": "2.0.0-beta.23", "mysql": "^2.11.1", "newrelic": "^1.28.1", From ff334e169cebb9c1f8ef11ef366b5ab19c3a00cc Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 11:43:51 -0700 Subject: [PATCH 205/800] Make npm start log output logs too --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c7d2fea4f..807c2d4b6 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "pm2 start ./pm2-dev.yml", + "start": "pm2 start ./pm2-dev.yml; pm2 logs --raw | bunyan -o short", "logs": "pm2 logs --raw | bunyan -o short", "stop": "pm2 delete all", "postinstall": "lerna bootstrap" From 98c17c9780f2e9c8627014c6462c0cf000f6ea6f Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 11:47:42 -0700 Subject: [PATCH 206/800] Fix api logger decorator --- packages/nylas-api/decorators/connections.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/nylas-api/decorators/connections.js b/packages/nylas-api/decorators/connections.js index f0fe1d437..161e9beb6 100644 --- a/packages/nylas-api/decorators/connections.js +++ b/packages/nylas-api/decorators/connections.js @@ -8,6 +8,9 @@ module.exports = (server) => { return DatabaseConnector.forAccount(account.id); }); server.decorate('request', 'logger', (request) => { - return global.Logger.forAccount(request.auth.credentials) + if (request.auth.credentials) { + return global.Logger.forAccount(request.auth.credentials) + } + return global.Logger }, {apply: true}); } From c0d7902f9fb4b40549d794a28146439fd52cd728 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 11 Jul 2016 11:48:54 -0700 Subject: [PATCH 207/800] Add npm run restart --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 807c2d4b6..a94202966 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "start": "pm2 start ./pm2-dev.yml; pm2 logs --raw | bunyan -o short", "logs": "pm2 logs --raw | bunyan -o short", "stop": "pm2 delete all", + "restart": "npm run stop && npm run start", "postinstall": "lerna bootstrap" }, "repository": { From 84815ce2fa884d6ebc3992003429dec38c5b0b17 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 11 Jul 2016 12:00:20 -0700 Subject: [PATCH 208/800] Add file watching to restart server --- .gitignore | 1 + package.json | 3 +-- pm2-dev.yml | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ce1018e2e..e5ddc09e2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dump.rdb *npm-debug.log storage/ lerna-debug.log +newrelic_agent.log # Elastic Beanstalk Files .elasticbeanstalk/* diff --git a/package.json b/package.json index a94202966..7a33802a6 100644 --- a/package.json +++ b/package.json @@ -27,10 +27,9 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "pm2 start ./pm2-dev.yml; pm2 logs --raw | bunyan -o short", + "start": "pm2 start ./pm2-dev.yml --watch; pm2 logs --raw | bunyan -o short", "logs": "pm2 logs --raw | bunyan -o short", "stop": "pm2 delete all", - "restart": "npm run stop && npm run start", "postinstall": "lerna bootstrap" }, "repository": { diff --git a/pm2-dev.yml b/pm2-dev.yml index a88d4e8c2..4f60094fd 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -4,6 +4,7 @@ apps: - script : packages/nylas-api/app.js + watch : true name : api env : PORT: 5100 @@ -14,12 +15,14 @@ apps: GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" NODE_ENV: 'development' - script : packages/nylas-sync/app.js + watch : true name : sync env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" NODE_ENV: 'development' - script : packages/nylas-dashboard/app.js + watch : true name : dashboard env : PORT: 5101 @@ -27,6 +30,7 @@ apps: DB_ENCRYPTION_PASSWORD : "d6F3Efeq" NODE_ENV: 'development' - script : packages/nylas-message-processor/app.js + watch : true name : processor env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" From 1c4434c136902fa30be0ac30fc0565c5e7834b47 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 12:12:17 -0700 Subject: [PATCH 209/800] Re-emit error events on bunyan log streams --- packages/nylas-core/logger.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js index da3bba76f..40c73ad31 100644 --- a/packages/nylas-core/logger.js +++ b/packages/nylas-core/logger.js @@ -21,6 +21,7 @@ function getLogStreams(name, env) { }, }), type: 'raw', + reemitErrorEvents: true, } return [stdoutStream, cloudwatchStream] } From 877a6bf6122da17aeff5c56b45e1ef470ad37106 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Mon, 11 Jul 2016 12:55:34 -0700 Subject: [PATCH 210/800] Add SyncbackRequest details to dashboard --- packages/nylas-dashboard/public/css/app.css | 78 +++++++- .../public/images/dropdown.png | Bin 0 -> 333 bytes packages/nylas-dashboard/public/index.html | 2 + packages/nylas-dashboard/public/js/app.jsx | 2 + .../nylas-dashboard/public/js/dropdown.jsx | 71 ++++++++ .../public/js/set-all-sync-policies.jsx | 23 ++- .../nylas-dashboard/public/js/sync-policy.jsx | 2 +- .../public/js/syncback-request-details.jsx | 171 ++++++++++++++++++ .../routes/syncback-requests.js | 81 +++++++++ 9 files changed, 419 insertions(+), 11 deletions(-) create mode 100644 packages/nylas-dashboard/public/images/dropdown.png create mode 100644 packages/nylas-dashboard/public/js/dropdown.jsx create mode 100644 packages/nylas-dashboard/public/js/syncback-request-details.jsx create mode 100644 packages/nylas-dashboard/routes/syncback-requests.js diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css index ceb513342..436426196 100644 --- a/packages/nylas-dashboard/public/css/app.css +++ b/packages/nylas-dashboard/public/css/app.css @@ -59,10 +59,14 @@ pre { .action-link { display: inline-block; - margin: 5px; color: rgba(16, 83, 161, 0.88); text-decoration: underline; cursor: pointer; + margin: 5px 0; +} + +.action-link.cancel { + margin-left: 5px; } .sync-policy textarea { @@ -76,6 +80,7 @@ pre { width: 50%; margin: auto; padding: 20px; + overflow: auto; } .modal-bg { @@ -88,6 +93,15 @@ pre { padding-top: 10%; } +.modal-close { + position: relative; + float: right; + top: -10px; + cursor: pointer; + font-size: 14px; + font-weight: bold; +} + .sync-graph { margin-top: 3px; } @@ -97,3 +111,65 @@ pre { margin-top: 5px; margin-bottom: 1px; } + +#syncback-request-details { + font-size: 15px; + color: black; +} + +#syncback-request-details .counts { + margin: 10px; +} + +#syncback-request-details span { + margin: 10px; +} + +#syncback-request-details table { + width: 100%; +} + +#syncback-request-details tr:nth-child(even) { + background-color: #F1F1F1; +} + +#syncback-request-details tr:not(:first-child):hover { + background-color: #C9C9C9; +} + +#syncback-request-details td, #syncback-request-details th { + text-align: center; + padding: 10px 5px; +} + +.dropdown-arrow { + margin: 0 5px; + height: 7px; + vertical-align: middle; +} + +.dropdown-options { + border: solid black 1px; + position: absolute; + background-color: white; + text-align: left; +} + +.dropdown-option { + padding: 0px 2px; +} + +.dropdown-option:hover { + background-color: rgb(114, 163, 255); +} + +.dropdown-selected { + display: inline; +} + +.dropdown-wrapper { + display: inline; + cursor: pointer; + position: absolute; + font-weight: normal; +} diff --git a/packages/nylas-dashboard/public/images/dropdown.png b/packages/nylas-dashboard/public/images/dropdown.png new file mode 100644 index 0000000000000000000000000000000000000000..a49a693f9ec1b6d0e552d4fe2f3e0dc748f7cbdc GIT binary patch literal 333 zcmV-T0kZyyP)Px$2T4RhR45f=U;u)E@HkKZh`2x?lM%!x1_XsC8u>>gT7lFu28PG^|7T>}0M$5s z!-|F3K=ywqA1mb_o>0Na_`e+_`Jdr`G}C`323{nUnE?^;Q-OS(EEG&;n)oL`<2ai z^X55UnL&yO8HC$_S>HhVQT!en8p`&EnQJvLRQ*7bAOK{a04lP8@IWNvK8On-27pBn zk`NX)gAi&k4FJi3y@ODM(`!H_*bTsH2uKh!jBpzOQjZcQAOU0m^4c$EPQQO&Iks)y fj0gi{H6S(s8clzSk0>{000000NkvXXu0mjf*8_nP literal 0 HcmV?d00001 diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html index 89227c3d8..a1c31b217 100644 --- a/packages/nylas-dashboard/public/index.html +++ b/packages/nylas-dashboard/public/index.html @@ -7,6 +7,8 @@ + + diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 5f740093b..660df19e8 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -6,6 +6,7 @@ const { SetAllSyncPolicies, AccountFilter, SyncGraph, + SyncbackRequestDetails, } = window; class Account extends React.Component { @@ -51,6 +52,7 @@ class Account extends React.Component {

{account.email_address} {active ? '🌕' : '🌑'}

{assignment} + selectedOnClick.call(this)}> + {this.state.selected} + dropdown arrow +
+ ); + + // All options, not shown if dropdown is closed + let options = []; + let optionsWrapper = ; + if (!this.state.closed) { + for (const opt of this.props.options) { + options.push( +
this.selectAndClose.call(this, opt)}> {opt}
+ ); + } + optionsWrapper = ( +
+ {options} +
+ ) + } + + return ( +
this.close.call(this)}> + {selected} + {optionsWrapper} +
+ ); + } + +} + +Dropdown.propTypes = { + options: React.PropTypes.arrayOf(React.PropTypes.string), + defaultOption: React.PropTypes.string, + onSelect: React.PropTypes.func, +} + +window.Dropdown = Dropdown; diff --git a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx index f9346c1e6..06bfe2e26 100644 --- a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx +++ b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx @@ -38,15 +38,20 @@ class SetAllSyncPolicies extends React.Component { render() { if (this.state.editMode) { return ( -
-
-
Sync Policy
- - - this.cancel.call(this)}> Cancel +
+ this.edit.call(this)}> + Set sync policies for currently displayed accounts + +
+
+
Sync Policy
+ + + this.cancel.call(this)}> Cancel +
) diff --git a/packages/nylas-dashboard/public/js/sync-policy.jsx b/packages/nylas-dashboard/public/js/sync-policy.jsx index bfbee8ffb..5efe9655b 100644 --- a/packages/nylas-dashboard/public/js/sync-policy.jsx +++ b/packages/nylas-dashboard/public/js/sync-policy.jsx @@ -43,7 +43,7 @@ class SyncPolicy extends React.Component { {this.props.stringifiedSyncPolicy} - this.cancel.call(this)}> Cancel + this.cancel.call(this)}> Cancel
) diff --git a/packages/nylas-dashboard/public/js/syncback-request-details.jsx b/packages/nylas-dashboard/public/js/syncback-request-details.jsx new file mode 100644 index 000000000..1aecdb220 --- /dev/null +++ b/packages/nylas-dashboard/public/js/syncback-request-details.jsx @@ -0,0 +1,171 @@ +const React = window.React; +const Dropdown = window.Dropdown; + +class SyncbackRequestDetails extends React.Component { + constructor(props) { + super(props); + this.state = { + open: false, + accountId: props.accountId, + syncbackRequests: null, + counts: null, + statusFilter: 'all', + }; + } + + getDetails() { + const req = new XMLHttpRequest(); + const url = `${window.location.protocol}/syncback-requests/${this.state.accountId}`; + req.open("GET", url, true); + req.onreadystatechange = () => { + if (req.readyState === XMLHttpRequest.DONE) { + if (req.status === 200) { + this.setState({syncbackRequests: req.responseText}); + } else { + console.error(req.responseText); + } + } + } + req.send(); + } + + getCounts() { + const since = Date.now() - 1000 * 60 * 60; // one hour ago + const req = new XMLHttpRequest(); + const url = `${window.location.protocol}/syncback-requests/${this.state.accountId}/counts?since=${since}`; + req.open("GET", url, true); + req.onreadystatechange = () => { + if (req.readyState === XMLHttpRequest.DONE) { + if (req.status === 200) { + this.setState({counts: JSON.parse(req.responseText)}); + } else { + console.error(req.responseText); + } + } + } + req.send(); + } + + setStatusFilter(statusFilter) { + this.setState({statusFilter: statusFilter}); + } + + open() { + this.getDetails(); + this.getCounts(); + this.setState({open: true}); + } + + close() { + this.setState({open: false}); + } + + render() { + if (this.state.open) { + let counts = Of requests created in the last hour: ... + if (this.state.counts) { + const total = this.state.counts.new + this.state.counts.failed + + this.state.counts.succeeded; + if (total === 0) { + counts = "No requests made in the last hour"; + } else { + counts = ( +
+ Of requests created in the last hour: + + {this.state.counts.failed / total * 100}% failed + + + {this.state.counts.succeeded / total * 100}% succeeded + + + {/* .new was throwing off my syntax higlighting, so ignoring linter*/} + {this.state.counts['new'] / total * 100}% are still new + +
+ ) + } + } + + let details = "Loading..." + if (this.state.syncbackRequests) { + let reqs = JSON.parse(this.state.syncbackRequests); + if (this.state.statusFilter !== 'all') { + reqs = reqs.filter((req) => req.status === this.state.statusFilter); + } + let rows = []; + if (reqs.length === 0) { + rows.push(No results--); + } + for (let i = reqs.length - 1; i >= 0; i--) { + const req = reqs[i]; + const date = new Date(req.createdAt); + rows.push( + {req.status} + {req.type} + {date.toLocaleTimeString()}, {date.toLocaleDateString()} + ) + } + details = ( + + + + + + + + {rows} + +
+ Status:  + this.setStatusFilter.call(this, status)} + /> + Type Created At
+ ); + } + + return ( +
+ Syncback Request Details +
+
+
this.close.call(this)}> + X +
+
+ {counts} + {details} +
+
+
+
+ ); + } + // else, the modal isn't open + return ( +
+ this.open.call(this)}> + Syncback Request Details + +
+ ); + } +} + +SyncbackRequestDetails.propTypes = { + accountId: React.PropTypes.number, +} + +window.SyncbackRequestDetails = SyncbackRequestDetails; diff --git a/packages/nylas-dashboard/routes/syncback-requests.js b/packages/nylas-dashboard/routes/syncback-requests.js new file mode 100644 index 000000000..267f1086a --- /dev/null +++ b/packages/nylas-dashboard/routes/syncback-requests.js @@ -0,0 +1,81 @@ +const Joi = require('joi'); +const {DatabaseConnector} = require(`nylas-core`); + +module.exports = (server) => { + server.route({ + method: 'GET', + path: '/syncback-requests/{account_id}', + config: { + description: 'Get the SyncbackRequests for an account', + notes: 'Notes go here', + tags: ['syncback-requests'], + validate: { + params: { + account_id: Joi.number().integer(), + }, + }, + response: { + schema: Joi.string(), + }, + }, + handler: (request, reply) => { + DatabaseConnector.forAccount(request.params.account_id).then((db) => { + const {SyncbackRequest} = db; + SyncbackRequest.findAll().then((syncbackRequests) => { + reply(JSON.stringify(syncbackRequests)) + }); + }); + }, + }); + + server.route({ + method: 'GET', + path: '/syncback-requests/{account_id}/counts', + config: { + description: 'Get stats on the statuses of SyncbackRequests', + notes: 'Notes go here', + tags: ['syncback-requests'], + validate: { + params: { + account_id: Joi.number().integer(), + }, + query: { + since: Joi.date().timestamp(), + }, + }, + response: { + schema: Joi.string(), + }, + }, + handler: (request, reply) => { + DatabaseConnector.forAccount(request.params.account_id).then((db) => { + const {SyncbackRequest} = db; + + const counts = { + 'new': null, + 'succeeded': null, + 'failed': null, + } + + const where = {}; + if (request.query.since) { + where.createdAt = {gt: request.query.since}; + } + + const countPromises = []; + for (const status of Object.keys(counts)) { + where.status = status.toUpperCase(); + countPromises.push( + SyncbackRequest.count({where: where}).then((count) => { + counts[status] = count; + }) + ); + } + + Promise.all(countPromises).then(() => { + reply(JSON.stringify(counts)); + }) + }); + }, + }); +}; From 8a63ace9e3f618a8e6873d1a86772e5eb330fc88 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Mon, 11 Jul 2016 13:27:44 -0700 Subject: [PATCH 211/800] Change config to watch just the 'packages' directory --- pm2-dev.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pm2-dev.yml b/pm2-dev.yml index 4f60094fd..b8efea594 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -4,7 +4,7 @@ apps: - script : packages/nylas-api/app.js - watch : true + watch : ["packages"] name : api env : PORT: 5100 @@ -15,14 +15,14 @@ apps: GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback" NODE_ENV: 'development' - script : packages/nylas-sync/app.js - watch : true + watch : ["packages"] name : sync env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" DB_ENCRYPTION_PASSWORD : "d6F3Efeq" NODE_ENV: 'development' - script : packages/nylas-dashboard/app.js - watch : true + watch : ["packages"] name : dashboard env : PORT: 5101 @@ -30,7 +30,7 @@ apps: DB_ENCRYPTION_PASSWORD : "d6F3Efeq" NODE_ENV: 'development' - script : packages/nylas-message-processor/app.js - watch : true + watch : ["packages"] name : processor env : DB_ENCRYPTION_ALGORITHM : "aes-256-ctr" From 6ff596b5a7df2e52f444bff7cbee503668ff26cc Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 13:46:50 -0700 Subject: [PATCH 212/800] Properly log uncaight errors in sync worker --- packages/nylas-sync/sync-worker.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index d0788ca78..2c11a5137 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -56,7 +56,8 @@ class SyncWorker { case MessageTypes.SYNCBACK_REQUESTED: this.syncNow({reason: 'Syncback Action Queued'}); break; default: - throw new Error(`Invalid message: ${msg}`) + const err = new Error(`Invalid message`) + this._logger.error({message: msg, err}, 'Invalid message') } } @@ -218,7 +219,9 @@ class SyncWorker { return Promise.resolve() } - throw new Error(`SyncWorker.onSyncDidComplete: Unknown afterSync behavior: ${afterSync}. Closing connection`) + this._logger.warn({after_sync: afterSync}, `SyncWorker.onSyncDidComplete: Unknown afterSync behavior`) + this.closeConnection() + return Promise.resolve() } isWaitingForNextSync() { From 5868fb520718f41e22ccf766171a4db368983d57 Mon Sep 17 00:00:00 2001 From: Annie Date: Mon, 11 Jul 2016 13:48:00 -0700 Subject: [PATCH 213/800] make changes requested before pull --- packages/nylas-api/routes/contacts.js | 3 +-- packages/nylas-message-processor/processors/contact.js | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/nylas-api/routes/contacts.js b/packages/nylas-api/routes/contacts.js index 778f348cf..c763d5b61 100644 --- a/packages/nylas-api/routes/contacts.js +++ b/packages/nylas-api/routes/contacts.js @@ -74,8 +74,7 @@ module.exports = (server) => { return reply(Serialization.jsonStringify(contact)) }) .catch((error) => { - console.log('Error fetching contacts: ', error) - reply(error) + request.info(error, 'Error fetching contacts') }) }) }, diff --git a/packages/nylas-message-processor/processors/contact.js b/packages/nylas-message-processor/processors/contact.js index 11d1423e1..7893c8738 100644 --- a/packages/nylas-message-processor/processors/contact.js +++ b/packages/nylas-message-processor/processors/contact.js @@ -1,12 +1,12 @@ class ContactProcessor { - verified(contact) { + verified(logger, contact) { // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages const regex = new RegExp(/^(noreply|no-reply|donotreply|mailer|support|webmaster|news(letter)?@)/ig) if (regex.test(contact.email) || contact.email.length > 60) { - console.log('Email address doesn\'t seem to be areal person') + logger.info('Email address doesn\'t seem to be areal person') return false } return true @@ -25,15 +25,16 @@ class ContactProcessor { } - processMessage({db, message}) { + processMessage({db, message, logger}) { const {Contact} = db; + this.logger = logger let allContacts = [] const fields = ['to', 'from', 'bcc', 'cc'] fields.forEach((field) => { allContacts = allContacts.concat(message[field]) }) - const filtered = allContacts.filter(this.verified) + const filtered = allContacts.filter(this.verified(logger)) const contactPromises = filtered.map((contact) => { return this.findOrCreateByContactId(Contact, contact, message.accountId) }) From b7301f5dc5e72c36ea6460ca6e21a174fe23e641 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 14:10:30 -0700 Subject: [PATCH 214/800] Fix more error handling in sync worker --- packages/nylas-sync/sync-worker.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 2c11a5137..0b7d760d4 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -56,8 +56,7 @@ class SyncWorker { case MessageTypes.SYNCBACK_REQUESTED: this.syncNow({reason: 'Syncback Action Queued'}); break; default: - const err = new Error(`Invalid message`) - this._logger.error({message: msg, err}, 'Invalid message') + this._logger.error({message: msg}, 'SyncWorker: Invalid message') } } @@ -65,10 +64,14 @@ class SyncWorker { if (!this.isWaitingForNextSync()) { return; } - this._getAccount().then((account) => { + this._getAccount() + .then((account) => { this._account = account; this.syncNow({reason: 'Account Modification'}); - }); + }) + .catch((err) => { + this._logger.error(err, 'SyncWorker: Error getting account for update') + }) } _onConnectionIdleUpdate() { @@ -210,18 +213,17 @@ class SyncWorker { if (afterSync === 'idle') { return this._getIdleFolder() .then((idleFolder) => this._conn.openBox(idleFolder.name)) - .then(() => this._logger.info('SyncWorker: - Idling on inbox category')) + .then(() => this._logger.info('SyncWorker: Idling on inbox category')) } if (afterSync === 'close') { - this._logger.info('SyncWorker: - Closing connection'); + this._logger.info('SyncWorker: Closing connection'); this.closeConnection() return Promise.resolve() } - this._logger.warn({after_sync: afterSync}, `SyncWorker.onSyncDidComplete: Unknown afterSync behavior`) - this.closeConnection() - return Promise.resolve() + this._logger.error({after_sync: afterSync}, `SyncWorker.onSyncDidComplete: Unknown afterSync behavior`) + throw new Error('SyncWorker.onSyncDidComplete: Unknown afterSync behavior') } isWaitingForNextSync() { From 1360599ffc921fb15a186cf585bbc4c13e1f1c69 Mon Sep 17 00:00:00 2001 From: Evan Morikawa Date: Mon, 11 Jul 2016 14:23:20 -0700 Subject: [PATCH 215/800] Fix deafult test account email address --- packages/nylas-api/routes/auth.js | 1 + packages/nylas-sync/app.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index 65c776927..d0cb6949e 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -76,6 +76,7 @@ module.exports = (server) => { validate: { query: { client_id: Joi.string().required(), + n1_id: Joi.string(), }, payload: { email: Joi.string().email().required(), diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index d0effc28f..14ae4c177 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -12,7 +12,7 @@ DatabaseConnector.forShared().then((db) => { Account.findAll().then((accounts) => { if (accounts.length === 0) { global.Logger.info(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) - global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) + global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) } manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => { manager.start(); From 0860f3027c87e2514fcbe4f947554f14542df158 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Mon, 11 Jul 2016 14:12:03 -0700 Subject: [PATCH 216/800] Fix two small errors --- packages/nylas-sync/imap/fetch-messages-in-folder.js | 2 +- packages/nylas-sync/sync-worker.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js index 0bf994f61..038dde66d 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-folder.js +++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js @@ -185,7 +185,7 @@ class FetchMessagesInFolder { this._logger.info({ key, num_messages: uids.length, - }`FetchMessagesInFolder: Fetching parts for messages`) + }, `FetchMessagesInFolder: Fetching parts for messages`) // note: the order of UIDs in the array doesn't matter, Gmail always // returns them in ascending (oldest => newest) order. diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 0b7d760d4..7fb2efca1 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -55,6 +55,9 @@ class SyncWorker { this._onAccountUpdated(); break; case MessageTypes.SYNCBACK_REQUESTED: this.syncNow({reason: 'Syncback Action Queued'}); break; + case MessageTypes.ACCOUNT_CREATED: + // No other processing currently required for account creation + break; default: this._logger.error({message: msg}, 'SyncWorker: Invalid message') } From 04ab0d9034f8e0cc38e1093a964d7c353ba6263f Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Mon, 11 Jul 2016 14:16:39 -0700 Subject: [PATCH 217/800] Change firstSyncCompletedAt to firstSyncCompletion, now explicitly 14 bytes. --- packages/nylas-api/serialization.js | 2 +- packages/nylas-core/models/shared/account.js | 4 ++-- packages/nylas-dashboard/public/js/app.jsx | 4 ++-- packages/nylas-sync/sync-worker.js | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js index 5d6a1c07c..3b0fa5f3b 100644 --- a/packages/nylas-api/serialization.js +++ b/packages/nylas-api/serialization.js @@ -22,7 +22,7 @@ function jsonSchema(modelName) { // connection_settings: Joi.object(), // sync_policy: Joi.object(), // sync_error: Joi.object().allow(null), - // first_sync_completed_at: Joi.number().allow(null), + // first_sync_completion: Joi.number().allow(null), // last_sync_completions: Joi.array(), // }) } diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js index e5b203666..9a61101a9 100644 --- a/packages/nylas-core/models/shared/account.js +++ b/packages/nylas-core/models/shared/account.js @@ -12,7 +12,7 @@ module.exports = (sequelize, Sequelize) => { connectionCredentials: Sequelize.TEXT, syncPolicy: JSONType('syncPolicy'), syncError: JSONType('syncError', {defaultValue: null}), - firstSyncCompletedAt: Sequelize.INTEGER, + firstSyncCompletion: Sequelize.INTEGER(14), lastSyncCompletions: JSONARRAYType('lastSyncCompletions'), }, { classMethods: { @@ -31,7 +31,7 @@ module.exports = (sequelize, Sequelize) => { connection_settings: this.connectionSettings, sync_policy: this.syncPolicy, sync_error: this.syncError, - first_sync_completed_at: this.firstSyncCompletedAt, + first_sync_completion: this.firstSyncCompletion, last_sync_completions: this.lastSyncCompletions, created_at: this.createdAt, } diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index 660df19e8..bed514410 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -44,8 +44,8 @@ class Account extends React.Component { const timeSinceLastSync = (Date.now() - newestSync) / 1000; let firstSyncDuration = "Incomplete"; - if (account.first_sync_completed_at) { - firstSyncDuration = (new Date(account.first_sync_completed_at) - new Date(account.created_at)) / 1000; + if (account.first_sync_completion) { + firstSyncDuration = (new Date(account.first_sync_completion) - new Date(account.created_at)) / 1000; } return ( diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 7fb2efca1..1dfb377b9 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -197,8 +197,8 @@ class SyncWorker { onSyncDidComplete() { const {afterSync} = this._account.syncPolicy; - if (!this._account.firstSyncCompletedAt) { - this._account.firstSyncCompletedAt = Date.now() + if (!this._account.firstSyncCompletion) { + this._account.firstSyncCompletion = Date.now() } const now = Date.now(); From bc10ff453e57c9375ba6c50a1909456fb156de6a Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Mon, 11 Jul 2016 15:06:36 -0700 Subject: [PATCH 218/800] Change stop script to use pm2 stop command instead of delete --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7a33802a6..d41b6612e 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "scripts": { "start": "pm2 start ./pm2-dev.yml --watch; pm2 logs --raw | bunyan -o short", "logs": "pm2 logs --raw | bunyan -o short", - "stop": "pm2 delete all", + "stop": "pm2 stop all", "postinstall": "lerna bootstrap" }, "repository": { From 960dbdeb8f995d209e4cde7e6ce0be446e44633b Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 15:40:03 -0700 Subject: [PATCH 219/800] Update logger for sync process manager - Log identity data always --- packages/nylas-sync/sync-process-manager.js | 28 +++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index d9cab0829..560bc1fad 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -40,10 +40,11 @@ class SyncProcessManager { this._workers = {}; this._listenForSyncsClient = null; this._exiting = false; + this._logger = global.Logger.child({identity: IDENTITY}) } start() { - global.Logger.info(`ProcessManager: Starting with ID ${IDENTITY}`) + this._logger.info(`ProcessManager: Starting with ID`) this.unassignAccountsAssignedTo(IDENTITY).then(() => { this.unassignAccountsMissingHeartbeats(); @@ -63,12 +64,12 @@ class SyncProcessManager { client.setAsync(key, Date.now()).then(() => client.expireAsync(key, HEARTBEAT_EXPIRES) ).then(() => - global.Logger.info("ProcessManager: 💘") + this._logger.info("ProcessManager: 💘") ) } onSigInt() { - global.Logger.info(`ProcessManager: Exiting...`) + this._logger.info(`ProcessManager: Exiting...`) this._exiting = true; this.unassignAccountsAssignedTo(IDENTITY).then(() => @@ -85,7 +86,7 @@ class SyncProcessManager { let unseenIds = [].concat(accountIds); - global.Logger.info("ProcessManager: Starting scan for accountIds in database that are not present in Redis.") + this._logger.info("ProcessManager: Starting scan for accountIds in database that are not present in Redis.") return forEachAccountList((foundProcessIdentity, foundIds) => { unseenIds = unseenIds.filter((a) => !foundIds.includes(`${a}`)) @@ -94,7 +95,10 @@ class SyncProcessManager { if (unseenIds.length === 0) { return; } - global.Logger.info(`ProcessManager: Adding account IDs ${unseenIds.join(',')} to ${ACCOUNTS_UNCLAIMED}.`) + this._logger.info({ + unseen_ids: unseenIds.join(', '), + channel: ACCOUNTS_UNCLAIMED, + }, `ProcessManager: Adding unseen account IDs to ACCOUNTS_UNCLAIMED channel.`) unseenIds.map((id) => client.lpushAsync(ACCOUNTS_UNCLAIMED, id)); }); } @@ -102,7 +106,7 @@ class SyncProcessManager { unassignAccountsMissingHeartbeats() { const client = PubsubConnector.broadcastClient(); - global.Logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.") + this._logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.") Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => { const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, ''); @@ -125,12 +129,15 @@ class SyncProcessManager { ) return unassignOne(0).then((returned) => { - global.Logger.info(`ProcessManager: Returned ${returned} accounts assigned to ${identity}.`) + this._logger.info({ + returned, + assigned_to: identity, + }, `ProcessManager: Returned accounts`) }); } update() { - global.Logger.info(`ProcessManager: Searching for an unclaimed account to sync.`) + this._logger.info(`ProcessManager: Searching for an unclaimed account to sync.`) this.acceptUnclaimedAccount().finally(() => { if (this._exiting) { @@ -170,7 +177,7 @@ class SyncProcessManager { if (this._exiting || this._workers[account.id]) { return; } - global.Logger.info(`ProcessManager: Starting worker for Account ${accountId}`) + this._logger.info({account_id: accountId}, `ProcessManager: Starting worker for Account`) this._workers[account.id] = new SyncWorker(account, db, () => { this.removeWorkerForAccountId(accountId) }); @@ -187,7 +194,8 @@ class SyncProcessManager { if (didRemove) { PubsubConnector.broadcastClient().rpushAsync(dst, accountId) } else { - throw new Error("Wanted to return item to pool, but didn't have claim on it.") + this._logger.error("Wanted to return item to pool, but didn't have claim on it.") + return } this._workers[accountId] = null; }); From d4cd2510db76c5d9dcfc17cf5755d0c93852c1a5 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 15:46:52 -0700 Subject: [PATCH 220/800] Fix pm2 run scripts --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index d41b6612e..f95dacb81 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,9 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "pm2 start ./pm2-dev.yml --watch; pm2 logs --raw | bunyan -o short", + "start": "pm2 kill && pm2 start ./pm2-dev.yml --watch && pm2 logs --raw | bunyan -o short", "logs": "pm2 logs --raw | bunyan -o short", - "stop": "pm2 stop all", + "stop": "pm2 kill", "postinstall": "lerna bootstrap" }, "repository": { From 3c63268eecfb39890d2b64b873cdb84c8e646797 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 16:01:18 -0700 Subject: [PATCH 221/800] Return a new token for existing account instead of duplicating accounts --- packages/nylas-api/routes/auth.js | 57 ++++++++++++++-------- packages/nylas-core/imap-connection.js | 7 ++- packages/nylas-dashboard/public/js/app.jsx | 2 + packages/nylas-sync/sync-worker.js | 7 ++- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index d0cb6949e..5cd1e2fb4 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -43,24 +43,33 @@ const buildAccountWith = ({name, email, provider, settings, credentials}) => { return DatabaseConnector.forShared().then((db) => { const {AccountToken, Account} = db; - const account = Account.build({ - name: name, - provider: provider, - emailAddress: email, - connectionSettings: settings, - syncPolicy: SyncPolicy.defaultPolicy(), - lastSyncCompletions: [], - }) - account.setCredentials(credentials); + return Account.find({ + where: { + emailAddress: email, + connectionSettings: JSON.stringify(settings), + }, + }).then((existing) => { + const account = existing || Account.build({ + name: name, + provider: provider, + emailAddress: email, + connectionSettings: settings, + syncPolicy: SyncPolicy.defaultPolicy(), + lastSyncCompletions: [], + }) - return account.save().then((saved) => - AccountToken.create({accountId: saved.id}).then((token) => - DatabaseConnector.prepareAccountDatabase(saved.id).thenReturn({ - account: saved, - token: token, - }) - ) - ); + // always update with the latest credentials + account.setCredentials(credentials); + + return account.save().then((saved) => + AccountToken.create({accountId: saved.id}).then((token) => + DatabaseConnector.prepareAccountDatabase(saved.id).thenReturn({ + account: saved, + token: token, + }) + ) + ); + }); }); } @@ -76,7 +85,6 @@ module.exports = (server) => { validate: { query: { client_id: Joi.string().required(), - n1_id: Joi.string(), }, payload: { email: Joi.string().email().required(), @@ -98,7 +106,11 @@ module.exports = (server) => { const {settings, email, provider, name} = request.payload; if (provider === 'imap') { - connectionChecks.push(IMAPConnection.connect({db: dbStub, settings})) + connectionChecks.push(IMAPConnection.connect({ + logger: request.logger, + settings: settings, + db: dbStub, + })); } Promise.all(connectionChecks).then(() => { @@ -187,9 +199,12 @@ module.exports = (server) => { client_id: GMAIL_CLIENT_ID, client_secret: GMAIL_CLIENT_SECRET, } - Promise.all([ - IMAPConnection.connect({db: {}, settings: Object.assign({}, settings, credentials)}), + IMAPConnection.connect({ + logger: request.logger, + settings: Object.assign({}, settings, credentials), + db: {}, + }), ]) .then(() => buildAccountWith({ diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index 05e703b49..d833a7936 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -156,8 +156,6 @@ class IMAPBox { } return this._imap.closeBoxAsync(expunge) } - - } @@ -178,6 +176,11 @@ class IMAPConnection extends EventEmitter { constructor({db, settings, logger = console} = {}) { super(); + + if (!(settings instanceof Object)) { + throw new Error("IMAPConnection: Must be instantiated with `settings`") + } + this._logger = logger; this._db = db; this._queue = []; diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index bed514410..ecf7a1f7d 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -1,4 +1,6 @@ /* eslint react/react-in-jsx-scope: 0*/ +/* eslint no-console: 0*/ + const React = window.React; const ReactDOM = window.ReactDOM; const { diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 1dfb377b9..7178e59cf 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -108,7 +108,12 @@ class SyncWorker { return Promise.reject(new Error("ensureConnection: There are no IMAP connection credentials for this account.")) } - const conn = new IMAPConnection({db: this._db, settings: Object.assign({}, settings, credentials), logger: this._logger}); + const conn = new IMAPConnection({ + db: this._db, + settings: Object.assign({}, settings, credentials), + logger: this._logger, + }); + conn.on('mail', () => { this._onConnectionIdleUpdate(); }) From 8e0cd92bad7fe719178586ae7064deb99a3d965b Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 16:02:53 -0700 Subject: [PATCH 222/800] Require a logger when instantiating a connection --- packages/nylas-core/imap-connection.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index d833a7936..c692f11bf 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -174,12 +174,15 @@ class IMAPConnection extends EventEmitter { return new IMAPConnection(...args).connect() } - constructor({db, settings, logger = console} = {}) { + constructor({db, settings, logger} = {}) { super(); if (!(settings instanceof Object)) { throw new Error("IMAPConnection: Must be instantiated with `settings`") } + if (!logger) { + throw new Error("IMAPConnection: Must be instantiated with `logger`") + } this._logger = logger; this._db = db; From 2dc31cb576cdf2dd6dbae0f23b494e2978a0bc10 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 15:50:55 -0700 Subject: [PATCH 223/800] Add number of syncing accounts to sync process manager heartbeat --- packages/nylas-sync/sync-process-manager.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index 560bc1fad..8d4042895 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -64,7 +64,9 @@ class SyncProcessManager { client.setAsync(key, Date.now()).then(() => client.expireAsync(key, HEARTBEAT_EXPIRES) ).then(() => - this._logger.info("ProcessManager: 💘") + this._logger.info({ + accounts_syncing_count: Object.keys(this._workers).length, + }, "ProcessManager: 💘") ) } From ae51646de62dc2a46ddbc2b1544f2531100b7b0f Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 16:33:56 -0700 Subject: [PATCH 224/800] Comment out new relic for now --- packages/nylas-api/app.js | 2 +- packages/nylas-sync/app.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index fbc8aedfa..dc9b08e08 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -1,4 +1,4 @@ -require('newrelic'); +// require('newrelic'); const Hapi = require('hapi'); const HapiSwagger = require('hapi-swagger'); diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index 14ae4c177..ac81a02f1 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -1,4 +1,4 @@ -require('newrelic'); +// require('newrelic'); global.Promise = require('bluebird'); const {DatabaseConnector, Logger} = require(`nylas-core`) const SyncProcessManager = require('./sync-process-manager'); From e673573fcdc5bf3d96af89de7f35f92b26dba26c Mon Sep 17 00:00:00 2001 From: Annie Date: Mon, 11 Jul 2016 16:37:39 -0700 Subject: [PATCH 225/800] added folder view api route --- packages/nylas-api/routes/auth.js | 1 + packages/nylas-api/routes/categories.js | 1 + packages/nylas-sync/app.js | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index 65c776927..d0cb6949e 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -76,6 +76,7 @@ module.exports = (server) => { validate: { query: { client_id: Joi.string().required(), + n1_id: Joi.string(), }, payload: { email: Joi.string().email().required(), diff --git a/packages/nylas-api/routes/categories.js b/packages/nylas-api/routes/categories.js index 7b62141f8..d0143cda6 100644 --- a/packages/nylas-api/routes/categories.js +++ b/packages/nylas-api/routes/categories.js @@ -17,6 +17,7 @@ module.exports = (server) => { query: { limit: Joi.number().integer().min(1).max(2000).default(100), offset: Joi.number().integer().min(0).default(0), + view: Joi.string().valid('expanded', 'count'), }, }, response: { diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index d0effc28f..14ae4c177 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -12,7 +12,7 @@ DatabaseConnector.forShared().then((db) => { Account.findAll().then((accounts) => { if (accounts.length === 0) { global.Logger.info(`Couldn't find any accounts to sync. Run this CURL command to auth one!`) - global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest2@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) + global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"`) } manager.ensureAccountIDsInRedis(accounts.map(a => a.id)).then(() => { manager.start(); From f8b8dfb87f381380f78055064b54624a724b4adb Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 16:40:48 -0700 Subject: [PATCH 226/800] Revert to running pm2 in no-daemon mode in dev - Add a stream for pretty logging in dev mode --- package.json | 4 ++-- packages/nylas-core/logger.js | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index f95dacb81..625d7d2a7 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "devDependencies": { "babel-eslint": "6.x", + "bunyan-prettystream": "^0.1.3", "eslint": "2.x", "eslint-config-airbnb": "8.x", "eslint-plugin-import": "1.x", @@ -27,8 +28,7 @@ "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz" }, "scripts": { - "start": "pm2 kill && pm2 start ./pm2-dev.yml --watch && pm2 logs --raw | bunyan -o short", - "logs": "pm2 logs --raw | bunyan -o short", + "start": "pm2 start ./pm2-dev.yml --no-daemon", "stop": "pm2 kill", "postinstall": "lerna bootstrap" }, diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js index 40c73ad31..1bdcb9002 100644 --- a/packages/nylas-core/logger.js +++ b/packages/nylas-core/logger.js @@ -1,17 +1,25 @@ const bunyan = require('bunyan') const createCWStream = require('bunyan-cloudwatch') +const PrettyStream = require('bunyan-prettystream'); const NODE_ENV = process.env.NODE_ENV || 'unknown' function getLogStreams(name, env) { + if (env === 'development') { + const prettyStdOut = new PrettyStream(); + prettyStdOut.pipe(process.stdout); + const stdoutStream = { + type: 'raw', + level: 'debug', + stream: prettyStdOut, + } + return [stdoutStream] + } + const stdoutStream = { stream: process.stdout, level: 'info', } - if (env === 'development') { - return [stdoutStream] - } - const cloudwatchStream = { stream: createCWStream({ logGroupName: `k2-${env}`, From 492ac21bb6874bc2a61485a2bd7b605519ee8463 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 16:56:18 -0700 Subject: [PATCH 227/800] Primitive account deletion via DELETE /account --- packages/nylas-api/routes/accounts.js | 25 +++++++++++++++++++++ packages/nylas-core/database-connector.js | 18 ++++++++++----- packages/nylas-core/hook-account-crud.js | 10 ++++++--- packages/nylas-core/imap-connection.js | 6 +++-- packages/nylas-core/message-types.js | 1 + packages/nylas-sync/sync-process-manager.js | 1 + packages/nylas-sync/sync-worker.js | 19 +++++++++++----- 7 files changed, 64 insertions(+), 16 deletions(-) diff --git a/packages/nylas-api/routes/accounts.js b/packages/nylas-api/routes/accounts.js index 6f0ca9d61..856bd5a66 100644 --- a/packages/nylas-api/routes/accounts.js +++ b/packages/nylas-api/routes/accounts.js @@ -1,4 +1,5 @@ const Serialization = require('../serialization'); +const {DatabaseConnector} = require('nylas-core'); module.exports = (server) => { server.route({ @@ -21,4 +22,28 @@ module.exports = (server) => { reply(Serialization.jsonStringify(account)); }, }); + + server.route({ + method: 'DELETE', + path: '/account', + config: { + description: 'Deletes the current account and all data from the Nylas Cloud.', + notes: 'Notes go here', + tags: ['accounts'], + validate: { + params: { + }, + }, + }, + handler: (request, reply) => { + const account = request.auth.credentials; + account.destroy().then((saved) => + DatabaseConnector.destroyAccountDatabase(saved.id).then(() => + reply(Serialization.jsonStringify({status: 'success'})) + ) + ).catch((err) => { + reply(err).code(500); + }) + }, + }); }; diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index b37fe81cf..0f87e1fa0 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -79,11 +79,7 @@ class DatabaseConnector { const dbname = `a-${accountId}`; if (process.env.DB_HOSTNAME) { - const sequelize = new Sequelize(null, process.env.DB_USERNAME, process.env.DB_PASSWORD, { - host: process.env.DB_HOSTNAME, - dialect: "mysql", - logging: false, - }) + const sequelize = this._sequelizePoolForDatabase(null); return sequelize.authenticate().then(() => sequelize.query(`CREATE DATABASE \`${dbname}\``) ); @@ -91,6 +87,18 @@ class DatabaseConnector { return Promise.resolve() } + destroyAccountDatabase(accountId) { + const dbname = `a-${accountId}`; + if (process.env.DB_HOSTNAME) { + const sequelize = this._sequelizePoolForDatabase(null); + return sequelize.authenticate().then(() => + sequelize.query(`CREATE DATABASE \`${dbname}\``) + ); + } + fs.removeFileSync(path.join(STORAGE_DIR, `${dbname}.sqlite`)); + return Promise.resolve() + } + _sequelizeForShared() { const sequelize = this._sequelizePoolForDatabase(`shared`); const modelsPath = path.join(__dirname, 'models/shared'); diff --git a/packages/nylas-core/hook-account-crud.js b/packages/nylas-core/hook-account-crud.js index 016569b07..5efc658f6 100644 --- a/packages/nylas-core/hook-account-crud.js +++ b/packages/nylas-core/hook-account-crud.js @@ -17,7 +17,11 @@ module.exports = (db, sequelize) => { }); } }) - // TODO delete account from redis - // sequelize.addHook("afterDelete", ({dataValues, $modelOptions}) => { - // }) + sequelize.addHook("afterDestroy", ({dataValues, $modelOptions}) => { + if ($modelOptions.name.singular === 'account') { + PubsubConnector.notifyAccount(dataValues.id, { + type: MessageTypes.ACCOUNT_DELETED, + }); + } + }) } diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index c692f11bf..bbbef526f 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -270,9 +270,11 @@ class IMAPConnection extends EventEmitter { } end() { + if (this._imap) { + this._imap.end(); + this._imap = null; + } this._queue = []; - this._imap.end(); - this._imap = null; this._connectPromise = null; } diff --git a/packages/nylas-core/message-types.js b/packages/nylas-core/message-types.js index 2b4889a8c..43bbb3ba6 100644 --- a/packages/nylas-core/message-types.js +++ b/packages/nylas-core/message-types.js @@ -1,5 +1,6 @@ module.exports = { ACCOUNT_CREATED: "ACCOUNT_CREATED", ACCOUNT_UPDATED: "ACCOUNT_UPDATED", + ACCOUNT_DELETED: "ACCOUNT_DELETED", SYNCBACK_REQUESTED: "SYNCBACK_REQUESTED", } diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index 560bc1fad..2ca05e354 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -178,6 +178,7 @@ class SyncProcessManager { return; } this._logger.info({account_id: accountId}, `ProcessManager: Starting worker for Account`) + this._workers[account.id] = new SyncWorker(account, db, () => { this.removeWorkerForAccountId(accountId) }); diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 7178e59cf..943b9f986 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -27,7 +27,6 @@ class SyncWorker { this._logger = global.Logger.forAccount(account) this._syncTimer = null; - this._expirationTimer = null; this._destroyed = false; this.syncNow({reason: 'Initial'}); @@ -37,6 +36,8 @@ class SyncWorker { } cleanup() { + clearTimeout(this._syncTimer); + this._syncTimer = null; this._destroyed = true; this._listener.dispose(); this.closeConnection() @@ -51,13 +52,19 @@ class SyncWorker { _onMessage(msg) { const {type} = JSON.parse(msg); switch (type) { - case MessageTypes.ACCOUNT_UPDATED: - this._onAccountUpdated(); break; - case MessageTypes.SYNCBACK_REQUESTED: - this.syncNow({reason: 'Syncback Action Queued'}); break; case MessageTypes.ACCOUNT_CREATED: // No other processing currently required for account creation break; + case MessageTypes.ACCOUNT_UPDATED: + this._onAccountUpdated(); + break; + case MessageTypes.ACCOUNT_DELETED: + this.cleanup(); + this._onExpired(); + break; + case MessageTypes.SYNCBACK_REQUESTED: + this.syncNow({reason: 'Syncback Action Queued'}); + break; default: this._logger.error({message: msg}, 'SyncWorker: Invalid message') } @@ -208,7 +215,7 @@ class SyncWorker { const now = Date.now(); const syncGraphTimeLength = 60 * 30; // 30 minutes, should be the same as SyncGraph.config.timeLength - let lastSyncCompletions = [...this._account.lastSyncCompletions] + let lastSyncCompletions = [].concat(this._account.lastSyncCompletions) lastSyncCompletions = [now, ...lastSyncCompletions] while (now - lastSyncCompletions[lastSyncCompletions.length - 1] > 1000 * syncGraphTimeLength) { lastSyncCompletions.pop(); From 826db2e6f4516cdf169d894e1df378b3ea69566a Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 16:57:27 -0700 Subject: [PATCH 228/800] Remove redis from pm2 --- pm2-dev.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pm2-dev.yml b/pm2-dev.yml index b8efea594..21c1ea226 100644 --- a/pm2-dev.yml +++ b/pm2-dev.yml @@ -1,8 +1,4 @@ apps: - - script : redis-server - name : redis - - - script : packages/nylas-api/app.js watch : ["packages"] name : api From 039648bae84a9da8d40e0b2e79e150b074161cae Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 17:04:05 -0700 Subject: [PATCH 229/800] Add bunyan-prettystream to regular dependencies --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 625d7d2a7..4073a8ef4 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "bluebird": "3.x.x", "bunyan": "1.8.0", "bunyan-cloudwatch": "2.0.0", + "bunyan-prettystream": "^0.1.3", "lerna": "2.0.0-beta.23", "mysql": "^2.11.1", "newrelic": "^1.28.1", @@ -18,7 +19,6 @@ }, "devDependencies": { "babel-eslint": "6.x", - "bunyan-prettystream": "^0.1.3", "eslint": "2.x", "eslint-config-airbnb": "8.x", "eslint-plugin-import": "1.x", From 413a77c3232157673b7b0d27bda17adfa66a9c67 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 17:42:49 -0700 Subject: [PATCH 230/800] Log unhandled errors in api and sync --- packages/nylas-api/app.js | 4 ++++ packages/nylas-sync/app.js | 4 ++++ packages/nylas-sync/sync-process-manager.js | 14 +++++++------- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js index dc9b08e08..8199a9687 100644 --- a/packages/nylas-api/app.js +++ b/packages/nylas-api/app.js @@ -14,6 +14,10 @@ const {DatabaseConnector, SchedulerUtils, Logger} = require(`nylas-core`); global.Promise = require('bluebird'); global.Logger = Logger.createLogger('nylas-k2-api') +const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error') +process.on('uncaughtException', onUnhandledError) +process.on('unhandledRejection', onUnhandledError) + const server = new Hapi.Server({ connections: { router: { diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js index ac81a02f1..16bf4509f 100644 --- a/packages/nylas-sync/app.js +++ b/packages/nylas-sync/app.js @@ -5,6 +5,10 @@ const SyncProcessManager = require('./sync-process-manager'); global.Logger = Logger.createLogger('nylas-k2-sync') +const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error') +process.on('uncaughtException', onUnhandledError) +process.on('unhandledRejection', onUnhandledError) + const manager = new SyncProcessManager(); DatabaseConnector.forShared().then((db) => { diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js index c32d2aa8b..3d205c384 100644 --- a/packages/nylas-sync/sync-process-manager.js +++ b/packages/nylas-sync/sync-process-manager.js @@ -61,13 +61,13 @@ class SyncProcessManager { updateHeartbeat() { const key = HEARTBEAT_FOR(IDENTITY); const client = PubsubConnector.broadcastClient(); - client.setAsync(key, Date.now()).then(() => - client.expireAsync(key, HEARTBEAT_EXPIRES) - ).then(() => - this._logger.info({ - accounts_syncing_count: Object.keys(this._workers).length, - }, "ProcessManager: 💘") - ) + client.setAsync(key, Date.now()) + .then(() => client.expireAsync(key, HEARTBEAT_EXPIRES)) + .then(() => { + this._logger.info({ + accounts_syncing_count: Object.keys(this._workers).length, + }, "ProcessManager: 💘") + }) } onSigInt() { From afeb0e7b27ad9aa1f5d0e9730b6eb2420ce4f188 Mon Sep 17 00:00:00 2001 From: Juan Tejada Date: Mon, 11 Jul 2016 17:57:43 -0700 Subject: [PATCH 231/800] Create one CloudWatch stream per host - This prevents `InvalidSequenceTokenException` when running in production with multiple machines in an elastic beanstalk environment - See https://github.com/mirkokiefer/bunyan-cloudwatch/issues/1#issuecomment-193116799 --- packages/nylas-core/logger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js index 1bdcb9002..b77ba2c29 100644 --- a/packages/nylas-core/logger.js +++ b/packages/nylas-core/logger.js @@ -1,3 +1,4 @@ +const os = require('os'); const bunyan = require('bunyan') const createCWStream = require('bunyan-cloudwatch') const PrettyStream = require('bunyan-prettystream'); @@ -23,7 +24,7 @@ function getLogStreams(name, env) { const cloudwatchStream = { stream: createCWStream({ logGroupName: `k2-${env}`, - logStreamName: `${name}-${env}`, + logStreamName: `${name}-${env}-${os.hostname()}`, cloudWatchLogsOptions: { region: 'us-east-1', }, From 73fc371825f2cbfbe7bb289306da308a9aa85de5 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 18:24:18 -0700 Subject: [PATCH 232/800] =?UTF-8?q?Make=20the=20graphs=20on=20the=20dashbo?= =?UTF-8?q?ard=20retina=20=F0=9F=92=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../nylas-dashboard/public/js/sync-graph.jsx | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/packages/nylas-dashboard/public/js/sync-graph.jsx b/packages/nylas-dashboard/public/js/sync-graph.jsx index 7c1a1fc15..abde9a400 100644 --- a/packages/nylas-dashboard/public/js/sync-graph.jsx +++ b/packages/nylas-dashboard/public/js/sync-graph.jsx @@ -4,17 +4,36 @@ const ReactDOM = window.ReactDOM; class SyncGraph extends React.Component { componentDidMount() { - this.drawGraph(); - } - - componentDidUpdate() { this.drawGraph(true); } - drawGraph(isUpdate) { + componentDidUpdate() { + this.drawGraph(false); + } + + drawGraph(isInitial) { const now = Date.now(); const config = SyncGraph.config; - const context = ReactDOM.findDOMNode(this).getContext('2d'); + const node = ReactDOM.findDOMNode(this); + const context = node.getContext('2d'); + + if (isInitial) { + const totalHeight = config.height + config.labelFontSize + config.labelTopMargin; + node.width = config.width * 2; + node.height = totalHeight * 2; + node.style.width = `${config.width}px`; + node.style.height = `${totalHeight}px`; + context.scale(2, 2); + + // Axis labels + context.fillStyle = config.labelColor; + context.font = `${config.labelFontSize}px sans-serif`; + const fontY = config.height + config.labelFontSize + config.labelTopMargin; + const nowText = "now"; + const nowWidth = context.measureText(nowText).width; + context.fillText(nowText, config.width - nowWidth - 1, fontY); + context.fillText("-30m", 1, fontY); + } // Background // (This hides any previous data points, so we don't have to clear the canvas) @@ -25,6 +44,7 @@ class SyncGraph extends React.Component { const pxPerSec = config.width / config.timeLength; context.strokeStyle = config.dataColor; context.beginPath(); + for (const syncTimeMs of this.props.syncTimestamps) { const secsAgo = (now - syncTimeMs) / 1000; const pxFromRight = secsAgo * pxPerSec; @@ -43,17 +63,6 @@ class SyncGraph extends React.Component { context.lineTo(px, config.height); } context.stroke(); - - // Axis labels - if (!isUpdate) { // only draw these on the initial render - context.fillStyle = config.labelColor; - context.font = `${config.labelFontSize}px sans-serif`; - const fontY = config.height + config.labelFontSize + config.labelTopMargin; - const nowText = "now"; - const nowWidth = context.measureText(nowText).width; - context.fillText(nowText, config.width - nowWidth - 1, fontY); - context.fillText("-30m", 1, fontY); - } } render() { @@ -82,7 +91,7 @@ SyncGraph.config = { labelTopMargin: 2, labelColor: 'black', backgroundColor: 'black', - dataColor: 'blue', + dataColor: '#43a1ff', } SyncGraph.propTypes = { From d575e63ede6b983865ba9b0ae7b932d90d4d7248 Mon Sep 17 00:00:00 2001 From: Annie Date: Mon, 11 Jul 2016 18:30:08 -0700 Subject: [PATCH 233/800] replaced console logs with new logger --- packages/nylas-message-processor/processors/contact.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/nylas-message-processor/processors/contact.js b/packages/nylas-message-processor/processors/contact.js index 7893c8738..e2482df35 100644 --- a/packages/nylas-message-processor/processors/contact.js +++ b/packages/nylas-message-processor/processors/contact.js @@ -1,12 +1,11 @@ class ContactProcessor { - verified(logger, contact) { + verified(contact) { // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages const regex = new RegExp(/^(noreply|no-reply|donotreply|mailer|support|webmaster|news(letter)?@)/ig) if (regex.test(contact.email) || contact.email.length > 60) { - logger.info('Email address doesn\'t seem to be areal person') return false } return true @@ -34,7 +33,7 @@ class ContactProcessor { fields.forEach((field) => { allContacts = allContacts.concat(message[field]) }) - const filtered = allContacts.filter(this.verified(logger)) + const filtered = allContacts.filter(this.verified) const contactPromises = filtered.map((contact) => { return this.findOrCreateByContactId(Contact, contact, message.accountId) }) From caab8474f76c9342471466a776b552c105cc085b Mon Sep 17 00:00:00 2001 From: Christine Spang Date: Mon, 11 Jul 2016 18:36:08 -0700 Subject: [PATCH 234/800] Add Linux setup instructions to README.md --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 76f21b673..f998912d6 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,17 @@ 5. Install Node 6+ via NVM: `nvm install 6` 6. Install Redis locally `brew install redis` +## New Computer (Linux - Debian/Ubuntu): +1. Install Node 6+ via NodeSource (trusted): + 1. `curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -` + 1. `sudo apt-get install -y nodejs` +2. Install Redis locally `sudo apt-get install -y redis-server redis-tools` + ## New to AWS: 1. Install [Elastic Beanstalk CLI](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html#eb-cli3-install-osx): `sudo pip install awsebcli` -2. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli` + 1. On Linux, you may need to install Python 3's pip via `sudo apt-get install python3-pip` and then run `pip3 install --user awsebcli`. This installs to your home directory and you need to have `~/.local/bin` in your $PATH. +2. Install [AWS CLI](https://aws.amazon.com/cli/): `brew install awscli` on Mac and `pip install --user awscli` on Linux. 1. Add your AWS IAM Security Credentials to `aws configure`. 1. These are at Console Home -> IAM -> Users -> {{Your Name}} -> Security Credentials. Note that your private key was only shown unpon creation. If From 4fb848e91f4452cf0f5513d9edccdaf232750835 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Mon, 11 Jul 2016 21:38:05 -0700 Subject: [PATCH 235/800] fix for re-authing the same account --- packages/nylas-api/routes/auth.js | 2 +- packages/nylas-core/database-connector.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js index b6ed5cbb8..e33f29a24 100644 --- a/packages/nylas-api/routes/auth.js +++ b/packages/nylas-api/routes/auth.js @@ -63,7 +63,7 @@ const buildAccountWith = ({name, email, provider, settings, credentials}) => { return account.save().then((saved) => AccountToken.create({accountId: saved.id}).then((token) => - DatabaseConnector.prepareAccountDatabase(saved.id).thenReturn({ + DatabaseConnector.ensureAccountDatabase(saved.id).thenReturn({ account: saved, token: token, }) diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js index 0f87e1fa0..0a025a298 100644 --- a/packages/nylas-core/database-connector.js +++ b/packages/nylas-core/database-connector.js @@ -75,13 +75,13 @@ class DatabaseConnector { return this._pools[accountId]; } - prepareAccountDatabase(accountId) { + ensureAccountDatabase(accountId) { const dbname = `a-${accountId}`; if (process.env.DB_HOSTNAME) { const sequelize = this._sequelizePoolForDatabase(null); return sequelize.authenticate().then(() => - sequelize.query(`CREATE DATABASE \`${dbname}\``) + sequelize.query(`CREATE DATABASE IF NOT EXISTS \`${dbname}\``) ); } return Promise.resolve() From b44272621d99f9bcaa1a25736c62b62745710b81 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 12 Jul 2016 00:59:41 -0700 Subject: [PATCH 236/800] Misc cleanup, put account ids on dashboard --- packages/nylas-dashboard/public/js/app.jsx | 2 +- packages/nylas-message-processor/processors/contact.js | 3 +++ packages/nylas-sync/sync-worker.js | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index ecf7a1f7d..efa32b442 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -52,7 +52,7 @@ class Account extends React.Component { return (
-

{account.email_address} {active ? '🌕' : '🌑'}

+

{account.email_address} [{account.id}] {active ? '🌕' : '🌑'}

{assignment} 60) { return false } diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js index 943b9f986..8a7a9fd0f 100644 --- a/packages/nylas-sync/sync-worker.js +++ b/packages/nylas-sync/sync-worker.js @@ -197,8 +197,8 @@ class SyncWorker { this._logger.error(error, `SyncWorker: Error while syncing account`) this.closeConnection() - if (error.source.includes('socket') || error.source.includes('timeout')) { - // Continue to retry if it was a network error + // Continue to retry if it was a network error + if (error.source && (error.source.includes('socket') || error.source.includes('timeout'))) { return Promise.resolve() } From ac627f1580dd3e0849889d0993f8132902f05843 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 12 Jul 2016 01:33:56 -0700 Subject: [PATCH 237/800] Remove use of Rx.toPromise(), which wasn't behaving as expected --- packages/nylas-core/imap-box.js | 156 ++++++++++++ packages/nylas-core/imap-connection.js | 238 +++++------------- packages/nylas-core/imap-errors.js | 21 ++ packages/nylas-core/models/account/file.js | 9 +- .../imap/fetch-messages-in-folder.js | 16 +- 5 files changed, 254 insertions(+), 186 deletions(-) create mode 100644 packages/nylas-core/imap-box.js create mode 100644 packages/nylas-core/imap-errors.js diff --git a/packages/nylas-core/imap-box.js b/packages/nylas-core/imap-box.js new file mode 100644 index 000000000..ec9283749 --- /dev/null +++ b/packages/nylas-core/imap-box.js @@ -0,0 +1,156 @@ +const _ = require('underscore'); + +const { + IMAPConnectionNotReadyError, +} = require('./imap-errors'); + +class IMAPBox { + constructor(imapConn, box) { + this._conn = imapConn + this._box = box + + return new Proxy(this, { + get(target, name) { + const prop = Reflect.get(target, name) + if (!prop) { + return Reflect.get(target._box, name) + } + if (_.isFunction(prop) && target._conn._imap._box.name !== target._box.name) { + return () => Promise.reject( + new Error(`IMAPBox::${name} - Can't operate on a mailbox that is no longer open on the current IMAPConnection.`) + ) + } + return prop + }, + }) + } + + /** + * @param {array|string} range - can be a single message identifier, + * a message identifier range (e.g. '2504:2507' or '*' or '2504:*'), + * an array of message identifiers, or an array of message identifier ranges. + * @return {Observable} that will feed each message as it becomes ready + */ + fetchEach(range, options, forEachMessageCallback) { + if (!options) { + throw new Error("IMAPBox.fetch now requires an options object.") + } + if (range.length === 0) { + return Promise.resolve() + } + + return this._conn.createConnectionPromise((resolve, reject) => { + const f = this._conn._imap.fetch(range, options); + f.on('message', (imapMessage) => { + const parts = {}; + let headers = null; + let attributes = null; + imapMessage.on('attributes', (attrs) => { + attributes = attrs; + }); + imapMessage.on('body', (stream, info) => { + const chunks = []; + + stream.on('data', (chunk) => { + chunks.push(chunk); + }); + + stream.once('end', () => { + const full = Buffer.concat(chunks).toString('utf8'); + if (info.which === 'HEADER') { + headers = full; + } else { + parts[info.which] = full; + } + }); + }); + imapMessage.once('end', () => { + forEachMessageCallback({attributes, headers, parts}); + }); + }) + f.once('error', reject); + f.once('end', resolve); + }); + } + + /** + * @return {Promise} that resolves to requested message + */ + fetchMessage(uid) { + if (!uid) { + throw new Error("IMAPConnection.fetchMessage requires a message uid.") + } + return this.fetchEach([uid], { + bodies: ['HEADER', 'TEXT'], + }) + } + + fetchMessageStream(uid, options) { + if (!uid) { + throw new Error("IMAPConnection.fetchStream requires a message uid.") + } + if (!options) { + throw new Error("IMAPConnection.fetchStream requires an options object.") + } + return this._conn.createConnectionPromise((resolve, reject) => { + const f = this._conn._imap.fetch(uid, options); + f.on('message', (imapMessage) => { + imapMessage.on('body', (stream) => { + resolve(stream) + }) + }) + f.once('error', reject) + }) + } + + /** + * @param {array|string} range - can be a single message identifier, + * a message identifier range (e.g. '2504:2507' or '*' or '2504:*'), + * an array of message identifiers, or an array of message identifier ranges. + * @return {Promise} that resolves to a map of uid -> attributes for every + * message in the range + */ + fetchUIDAttributes(range) { + return this._conn.createConnectionPromise((resolve, reject) => { + const attributesByUID = {}; + const f = this._conn._imap.fetch(range, {}); + f.on('message', (msg) => { + msg.on('attributes', (attrs) => { + attributesByUID[attrs.uid] = attrs; + }) + }); + f.once('error', reject); + f.once('end', () => resolve(attributesByUID)); + }); + } + + addFlags(range, flags) { + if (!this._conn._imap) { + throw new IMAPConnectionNotReadyError(`IMAPBox::addFlags`) + } + return this._conn._imap.addFlagsAsync(range, flags) + } + + delFlags(range, flags) { + if (!this._conn._imap) { + throw new IMAPConnectionNotReadyError(`IMAPBox::delFlags`) + } + return this._conn._imap.delFlagsAsync(range, flags) + } + + moveFromBox(range, folderName) { + if (!this._conn._imap) { + throw new IMAPConnectionNotReadyError(`IMAPBox::moveFromBox`) + } + return this._conn._imap.moveAsync(range, folderName) + } + + closeBox({expunge = true} = {}) { + if (!this._conn._imap) { + throw new IMAPConnectionNotReadyError(`IMAPBox::closeBox`) + } + return this._conn._imap.closeBoxAsync(expunge) + } +} + +module.exports = IMAPBox; diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index bbbef526f..8c37acd28 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -1,163 +1,13 @@ -const Rx = require('rx') const Imap = require('imap'); const _ = require('underscore'); const xoauth2 = require('xoauth2'); const EventEmitter = require('events'); -class IMAPConnectionNotReadyError extends Error { - constructor(funcName) { - super(`${funcName} - You must call connect() first.`); - - // hack so that the error matches the ones used by node-imap - this.source = 'socket'; - } -} - -class IMAPBox { - - constructor(imapConn, box) { - this._imap = imapConn - this._box = box - return new Proxy(this, { - get(target, name) { - const prop = Reflect.get(target, name) - if (!prop) { - return Reflect.get(target._box, name) - } - if (_.isFunction(prop) && target._imap._box.name !== target._box.name) { - return () => Promise.reject( - new Error(`IMAPBox::${name} - Can't operate on a mailbox that is no longer open on the current IMAPConnection.`) - ) - } - return prop - }, - }) - } - - /** - * @param {array|string} range - can be a single message identifier, - * a message identifier range (e.g. '2504:2507' or '*' or '2504:*'), - * an array of message identifiers, or an array of message identifier ranges. - * @return {Observable} that will feed each message as it becomes ready - */ - fetch(range, options) { - if (!options) { - throw new Error("IMAPBox.fetch now requires an options object.") - } - if (range.length === 0) { - return Rx.Observable.empty() - } - return Rx.Observable.create((observer) => { - const f = this._imap.fetch(range, options); - f.on('message', (imapMessage) => { - const parts = {}; - let headers = null; - let attributes = null; - imapMessage.on('attributes', (attrs) => { - attributes = attrs; - }); - imapMessage.on('body', (stream, info) => { - const chunks = []; - - stream.on('data', (chunk) => { - chunks.push(chunk); - }); - - stream.once('end', () => { - const full = Buffer.concat(chunks).toString('utf8'); - if (info.which === 'HEADER') { - headers = full; - } else { - parts[info.which] = full; - } - }); - }); - imapMessage.once('end', () => { - observer.onNext({attributes, headers, parts}); - }); - }) - f.once('error', (error) => observer.onError(error)) - f.once('end', () => observer.onCompleted()) - }) - } - - fetchStream({uid, options}) { - if (!uid) { - throw new Error("IMAPConnection.fetchStream requires a message uid.") - } - if (!options) { - throw new Error("IMAPConnection.fetchStream requires an options object.") - } - return new Promise((resolve, reject) => { - const f = this._imap.fetch(uid, options); - f.on('message', (imapMessage) => { - imapMessage.on('body', (stream) => { - resolve(stream) - }) - }) - f.once('error', reject) - }) - } - - /** - * @return {Promise} that resolves to requested message - */ - fetchMessage(uid) { - return this.fetch([uid], { - bodies: ['HEADER', 'TEXT'], - }).toPromise() - } - - /** - * @param {array|string} range - can be a single message identifier, - * a message identifier range (e.g. '2504:2507' or '*' or '2504:*'), - * an array of message identifiers, or an array of message identifier ranges. - * @return {Promise} that resolves to a map of uid -> attributes for every - * message in the range - */ - fetchUIDAttributes(range) { - return new Promise((resolve, reject) => { - const attributesByUID = {}; - const f = this._imap.fetch(range, {}); - f.on('message', (msg) => { - msg.on('attributes', (attrs) => { - attributesByUID[attrs.uid] = attrs; - }) - }); - f.once('error', reject); - f.once('end', () => resolve(attributesByUID)); - }); - } - - addFlags(range, flags) { - if (!this._imap) { - throw new IMAPConnectionNotReadyError(`IMAPBox::addFlags`) - } - return this._imap.addFlagsAsync(range, flags) - } - - delFlags(range, flags) { - if (!this._imap) { - throw new IMAPConnectionNotReadyError(`IMAPBox::delFlags`) - } - return this._imap.delFlagsAsync(range, flags) - } - - moveFromBox(range, folderName) { - if (!this._imap) { - throw new IMAPConnectionNotReadyError(`IMAPBox::moveFromBox`) - } - return this._imap.moveAsync(range, folderName) - } - - closeBox({expunge = true} = {}) { - if (!this._imap) { - throw new IMAPConnectionNotReadyError(`IMAPBox::closeBox`) - } - return this._imap.closeBoxAsync(expunge) - } -} - +const IMAPBox = require('./imap-box'); +const { + IMAPConnectionNotReadyError, + IMAPConnectionEndedError, +} = require('./imap-errors'); const Capabilities = { Gmail: 'X-GM-EXT-1', @@ -195,8 +45,9 @@ class IMAPConnection extends EventEmitter { connect() { if (!this._connectPromise) { - this._connectPromise = this._resolveIMAPSettings() - .then((settings) => this._buildUnderlyingConnection(settings)) + this._connectPromise = this._resolveIMAPSettings().then((settings) => + this._buildUnderlyingConnection(settings) + ); } return this._connectPromise; } @@ -237,16 +88,6 @@ class IMAPConnection extends EventEmitter { return new Promise((resolve, reject) => { this._imap = Promise.promisifyAll(new Imap(settings)); - this._imap.once('end', () => { - this._logger.info('Underlying IMAP Connection ended'); - this._connectPromise = null; - this._imap = null; - }); - - this._imap.on('alert', (msg) => { - this._logger.info({imap_server_msg: msg}, `IMAP server message`) - }) - // Emitted when new mail arrives in the currently open mailbox. // Fix https://github.com/mscdex/node-imap/issues/445 let lastMailEventBox = null; @@ -263,8 +104,26 @@ class IMAPConnection extends EventEmitter { // Emitted when message metadata (e.g. flags) changes externally. this._imap.on('update', () => this.emit('update')) - this._imap.once('ready', () => resolve(this)); - this._imap.once('error', reject); + + this._imap.once('ready', () => { + resolve(this) + }); + + this._imap.once('error', (err) => { + this.end(); + reject(err); + }); + + this._imap.once('end', () => { + this._logger.info('Underlying IMAP Connection ended'); + this._connectPromise = null; + this._imap = null; + }); + + this._imap.on('alert', (msg) => { + this._logger.info({imap_server_msg: msg}, `IMAP server message`) + }); + this._imap.connect(); }); } @@ -293,7 +152,7 @@ class IMAPConnection extends EventEmitter { throw new IMAPConnectionNotReadyError(`IMAPConnection::openBox`) } return this._imap.openBoxAsync(folderName, readOnly).then((box) => - new IMAPBox(this._imap, box) + new IMAPBox(this, box) ) } @@ -337,6 +196,45 @@ class IMAPConnection extends EventEmitter { }); } + /* + Equivalent to new Promise, but allows you to easily create promises + which are also rejected when the IMAP connection is closed or ends. + This is important because node-imap sometimes just hangs the current + fetch / action forever after emitting an `end` event. + */ + createConnectionPromise(callback) { + if (!this._imap) { + throw new IMAPConnectionNotReadyError(`IMAPConnection::createConnectionPromise`) + } + + let onEnded = null; + let onErrored = null; + + return new Promise((resolve, reject) => { + let returned = false; + onEnded = () => { + returned = true; + reject(new IMAPConnectionEndedError()); + }; + onErrored = (error) => { + returned = true; + reject(error || new Error("Unspecified IMAP error.")); + }; + + this._imap.once('error', onEnded); + this._imap.once('end', onErrored); + + const cresolve = (...args) => (!returned ? resolve(...args) : null) + const creject = (...args) => (!returned ? reject(...args) : null) + return callback(cresolve, creject) + }).finally(() => { + if (this._imap) { + this._imap.removeListener('error', onEnded); + this._imap.removeListener('end', onErrored); + } + }); + } + processNextOperation() { if (this._currentOperation) { return; diff --git a/packages/nylas-core/imap-errors.js b/packages/nylas-core/imap-errors.js new file mode 100644 index 000000000..3161deff5 --- /dev/null +++ b/packages/nylas-core/imap-errors.js @@ -0,0 +1,21 @@ + +// "Source" is a hack so that the error matches the ones used by node-imap + +class IMAPConnectionNotReadyError extends Error { + constructor(funcName) { + super(`${funcName} - You must call connect() first.`); + this.source = 'socket'; + } +} + +class IMAPConnectionEndedError extends Error { + constructor(msg = "The IMAP Connection was ended.") { + super(msg); + this.source = 'socket'; + } +} + +module.exports = { + IMAPConnectionNotReadyError, + IMAPConnectionEndedError, +}; diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js index 655e8b13d..ba52ed4d2 100644 --- a/packages/nylas-core/models/account/file.js +++ b/packages/nylas-core/models/account/file.js @@ -24,12 +24,9 @@ module.exports = (sequelize, Sequelize) => { .then(({message, connection}) => { return message.getFolder() .then((folder) => connection.openBox(folder.name)) - .then((imapBox) => imapBox.fetchStream({ - uid: message.folderImapUID, - options: { - bodies: [this.partId], - struct: true, - }, + .then((imapBox) => imapBox.fetchMessageStream(message.folderImapUID, { + bodies: [this.partId], + struct: true, })) .then((stream) => { if (stream) { diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js index 038dde66d..9ce3fe108 100644 --- a/packages/nylas-sync/imap/fetch-messages-in-folder.js +++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js @@ -166,8 +166,7 @@ class FetchMessagesInFolder { _fetchMessagesAndQueueForProcessing(range) { const uidsByPart = {}; - const $structs = this._box.fetch(range, {struct: true}) - $structs.subscribe(({attributes}) => { + return this._box.fetchEach(range, {struct: true}, ({attributes}) => { const desiredParts = this._getDesiredMIMEParts(attributes.struct); if (desiredParts.length === 0) { return; @@ -175,13 +174,13 @@ class FetchMessagesInFolder { const key = JSON.stringify(desiredParts); uidsByPart[key] = uidsByPart[key] || []; uidsByPart[key].push(attributes.uid); - }); - - return $structs.toPromise().then(() => { + }) + .then(() => { return Promise.each(Object.keys(uidsByPart), (key) => { const uids = uidsByPart[key]; const desiredParts = JSON.parse(key); const bodies = ['HEADER'].concat(desiredParts.map(p => p.id)); + this._logger.info({ key, num_messages: uids.length, @@ -190,17 +189,14 @@ class FetchMessagesInFolder { // note: the order of UIDs in the array doesn't matter, Gmail always // returns them in ascending (oldest => newest) order. - const $body = this._box.fetch(uids, {bodies, struct: true}) - $body.subscribe((msg) => { + return this._box.fetchEach(uids, {bodies, struct: true}, (msg) => { msg.body = {}; for (const {id, mimetype} of desiredParts) { msg.body[mimetype] = msg.parts[id]; } this._processMessage(msg); }); - - return $body.toPromise(); - }) + }); }); } From c4736cb4d18061a38df8553c2dd6acf0434b3001 Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 12 Jul 2016 01:34:13 -0700 Subject: [PATCH 238/800] Copy test_accounts.py into curl format for easy testing --- test_accounts.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test_accounts.txt diff --git a/test_accounts.txt b/test_accounts.txt new file mode 100644 index 000000000..0049f9ad6 --- /dev/null +++ b/test_accounts.txt @@ -0,0 +1,16 @@ +Yahoo: +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdiddle1861@yahoo.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdiddle1861@yahoo.com",imap_host":"imap.mail.yahoo.com","imap_port":993,"smtp_host":"smtp.mail.yahoo.com","smtp_port":0,"smtp_username":"benbitdiddle1861@yahoo.com", "smtp_password":"EverybodyLovesIMAPv4","imap_password":"EverybodyLovesIMAPv4","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"cypresstest@yahoo.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"cypresstest@yahoo.com","imap_host":"imap.mail.yahoo.com","imap_port":993,"smtp_host":"smtp.mail.yahoo.com","smtp_port":0,"smtp_username":"cypresstest@yahoo.com", "smtp_password":"IHate2Gmail","imap_password":"IHate2Gmail","ssl_required":true}}' "http://localhost:5100.com/auth?client_id=123" + +Aol: +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdit@aol.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdit@aol.com","imap_host":"imap.aol.com","imap_port":993,"smtp_host":"smtp.aol.com","smtp_port":0,"smtp_username":"benbitdit@aol.com", "smtp_password":"IHate2Gmail","imap_password":"IHate2Gmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" + +iCloud: +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inbox.systemtest@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inbox.systemtest@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"inbox.systemtest@icloud.com", "smtp_password":"iHate2Gmai","imap_password":"iHate2Gmai","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inbox.watchdog@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inbox.watchdog@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"inbox.watchdog@icloud.com", "smtp_password":"iHate2Gmai","imap_password":"iHate2Gmai","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"benbitdiddle@icloud.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"benbitdiddle@icloud.com","imap_host":"imap.mail.me.com","imap_port":993,"smtp_host":"smtp.mail.me.com","smtp_port":0,"smtp_username":"benbitdiddle@icloud.com", "smtp_password":"ih4teIMAP","imap_password":"ih4teIMAP","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" + +Others: +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"nylastest@runbox.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"nylastest","imap_host":"mail.runbox.com","imap_port":993,"smtp_host":"mail.runbox.com","smtp_port":0,"smtp_username":"nylastest", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"securemail@defendcyber.space", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"securemail@defendcyber.space","imap_host":"imap.secureserver.net","imap_port":143,"smtp_host":"smtpout.secureserver.net","smtp_port":25,"smtp_username":"securemail@defendcyber.space", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":false}}' "http://localhost:5100/auth?client_id=123" +curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest4@gmail.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest4@gmail.com","imap_host":"imap.gmail.com","imap_port":993,"smtp_host":"smtp.gmail.com","smtp_port":465,"smtp_username":"inboxapptest4@gmail.com", "smtp_password":"ihategmail","imap_password":"ihategmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123" From b0a6ec066bb1ecd63094a78e54b4742a1a544ebf Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 12 Jul 2016 09:53:04 -0700 Subject: [PATCH 239/800] Sync non-active accounts more infrequently --- packages/nylas-core/sync-policy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nylas-core/sync-policy.js b/packages/nylas-core/sync-policy.js index f3cf89413..bc6d330a8 100644 --- a/packages/nylas-core/sync-policy.js +++ b/packages/nylas-core/sync-policy.js @@ -4,7 +4,7 @@ class SyncPolicy { afterSync: 'idle', intervals: { active: 30 * 1000, - inactive: 120 * 1000, + inactive: 5 * 60 * 1000, }, folderSyncOptions: { deepFolderScan: 10 * 60 * 1000, From 674da272963644254873a9301166a130d4c41d5d Mon Sep 17 00:00:00 2001 From: Ben Gotow Date: Tue, 12 Jul 2016 12:15:46 -0700 Subject: [PATCH 240/800] Fix flipped errored / ended handlers --- packages/nylas-core/imap-connection.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js index 8c37acd28..0d23f068f 100644 --- a/packages/nylas-core/imap-connection.js +++ b/packages/nylas-core/imap-connection.js @@ -218,19 +218,19 @@ class IMAPConnection extends EventEmitter { }; onErrored = (error) => { returned = true; - reject(error || new Error("Unspecified IMAP error.")); + reject(error); }; - this._imap.once('error', onEnded); - this._imap.once('end', onErrored); + this._imap.once('error', onErrored); + this._imap.once('end', onEnded); const cresolve = (...args) => (!returned ? resolve(...args) : null) const creject = (...args) => (!returned ? reject(...args) : null) return callback(cresolve, creject) }).finally(() => { if (this._imap) { - this._imap.removeListener('error', onEnded); - this._imap.removeListener('end', onErrored); + this._imap.removeListener('error', onErrored); + this._imap.removeListener('end', onEnded); } }); } From 3e6e5e95b95c89173d53f29fbb05592e5df5cd05 Mon Sep 17 00:00:00 2001 From: Halla Moore Date: Mon, 11 Jul 2016 16:16:33 -0700 Subject: [PATCH 241/800] Add dashboard functionality to clear sync errors --- packages/nylas-dashboard/public/js/app.jsx | 22 ++++++++++++++++ packages/nylas-dashboard/routes/account.js | 30 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 packages/nylas-dashboard/routes/account.js diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx index efa32b442..0a4acad2b 100644 --- a/packages/nylas-dashboard/public/js/app.jsx +++ b/packages/nylas-dashboard/public/js/app.jsx @@ -12,6 +12,27 @@ const { } = window; class Account extends React.Component { + constructor(props) { + super(props); + this.state = { + accountId: props.account.id, + } + } + clearError() { + const req = new XMLHttpRequest(); + const url = `${window.location.protocol}/accounts/${this.state.accountId}/clear-sync-error`; + req.open("PUT", url, true); + req.onreadystatechange = () => { + if (req.readyState === XMLHttpRequest.DONE) { + if (req.status === 200) { + // Would setState here, but external updates currently refresh the account + } else { + console.error(req.responseText); + } + } + } + req.send(); + } renderError() { const {account} = this.props; @@ -24,6 +45,7 @@ class Account extends React.Component { return (
Error
+ this.clearError()}>Clear Error 
               {JSON.stringify(error, null, 2)}
diff --git a/packages/nylas-dashboard/routes/account.js b/packages/nylas-dashboard/routes/account.js
new file mode 100644
index 000000000..a7c8c4fe5
--- /dev/null
+++ b/packages/nylas-dashboard/routes/account.js
@@ -0,0 +1,30 @@
+const Joi = require('joi');
+const {DatabaseConnector} = require(`nylas-core`);
+
+module.exports = (server) => {
+  server.route({
+    method: 'PUT',
+    path: '/accounts/{accountId}/clear-sync-error',
+    config: {
+      description: 'Clears the sync error for the given account',
+      notes: 'Notes go here',
+      tags: ['accounts', 'sync-error'],
+      validate: {
+        params: {
+          accountId: Joi.number().integer(),
+        },
+      },
+      response: {
+        schema: Joi.string(),
+      },
+    },
+    handler: (request, reply) => {
+      DatabaseConnector.forShared().then(({Account}) => {
+        Account.find({where: {id: request.params.accountId}}).then((account) => {
+          account.syncError = null;
+          account.save().then(() => reply("Success"));
+        })
+      })
+    },
+  });
+};

From 84d89792656bdb3cf8f897dd7e6ac896e05408dd Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 13:24:48 -0700
Subject: [PATCH 242/800] Add ability to auth gmail account via `/auth`

---
 packages/nylas-api/routes/auth.js | 65 +++++++++++++++++++++----------
 1 file changed, 45 insertions(+), 20 deletions(-)

diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js
index e33f29a24..a1734f8f0 100644
--- a/packages/nylas-api/routes/auth.js
+++ b/packages/nylas-api/routes/auth.js
@@ -33,6 +33,12 @@ const imapSmtpSettings = Joi.object().keys({
   ssl_required: Joi.boolean().required(),
 }).required();
 
+const gmailSettings = Joi.object().keys({
+  google_client_id: Joi.string().required(),
+  google_client_secret: Joi.string().required(),
+  google_refresh_token: Joi.string().required(),
+});
+
 const exchangeSettings = Joi.object().keys({
   username: Joi.string().required(),
   password: Joi.string().required(),
@@ -90,8 +96,8 @@ module.exports = (server) => {
         payload: {
           email: Joi.string().email().required(),
           name: Joi.string().required(),
-          provider: Joi.string().required(),
-          settings: Joi.alternatives().try(imapSmtpSettings, exchangeSettings),
+          provider: Joi.string().valid('imap', 'gmail').required(),
+          settings: Joi.alternatives().try(imapSmtpSettings, exchangeSettings, gmailSettings),
         },
       },
       response: {
@@ -106,28 +112,48 @@ module.exports = (server) => {
       const connectionChecks = [];
       const {settings, email, provider, name} = request.payload;
 
+      let connectionSettings = null;
+      let connectionCredentials = null;
+
       if (provider === 'imap') {
-        connectionChecks.push(IMAPConnection.connect({
-          logger: request.logger,
-          settings: settings,
-          db: dbStub,
-        }));
+        connectionSettings = _.pick(settings, [
+          'imap_host', 'imap_port',
+          'smtp_host', 'smtp_port',
+          'ssl_required',
+        ]);
+        connectionCredentials = _.pick(settings, [
+          'imap_username', 'imap_password',
+          'smtp_username', 'smtp_password',
+        ]);
       }
 
+      if (provider === 'gmail') {
+        connectionSettings = {
+          imap_username: email,
+          imap_host: 'imap.gmail.com',
+          imap_port: 993,
+          ssl_required: true,
+        }
+        connectionCredentials = {
+          client_id: settings.google_client_id,
+          client_secret: settings.google_client_secret,
+          refresh_token: settings.google_refresh_token,
+        }
+      }
+
+      connectionChecks.push(IMAPConnection.connect({
+        settings: Object.assign({}, connectionSettings, connectionCredentials),
+        logger: request.logger,
+        db: dbStub,
+      }));
+
       Promise.all(connectionChecks).then(() => {
         return buildAccountWith({
-          name,
-          email,
-          provider: Provider.IMAP,
-          settings: _.pick(settings, [
-            'imap_host', 'imap_port',
-            'smtp_host', 'smtp_port',
-            'ssl_required',
-          ]),
-          credentials: _.pick(settings, [
-            'imap_username', 'imap_password',
-            'smtp_username', 'smtp_password',
-          ]),
+          name: name,
+          email: email,
+          provider: provider,
+          settings: connectionSettings,
+          credentials: connectionCredentials,
         })
       })
       .then(({account, token}) => {
@@ -195,7 +221,6 @@ module.exports = (server) => {
             ssl_required: true,
           }
           const credentials = {
-            access_token: tokens.access_token,
             refresh_token: tokens.refresh_token,
             client_id: GMAIL_CLIENT_ID,
             client_secret: GMAIL_CLIENT_SECRET,

From dfa16fb5fc2f503570658c498517178bb9c54b8b Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 15:04:12 -0700
Subject: [PATCH 243/800] Allow firstSyncCompletion to be null

---
 packages/nylas-core/models/shared/account.js | 6 +++++-
 packages/nylas-sync/sync-worker.js           | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js
index 9a61101a9..da16219d3 100644
--- a/packages/nylas-core/models/shared/account.js
+++ b/packages/nylas-core/models/shared/account.js
@@ -12,7 +12,11 @@ module.exports = (sequelize, Sequelize) => {
     connectionCredentials: Sequelize.TEXT,
     syncPolicy: JSONType('syncPolicy'),
     syncError: JSONType('syncError', {defaultValue: null}),
-    firstSyncCompletion: Sequelize.INTEGER(14),
+    firstSyncCompletion: {
+      type: Sequelize.INTEGER(14),
+      allowNull: true,
+      defaultValue: null,
+    },
     lastSyncCompletions: JSONARRAYType('lastSyncCompletions'),
   }, {
     classMethods: {
diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 8a7a9fd0f..51a8e9544 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -208,12 +208,12 @@ class SyncWorker {
 
   onSyncDidComplete() {
     const {afterSync} = this._account.syncPolicy;
+    const now = Date.now();
 
     if (!this._account.firstSyncCompletion) {
-      this._account.firstSyncCompletion = Date.now()
+      this._account.firstSyncCompletion = now;
     }
 
-    const now = Date.now();
     const syncGraphTimeLength = 60 * 30; // 30 minutes, should be the same as SyncGraph.config.timeLength
     let lastSyncCompletions = [].concat(this._account.lastSyncCompletions)
     lastSyncCompletions = [now, ...lastSyncCompletions]

From 7d92438d57704a22f7f86808a321499cfdfeec43 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 15:23:55 -0700
Subject: [PATCH 244/800] Change firstSyncCompletion to string, cast to int for
 JSON

---
 packages/nylas-core/models/shared/account.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js
index da16219d3..df5d537d2 100644
--- a/packages/nylas-core/models/shared/account.js
+++ b/packages/nylas-core/models/shared/account.js
@@ -13,7 +13,7 @@ module.exports = (sequelize, Sequelize) => {
     syncPolicy: JSONType('syncPolicy'),
     syncError: JSONType('syncError', {defaultValue: null}),
     firstSyncCompletion: {
-      type: Sequelize.INTEGER(14),
+      type: Sequelize.STRING(14),
       allowNull: true,
       defaultValue: null,
     },
@@ -35,7 +35,7 @@ module.exports = (sequelize, Sequelize) => {
           connection_settings: this.connectionSettings,
           sync_policy: this.syncPolicy,
           sync_error: this.syncError,
-          first_sync_completion: this.firstSyncCompletion,
+          first_sync_completion: this.firstSyncCompletion / 1,
           last_sync_completions: this.lastSyncCompletions,
           created_at: this.createdAt,
         }

From 8b1e9564cf997c992c784dbe5c6bdda1518a8945 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 16:11:29 -0700
Subject: [PATCH 245/800] =?UTF-8?q?Fix=20UTF7=20support=20=E2=80=94=20modu?=
 =?UTF-8?q?le=20didn=E2=80=99t=20support=20Node6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 package.json                     | 4 +++-
 packages/nylas-core/package.json | 1 -
 packages/nylas-sync/package.json | 1 -
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index 4073a8ef4..9e9343ea6 100644
--- a/package.json
+++ b/package.json
@@ -15,7 +15,9 @@
     "redis": "2.x.x",
     "rx": "4.x.x",
     "sequelize": "3.x.x",
-    "underscore": "1.x.x"
+    "underscore": "1.x.x",
+    "utf7": "git@github.com:truebit/utf7.git#1f753bac59b99d93b17a5ef11681e232465e2558",
+    "imap": "0.8.x"
   },
   "devDependencies": {
     "babel-eslint": "6.x",
diff --git a/packages/nylas-core/package.json b/packages/nylas-core/package.json
index 3c151ff87..ac3938cb1 100644
--- a/packages/nylas-core/package.json
+++ b/packages/nylas-core/package.json
@@ -5,7 +5,6 @@
   "main": "index.js",
   "dependencies": {
     "bunyan": "^1.8.1",
-    "imap": "0.8.x",
     "xoauth2": "1.x.x"
   },
   "author": "Nylas",
diff --git a/packages/nylas-sync/package.json b/packages/nylas-sync/package.json
index aaf58b528..17f307081 100644
--- a/packages/nylas-sync/package.json
+++ b/packages/nylas-sync/package.json
@@ -3,7 +3,6 @@
   "version": "0.0.1",
   "description": "Nylas Sync Engine",
   "dependencies": {
-    "imap": "0.8.17",
     "nylas-core": "0.x.x",
     "nylas-message-processor": "0.x.x",
     "xoauth2": "1.1.0"

From ae082c5b53fa0a5b9416ab4fe3475586e59d66db Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 16:12:04 -0700
Subject: [PATCH 246/800] Fix support for Gmail accts with non-english folders

---
 packages/nylas-sync/imap/fetch-folder-list.js | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/packages/nylas-sync/imap/fetch-folder-list.js b/packages/nylas-sync/imap/fetch-folder-list.js
index d5c5a0000..d59d5794e 100644
--- a/packages/nylas-sync/imap/fetch-folder-list.js
+++ b/packages/nylas-sync/imap/fetch-folder-list.js
@@ -1,6 +1,6 @@
 const {Provider} = require('nylas-core');
 
-const GMAIL_FOLDERS = ['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam'];
+const GMAIL_ROLES_WITH_FOLDERS = ['all', 'trash', 'junk'];
 
 class FetchFolderList {
   constructor(provider, logger = console) {
@@ -12,9 +12,9 @@ class FetchFolderList {
     return `FetchFolderList`;
   }
 
-  _classForMailbox(boxName, box, {Folder, Label}) {
+  _classForMailboxWithRole(role, {Folder, Label}) {
     if (this._provider === Provider.Gmail) {
-      return GMAIL_FOLDERS.includes(boxName) ? Folder : Label;
+      return GMAIL_ROLES_WITH_FOLDERS.includes(role) ? Folder : Label;
     }
     return Folder;
   }
@@ -65,11 +65,12 @@ class FetchFolderList {
 
       let category = categories.find((cat) => cat.name === boxName);
       if (!category) {
-        const Klass = this._classForMailbox(boxName, box, this._db);
+        const role = this._roleForMailbox(boxName, box);
+        const Klass = this._classForMailboxWithRole(role, this._db);
         category = Klass.build({
           name: boxName,
           accountId: this._db.accountId,
-          role: this._roleForMailbox(boxName, box),
+          role: role,
         });
         created.push(category);
       }

From 9d7343eb918f879f6bf86d6c8c40c0b1260943bc Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 16:24:03 -0700
Subject: [PATCH 247/800] Fix UTF7 reference - prod will not use ssh

---
 package.json                                 | 2 +-
 packages/nylas-core/models/shared/account.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 9e9343ea6..a6cdce36b 100644
--- a/package.json
+++ b/package.json
@@ -16,7 +16,7 @@
     "rx": "4.x.x",
     "sequelize": "3.x.x",
     "underscore": "1.x.x",
-    "utf7": "git@github.com:truebit/utf7.git#1f753bac59b99d93b17a5ef11681e232465e2558",
+    "utf7": "https://github.com/truebit/utf7/archive/1f753bac59b99d93b17a5ef11681e232465e2558.tar.gz",
     "imap": "0.8.x"
   },
   "devDependencies": {
diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js
index df5d537d2..4e1297025 100644
--- a/packages/nylas-core/models/shared/account.js
+++ b/packages/nylas-core/models/shared/account.js
@@ -42,7 +42,7 @@ module.exports = (sequelize, Sequelize) => {
       },
 
       errored: function errored() {
-        return this.syncError != null
+        return this.syncError != null;
       },
 
       setCredentials: function setCredentials(json) {

From 83a6a8008b5533e4fe6be427c406adbaffacc1e0 Mon Sep 17 00:00:00 2001
From: Annie 
Date: Tue, 12 Jul 2016 16:26:57 -0700
Subject: [PATCH 248/800] Add thread id to message model

---
 packages/nylas-core/models/account/message.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js
index e5fffdacc..4db6f4d24 100644
--- a/packages/nylas-core/models/account/message.js
+++ b/packages/nylas-core/models/account/message.js
@@ -109,6 +109,7 @@ module.exports = (sequelize, Sequelize) => {
           unread: this.unread,
           starred: this.starred,
           folder: this.folder,
+          thread_id: this.threadId,
         };
       },
     },

From 562cedf971707913036bc70c302bb3d0c0f998a5 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 16:46:14 -0700
Subject: [PATCH 249/800] Fix sync worker stopping forever on errors

---
 packages/nylas-sync/sync-worker.js | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 51a8e9544..992ef3a52 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -174,19 +174,21 @@ class SyncWorker {
     clearTimeout(this._syncTimer);
     this._syncTimer = null;
 
-    if (!process.env.SYNC_AFTER_ERRORS && this._account.errored()) {
-      this._logger.info(`SyncWorker: Account is in error state - Skipping sync`)
-      return
-    }
-    this._logger.info({reason}, `SyncWorker: Account sync started`)
+    this._account.reload().then(() => {
+      if (!process.env.SYNC_AFTER_ERRORS && this._account.errored()) {
+        this._logger.info(`SyncWorker: Account is in error state - Skipping sync`)
+        return Promise.resolve();
+      }
+      this._logger.info({reason}, `SyncWorker: Account sync started`)
 
-    this.ensureConnection()
-    .then(() => this._account.update({syncError: null}))
-    .then(() => this.syncbackMessageActions())
-    .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider)))
-    .then(() => this.syncAllCategories())
-    .then(() => this.onSyncDidComplete())
-    .catch((error) => this.onSyncError(error))
+      return this._account.update({syncError: null})
+      .then(() => this.ensureConnection())
+      .then(() => this.syncbackMessageActions())
+      .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider)))
+      .then(() => this.syncAllCategories())
+      .then(() => this.onSyncDidComplete())
+      .catch((error) => this.onSyncError(error))
+    })
     .finally(() => {
       this._lastSyncTime = Date.now()
       this.scheduleNextSync()

From a62e858389ec8e1fd9334b7ca934caadd1e53dd7 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 17:41:16 -0700
Subject: [PATCH 250/800] Make charset explicit on connection and models

---
 packages/nylas-core/database-connector.js     | 1 +
 packages/nylas-core/models/account/contact.js | 1 +
 packages/nylas-core/models/account/file.js    | 1 +
 packages/nylas-core/models/account/folder.js  | 1 +
 packages/nylas-core/models/account/label.js   | 1 +
 packages/nylas-core/models/account/message.js | 1 +
 packages/nylas-core/models/account/thread.js  | 1 +
 packages/nylas-core/models/shared/account.js  | 1 +
 8 files changed, 8 insertions(+)

diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js
index 0a025a298..6b84a1c47 100644
--- a/packages/nylas-core/database-connector.js
+++ b/packages/nylas-core/database-connector.js
@@ -39,6 +39,7 @@ class DatabaseConnector {
       return new Sequelize(dbname, process.env.DB_USERNAME, process.env.DB_PASSWORD, {
         host: process.env.DB_HOSTNAME,
         dialect: "mysql",
+        charset: 'utf8',
         logging: false,
       });
     }
diff --git a/packages/nylas-core/models/account/contact.js b/packages/nylas-core/models/account/contact.js
index 243787a64..5582e7a8b 100644
--- a/packages/nylas-core/models/account/contact.js
+++ b/packages/nylas-core/models/account/contact.js
@@ -5,6 +5,7 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     email: Sequelize.STRING,
   }, {
+    charset: 'utf8',
     instanceMethods: {
       toJSON: function toJSON() {
         return {
diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js
index ba52ed4d2..96001628f 100644
--- a/packages/nylas-core/models/account/file.js
+++ b/packages/nylas-core/models/account/file.js
@@ -9,6 +9,7 @@ module.exports = (sequelize, Sequelize) => {
     contentType: Sequelize.STRING(500),
     size: Sequelize.INTEGER,
   }, {
+    charset: 'utf8',
     classMethods: {
       associate: ({Message}) => {
         File.belongsTo(Message)
diff --git a/packages/nylas-core/models/account/folder.js b/packages/nylas-core/models/account/folder.js
index 5cec31d41..fa6af016b 100644
--- a/packages/nylas-core/models/account/folder.js
+++ b/packages/nylas-core/models/account/folder.js
@@ -8,6 +8,7 @@ module.exports = (sequelize, Sequelize) => {
     role: Sequelize.STRING,
     syncState: JSONType('syncState'),
   }, {
+    charset: 'utf8',
     classMethods: {
       associate: ({Message, Thread}) => {
         Folder.hasMany(Message)
diff --git a/packages/nylas-core/models/account/label.js b/packages/nylas-core/models/account/label.js
index 647955210..11263b95f 100644
--- a/packages/nylas-core/models/account/label.js
+++ b/packages/nylas-core/models/account/label.js
@@ -5,6 +5,7 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     role: Sequelize.STRING,
   }, {
+    charset: 'utf8',
     classMethods: {
       associate: ({Message, Thread}) => {
         Label.belongsToMany(Message, {through: 'message_labels'})
diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js
index 4db6f4d24..13ff33d24 100644
--- a/packages/nylas-core/models/account/message.js
+++ b/packages/nylas-core/models/account/message.js
@@ -25,6 +25,7 @@ module.exports = (sequelize, Sequelize) => {
     folderImapUID: { type: Sequelize.STRING, allowNull: true},
     folderImapXGMLabels: { type: Sequelize.TEXT, allowNull: true},
   }, {
+    charset: 'utf8',
     indexes: [
       {
         unique: true,
diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js
index 0b77cbd1f..99419696c 100644
--- a/packages/nylas-core/models/account/thread.js
+++ b/packages/nylas-core/models/account/thread.js
@@ -15,6 +15,7 @@ module.exports = (sequelize, Sequelize) => {
     lastMessageSentDate: Sequelize.DATE,
     participants: JSONARRAYType('participants'),
   }, {
+    charset: 'utf8',
     indexes: [
       { fields: ['subject'] },
       { fields: ['threadId'] },
diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js
index 4e1297025..7db4b4af0 100644
--- a/packages/nylas-core/models/shared/account.js
+++ b/packages/nylas-core/models/shared/account.js
@@ -19,6 +19,7 @@ module.exports = (sequelize, Sequelize) => {
     },
     lastSyncCompletions: JSONARRAYType('lastSyncCompletions'),
   }, {
+    charset: 'utf8',
     classMethods: {
       associate: ({AccountToken}) => {
         Account.hasMany(AccountToken, {as: 'tokens'})

From 3e8623d38332c99aa0a74d6eb2301db9f55e6b90 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 18:03:08 -0700
Subject: [PATCH 251/800] Move charset definition to single location

---
 packages/nylas-core/database-connector.js     | 4 ++++
 packages/nylas-core/models/account/contact.js | 1 -
 packages/nylas-core/models/account/file.js    | 1 -
 packages/nylas-core/models/account/folder.js  | 1 -
 packages/nylas-core/models/account/label.js   | 1 -
 packages/nylas-core/models/account/message.js | 1 -
 packages/nylas-core/models/account/thread.js  | 1 -
 packages/nylas-core/models/shared/account.js  | 1 -
 8 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/packages/nylas-core/database-connector.js b/packages/nylas-core/database-connector.js
index 6b84a1c47..36301655b 100644
--- a/packages/nylas-core/database-connector.js
+++ b/packages/nylas-core/database-connector.js
@@ -41,6 +41,10 @@ class DatabaseConnector {
         dialect: "mysql",
         charset: 'utf8',
         logging: false,
+        define: {
+          charset: 'utf8',
+          collate: 'utf8_general_ci',
+        },
       });
     }
 
diff --git a/packages/nylas-core/models/account/contact.js b/packages/nylas-core/models/account/contact.js
index 5582e7a8b..243787a64 100644
--- a/packages/nylas-core/models/account/contact.js
+++ b/packages/nylas-core/models/account/contact.js
@@ -5,7 +5,6 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     email: Sequelize.STRING,
   }, {
-    charset: 'utf8',
     instanceMethods: {
       toJSON: function toJSON() {
         return {
diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js
index 96001628f..ba52ed4d2 100644
--- a/packages/nylas-core/models/account/file.js
+++ b/packages/nylas-core/models/account/file.js
@@ -9,7 +9,6 @@ module.exports = (sequelize, Sequelize) => {
     contentType: Sequelize.STRING(500),
     size: Sequelize.INTEGER,
   }, {
-    charset: 'utf8',
     classMethods: {
       associate: ({Message}) => {
         File.belongsTo(Message)
diff --git a/packages/nylas-core/models/account/folder.js b/packages/nylas-core/models/account/folder.js
index fa6af016b..5cec31d41 100644
--- a/packages/nylas-core/models/account/folder.js
+++ b/packages/nylas-core/models/account/folder.js
@@ -8,7 +8,6 @@ module.exports = (sequelize, Sequelize) => {
     role: Sequelize.STRING,
     syncState: JSONType('syncState'),
   }, {
-    charset: 'utf8',
     classMethods: {
       associate: ({Message, Thread}) => {
         Folder.hasMany(Message)
diff --git a/packages/nylas-core/models/account/label.js b/packages/nylas-core/models/account/label.js
index 11263b95f..647955210 100644
--- a/packages/nylas-core/models/account/label.js
+++ b/packages/nylas-core/models/account/label.js
@@ -5,7 +5,6 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     role: Sequelize.STRING,
   }, {
-    charset: 'utf8',
     classMethods: {
       associate: ({Message, Thread}) => {
         Label.belongsToMany(Message, {through: 'message_labels'})
diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js
index 13ff33d24..4db6f4d24 100644
--- a/packages/nylas-core/models/account/message.js
+++ b/packages/nylas-core/models/account/message.js
@@ -25,7 +25,6 @@ module.exports = (sequelize, Sequelize) => {
     folderImapUID: { type: Sequelize.STRING, allowNull: true},
     folderImapXGMLabels: { type: Sequelize.TEXT, allowNull: true},
   }, {
-    charset: 'utf8',
     indexes: [
       {
         unique: true,
diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js
index 99419696c..0b77cbd1f 100644
--- a/packages/nylas-core/models/account/thread.js
+++ b/packages/nylas-core/models/account/thread.js
@@ -15,7 +15,6 @@ module.exports = (sequelize, Sequelize) => {
     lastMessageSentDate: Sequelize.DATE,
     participants: JSONARRAYType('participants'),
   }, {
-    charset: 'utf8',
     indexes: [
       { fields: ['subject'] },
       { fields: ['threadId'] },
diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js
index 7db4b4af0..4e1297025 100644
--- a/packages/nylas-core/models/shared/account.js
+++ b/packages/nylas-core/models/shared/account.js
@@ -19,7 +19,6 @@ module.exports = (sequelize, Sequelize) => {
     },
     lastSyncCompletions: JSONARRAYType('lastSyncCompletions'),
   }, {
-    charset: 'utf8',
     classMethods: {
       associate: ({AccountToken}) => {
         Account.hasMany(AccountToken, {as: 'tokens'})

From 8e790a0e155507cea1009a210352882e7394c83d Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Tue, 12 Jul 2016 17:27:41 -0700
Subject: [PATCH 252/800] Spruce up the dashboard

Consloidate modal functionality into its own component and make
general appearance fixes, especially with the Syncback Request
Details.
---
 packages/nylas-dashboard/public/css/app.css   |  42 +++-
 .../nylas-dashboard/public/images/close.png   | Bin 0 -> 15187 bytes
 packages/nylas-dashboard/public/index.html    |   1 +
 packages/nylas-dashboard/public/js/app.jsx    |   2 +-
 .../nylas-dashboard/public/js/dropdown.jsx    |   4 +-
 packages/nylas-dashboard/public/js/modal.jsx  | 102 +++++++++
 .../public/js/set-all-sync-policies.jsx       |  59 +++---
 .../nylas-dashboard/public/js/sync-policy.jsx |   4 +-
 .../public/js/syncback-request-details.jsx    | 196 ++++++++----------
 9 files changed, 251 insertions(+), 159 deletions(-)
 create mode 100644 packages/nylas-dashboard/public/images/close.png
 create mode 100644 packages/nylas-dashboard/public/js/modal.jsx

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index 436426196..754519847 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -51,14 +51,16 @@ pre {
   overflow: hidden;
 }
 
-#set-all-sync {
-  display: block;
-  margin-bottom: 10px;
+#open-all-sync {
   color: #ffffff;
 }
 
+.right-action {
+  float: right;
+  margin-top: 10px;
+}
+
 .action-link {
-  display: inline-block;
   color: rgba(16, 83, 161, 0.88);
   text-decoration: underline;
   cursor: pointer;
@@ -66,7 +68,7 @@ pre {
 }
 
 .action-link.cancel {
-  margin-left: 5px;
+  margin-top: 10px;
 }
 
 .sync-policy textarea {
@@ -78,8 +80,9 @@ pre {
 .modal {
   background-color: white;
   width: 50%;
-  margin: auto;
+  margin: 10vh auto;
   padding: 20px;
+  max-height: calc(80vh - 40px); /* minus padding */
   overflow: auto;
 }
 
@@ -90,16 +93,27 @@ pre {
   left: 0;
   top: 0;
   background-color: rgba(0, 0, 0, 0.3);
-  padding-top: 10%;
+}
+
+.modal-close-wrapper {
+  position: relative;
+  height: 0;
+  width: 0;
+  float: right;
+  top: -10px;
 }
 
 .modal-close {
-  position: relative;
-  float: right;
-  top: -10px;
+  position: absolute;
   cursor: pointer;
   font-size: 14px;
   font-weight: bold;
+  background: url('../images/close.png') center center no-repeat;
+  background-size: 12px auto;
+  height: 12px;
+  width: 12px;
+  top: 12px;
+  right: 12px;
 }
 
 .sync-graph {
@@ -127,6 +141,10 @@ pre {
 
 #syncback-request-details table {
   width: 100%;
+  border: solid black 1px;
+  box-shadow: 1px 1px #333333;
+  margin: 10px 0;
+  border-collapse: collapse;
 }
 
 #syncback-request-details tr:nth-child(even) {
@@ -140,6 +158,7 @@ pre {
 #syncback-request-details td, #syncback-request-details th {
   text-align: center;
   padding: 10px 5px;
+  border: solid black 1px;
 }
 
 .dropdown-arrow {
@@ -153,9 +172,11 @@ pre {
   position: absolute;
   background-color: white;
   text-align: left;
+  display: inline;
 }
 
 .dropdown-option {
+  position: relative;
   padding: 0px 2px;
 }
 
@@ -170,6 +191,5 @@ pre {
 .dropdown-wrapper {
   display: inline;
   cursor: pointer;
-  position: absolute;
   font-weight: normal;
 }
diff --git a/packages/nylas-dashboard/public/images/close.png b/packages/nylas-dashboard/public/images/close.png
new file mode 100644
index 0000000000000000000000000000000000000000..8c82207ef1dc9a6811d2e547e239a4b932a3e755
GIT binary patch
literal 15187
zcmeI3TWl0n7{>=e(E=9R3Q9sUEP{f}&SkGN*&~>h
z?TeI)rcn~$ftx-UeNYlaF+3Ov5o35W8sa5rG$Do<&_s+cpeEv(+1vE&_Sh)Km$S+4
z&UgOj`@VC2=X_`8WoG(%JC`-BYN9A=S$9{WAO5ehzjxmSe^2ym`xO4&lj|BbDQZcp
z{arvEfAs;160d24L)K7oo1_?7udEs=;4Nlzus20THWzcUG6pO<1x7SI>bZXQLl3Q~
zQO}csB$v#^!Kl_Xl?MY;y@Sfsm?Em4&9SCPQGx-oz>?`=Hlv$TG3s&RO7PllW<9ji
z#Ttux+U)8enwI(#Tg&O774?i-R!(BsLZRR-_`F7bgylt1WI2Hq1P1nC%t_sni;QlrEd{B>
zNdQyHYdK3ZblQ$9r;Kqc>hai#s-N<_vbky^-7K?%6xpJjV|g#fHgZyxs!wh_pK*q!
zDlEu=EYK|z`tc3@a-)W2n4`uVDXa3FiQ$?glhxb}$D7SIOl?{n6A(ceX+uwQa54wj
zeqb8oc?EP#z^SZlGb5x?AnWZ
zD#cel&Z?)C?c>z2BrEp07E|of0BVGt)mj@?)wArT)i3A#5R~G(+~>|w7zH}6uu7qv
zUULUAjkHye^PqhM?vVLaeJSp2M>Ii%weJKO?LSxwQ=a&D3o*aC`ezGKe>(l2t5Dm<
z%BZZ505!%|c5tO(ehXJx&e^3^DcAZ^jM#T336?S3QnFJ%e1xrQt7R_rAF|6aGw$>c
z2^{>ey!~5tb<3#hFy7TIqpmg2?Yd^g;DPE46lr#O*|*Qq`91)j^gt{a2>1dK7Hvj7
z`m`d*3?(#p5S#XaA_U7msL;@HcrHhmP
zsLkkhMnz(gBnS|^NpRu$AS%Iy0KuCC7oHEI5?lxnyh(83`5-F6g#f{u1Q(tUq7qyP
z5WGon;rSpc!G!?9n*c^cF
zB^{-xmByiGE^LK;uXHEc2aDgF{3_b?F}L*2oo^0wzp|pId0p{p%dntc+b*^BymN5L
z#LQ!RFCF$Yui6`r{qk9S=DhZH>-6F;JICnU!{@d>z39)Dn-{6cmYbcIhq?Wm#ya@+
zE6@9;&n7NEI=%VXHM_G1_nkNHxbf>p=N24}>%YJJ`p|t}Z|OX}@x8^9eCwe-Cyze3
zp*`@)zAs*D`C-eZzI)9zPrSAF*z)Exh4rU8SGVuDbaCbN-UDx({=WA>*U9(aZJW8j
zc}4q
   
   
+  
   
   
   
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 0a4acad2b..ac04fd317 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -45,7 +45,7 @@ class Account extends React.Component {
       return (
         

           
Error

-           this.clearError()}>Clear Error
+          
 this.clearError()}>Clear Error

           

             
               {JSON.stringify(error, null, 2)}
diff --git a/packages/nylas-dashboard/public/js/dropdown.jsx b/packages/nylas-dashboard/public/js/dropdown.jsx
index f7c001f3a..1c9e5dd9c 100644
--- a/packages/nylas-dashboard/public/js/dropdown.jsx
+++ b/packages/nylas-dashboard/public/js/dropdown.jsx
@@ -38,7 +38,7 @@ class Dropdown extends React.Component {
 
     // All options, not shown if dropdown is closed
     let options = [];
-    let optionsWrapper = ;
+    let optionsWrapper = ;
     if (!this.state.closed) {
       for (const opt of this.props.options) {
         options.push(
@@ -54,8 +54,8 @@ class Dropdown extends React.Component {
 
     return (
       
 this.close.call(this)}>
-        {selected}
         {optionsWrapper}
+        {selected}
       

     );
   }
diff --git a/packages/nylas-dashboard/public/js/modal.jsx b/packages/nylas-dashboard/public/js/modal.jsx
new file mode 100644
index 000000000..ac07a3fe0
--- /dev/null
+++ b/packages/nylas-dashboard/public/js/modal.jsx
@@ -0,0 +1,102 @@
+const React = window.React;
+
+class Modal extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      open: false,
+      onOpen: props.onOpen || () => {},
+    }
+  }
+
+  componentDidMount() {
+    this.keydownHandler = (e) => {
+      // Close modal on escape
+      if (e.keyCode === 27) {
+        this.close();
+      }
+    }
+    document.addEventListener('keydown', this.keydownHandler);
+  }
+
+  componentWillUnmount() {
+    document.removeEventListener('keydown', this.keydownHandler);
+  }
+
+  open() {
+    this.setState({open: true});
+    this.state.onOpen();
+  }
+
+  close() {
+    this.setState({open: false});
+  }
+
+  // type can be 'button' or 'div'.
+  // Always closes modal after the callback
+  renderActionElem({title, type = 'button', action = () => {}, className = ""}) {
+    const callback = (e) => {
+      action(e);
+      this.close();
+    }
+    if (type === 'button') {
+      return (
+        
+      )
+    }
+    return (
+      

+        {title}
+      

+    )
+  }
+
+  render() {
+    const activator = (
+      
 this.open.call(this)}
+      >
+        {this.props.openLink.text}
+      

+    )
+    if (!this.state.open) {
+      return activator;
+    }
+
+    const actionElems = [];
+    if (this.props.actionElems) {
+      for (const config of this.props.actionElems) {
+        actionElems.push(this.renderActionElem(config));
+      }
+    }
+
+    return (
+      

+        {activator}
+        

+          

+            

+              
 this.close.call(this)}>

+            

+            {this.props.children}
+            {actionElems}
+          

+        

+      

+    )
+  }
+}
+
+Modal.propTypes = {
+  openLink: React.PropTypes.object,
+  className: React.PropTypes.string,
+  id: React.PropTypes.string,
+  onOpen: React.PropTypes.func,
+  actionElems: React.PropTypes.arrayOf(React.PropTypes.object),
+}
+
+window.Modal = Modal;
diff --git a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
index 06bfe2e26..3e8e3451d 100644
--- a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
+++ b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
@@ -1,14 +1,7 @@
 const React = window.React;
+const Modal = window.Modal;
 
 class SetAllSyncPolicies extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {editMode: false};
-  }
-
-  edit() {
-    this.setState({editMode: true})
-  }
 
   applyToAllAccounts(accountIds) {
     const req = new XMLHttpRequest();
@@ -31,35 +24,31 @@ class SetAllSyncPolicies extends React.Component {
     }));
   }
 
-  cancel() {
-    this.setState({editMode: false});
-  }
-
   render() {
-    if (this.state.editMode) {
-      return (
-        

-           this.edit.call(this)}>
-            Set sync policies for currently displayed accounts
-          
-          

-            

-              
Sync Policy

-              
-              
-               this.cancel.call(this)}> Cancel 
-            

-          

-        

-      )
-    }
     return (
-       this.edit.call(this)}>
-        Set sync policies for currently displayed accounts
-      
+       this.applyToAllAccounts.call(this, this.props.accountIds),
+            type: 'button',
+            className: 'right-action',
+          }, {
+            title: "Cancel",
+            type: 'div',
+            className: 'action-link cancel',
+          },
+        ]}
+      >
+        
Sync Policy

+        
+      
     )
   }
 }
diff --git a/packages/nylas-dashboard/public/js/sync-policy.jsx b/packages/nylas-dashboard/public/js/sync-policy.jsx
index 5efe9655b..a0dd05315 100644
--- a/packages/nylas-dashboard/public/js/sync-policy.jsx
+++ b/packages/nylas-dashboard/public/js/sync-policy.jsx
@@ -43,7 +43,7 @@ class SyncPolicy extends React.Component {
             {this.props.stringifiedSyncPolicy}
           
           
-           this.cancel.call(this)}> Cancel 
+          
 this.cancel.call(this)}> Cancel 

         

 
       )
@@ -52,7 +52,7 @@ class SyncPolicy extends React.Component {
       

         
Sync Policy

         
{this.props.stringifiedSyncPolicy}

-         this.edit.call(this)}> Edit 
+        
 this.edit.call(this)}> Edit 

       

     )
   }
diff --git a/packages/nylas-dashboard/public/js/syncback-request-details.jsx b/packages/nylas-dashboard/public/js/syncback-request-details.jsx
index 1aecdb220..5de9c99a4 100644
--- a/packages/nylas-dashboard/public/js/syncback-request-details.jsx
+++ b/packages/nylas-dashboard/public/js/syncback-request-details.jsx
@@ -1,11 +1,10 @@
 const React = window.React;
-const Dropdown = window.Dropdown;
+const {Dropdown, Modal} = window;
 
 class SyncbackRequestDetails extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      open: false,
       accountId: props.accountId,
       syncbackRequests: null,
       counts: null,
@@ -50,117 +49,98 @@ class SyncbackRequestDetails extends React.Component {
     this.setState({statusFilter: statusFilter});
   }
 
-  open() {
-    this.getDetails();
-    this.getCounts();
-    this.setState({open: true});
-  }
-
-  close() {
-    this.setState({open: false});
-  }
-
   render() {
-    if (this.state.open) {
-      let counts =  Of requests created in the last hour: ... 
-      if (this.state.counts) {
-        const total = this.state.counts.new + this.state.counts.failed
-          + this.state.counts.succeeded;
-        if (total === 0) {
-          counts = "No requests made in the last hour";
-        } else {
-          counts = (
-            

-              Of requests created in the last hour:
-              
-                {this.state.counts.failed / total * 100}% failed
-              
-              
-                {this.state.counts.succeeded / total * 100}% succeeded
-              
-              
-                {/* .new was throwing off my syntax higlighting, so ignoring linter*/}
-                {this.state.counts['new'] / total * 100}% are still new
-              
-            

-          )
-        }
-      }
-
-      let details = "Loading..."
-      if (this.state.syncbackRequests) {
-        let reqs = JSON.parse(this.state.syncbackRequests);
-        if (this.state.statusFilter !== 'all') {
-          reqs = reqs.filter((req) => req.status === this.state.statusFilter);
-        }
-        let rows = [];
-        if (reqs.length === 0) {
-          rows.push(No results--);
-        }
-        for (let i = reqs.length - 1; i >= 0; i--) {
-          const req = reqs[i];
-          const date = new Date(req.createdAt);
-          rows.push(
-             {req.status} 
-             {req.type} 
-             {date.toLocaleTimeString()}, {date.toLocaleDateString()} 
-          )
-        }
-        details = (
-          
-            
-              
-                
-                
-                
-              
-              {rows}
-            
-          

-                  Status: 
-                   this.setStatusFilter.call(this, status)}
-                  />
-                 Type  Created At 

-        );
-      }
-
-      return (
-        

-          Syncback Request Details 
-          

-            

-              
 this.close.call(this)}>
-                X
-              

-              

-                {counts}
-                {details}
-              

-            

+    let counts = "Loading...";
+    if (this.state.counts) {
+      const total = this.state.counts.new + this.state.counts.failed
+        + this.state.counts.succeeded;
+      if (total === 0) {
+        counts = 
 No requests made in the last hour 

+      } else {
+        counts = (
+          

+            Of requests created in the last hour:
+            
+              {this.state.counts.failed / total * 100}% failed
+            
+            
+              {this.state.counts.succeeded / total * 100}% succeeded
+            
+            
+              {/* .new was throwing off my syntax higlighting, so ignoring linter*/}
+              {this.state.counts['new'] / total * 100}% are still new
+            
           

-        

+        )
+      }
+    }
+
+    let details = "Loading..."
+    if (this.state.syncbackRequests) {
+      let reqs = JSON.parse(this.state.syncbackRequests);
+      if (this.state.statusFilter !== 'all') {
+        reqs = reqs.filter((req) => req.status === this.state.statusFilter);
+      }
+      let rows = [];
+      if (reqs.length === 0) {
+        rows.push(No results--);
+      }
+      for (let i = reqs.length - 1; i >= 0; i--) {
+        const req = reqs[i];
+        const date = new Date(req.createdAt);
+        rows.push(
+           {req.status} 
+           {req.type} 
+           {date.toLocaleTimeString()}, {date.toLocaleDateString()} 
+        )
+      }
+      details = (
+        
+          
+            
+            
+            
+          
+          {rows}
+        

+              Status: 
+               this.setStatusFilter.call(this, status)}
+              />
+             Type  Created At 

       );
     }
-    // else, the modal isn't open
+
     return (
-      

-         this.open.call(this)}>
-          Syncback Request Details
-        
-      

-    );
+       {
+          this.getDetails();
+          this.getCounts();
+        }}
+      >
+        
Recent Stats

+        {counts}
+        

+        
Stored Syncback Requests

+        {details}
+      
+    )
   }
 }
 

From 1ce430cd2a2c1ed723ed6b9c3b693ddb1516ab36 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Tue, 12 Jul 2016 18:26:57 -0700
Subject: [PATCH 253/800] Add an auto refresh option to the dashboard's
 syncback requests modal

---
 packages/nylas-dashboard/public/js/modal.jsx  |  3 +++
 .../public/js/syncback-request-details.jsx    | 23 +++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/packages/nylas-dashboard/public/js/modal.jsx b/packages/nylas-dashboard/public/js/modal.jsx
index ac07a3fe0..1c5a3f032 100644
--- a/packages/nylas-dashboard/public/js/modal.jsx
+++ b/packages/nylas-dashboard/public/js/modal.jsx
@@ -6,6 +6,7 @@ class Modal extends React.Component {
     this.state = {
       open: false,
       onOpen: props.onOpen || () => {},
+      onClose: props.onClose || () => {},
     }
   }
 
@@ -30,6 +31,7 @@ class Modal extends React.Component {
 
   close() {
     this.setState({open: false});
+    this.state.onClose();
   }
 
   // type can be 'button' or 'div'.
@@ -96,6 +98,7 @@ Modal.propTypes = {
   className: React.PropTypes.string,
   id: React.PropTypes.string,
   onOpen: React.PropTypes.func,
+  onClose: React.PropTypes.func,
   actionElems: React.PropTypes.arrayOf(React.PropTypes.object),
 }
 
diff --git a/packages/nylas-dashboard/public/js/syncback-request-details.jsx b/packages/nylas-dashboard/public/js/syncback-request-details.jsx
index 5de9c99a4..c8296c1dc 100644
--- a/packages/nylas-dashboard/public/js/syncback-request-details.jsx
+++ b/packages/nylas-dashboard/public/js/syncback-request-details.jsx
@@ -9,6 +9,7 @@ class SyncbackRequestDetails extends React.Component {
       syncbackRequests: null,
       counts: null,
       statusFilter: 'all',
+      refreshInterval: null,
     };
   }
 
@@ -49,6 +50,19 @@ class SyncbackRequestDetails extends React.Component {
     this.setState({statusFilter: statusFilter});
   }
 
+  setAutoRefresh() {
+    if (document.getElementById('syncback-requests-auto')) {
+      const interval = setInterval(() => {
+        this.getCounts();
+        this.getDetails();
+      }, 3000);
+      this.setState({refreshInterval: interval});
+    } else {
+      clearInterval(this.state.refreshInterval);
+      this.setState({refreshInterval: null});
+    }
+  }
+
   render() {
     let counts = "Loading...";
     if (this.state.counts) {
@@ -133,7 +147,16 @@ class SyncbackRequestDetails extends React.Component {
           this.getDetails();
           this.getCounts();
         }}
+        onClose={() => {
+          clearInterval(this.state.refreshInterval);
+        }}
       >
+         this.setAutoRefresh.call(this)}
+        />
+        Auto Refresh 


         
Recent Stats

         {counts}
         


From e658f066dd0ea90ddbb62175e1bd020e032bcb80 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 12 Jul 2016 18:30:47 -0700
Subject: [PATCH 254/800] Change transaction schema to match current API

---
 packages/nylas-api/routes/delta.js                | 10 +++++-----
 packages/nylas-core/hook-transaction-log.js       |  8 +++++---
 packages/nylas-core/models/account/transaction.js |  4 ++--
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js
index 401568a9a..fbdd263e0 100644
--- a/packages/nylas-api/routes/delta.js
+++ b/packages/nylas-api/routes/delta.js
@@ -9,12 +9,12 @@ function keepAlive(request) {
 
 function inflateTransactions(db, transactionModels = []) {
   const transactions = _.pluck(transactionModels, "dataValues")
-  const byModel = _.groupBy(transactions, "modelName");
+  const byModel = _.groupBy(transactions, "object");
   const byObjectIds = _.groupBy(transactions, "objectId");
 
-  return Promise.all(Object.keys(byModel).map((modelName) => {
-    const ids = _.pluck(byModel[modelName], "objectId");
-    const modelConstructorName = modelName.charAt(0).toUpperCase() + modelName.slice(1);
+  return Promise.all(Object.keys(byModel).map((object) => {
+    const ids = _.pluck(byModel[object], "objectId");
+    const modelConstructorName = object.charAt(0).toUpperCase() + object.slice(1);
     const ModelKlass = db[modelConstructorName]
     let includes = [];
     if (ModelKlass.requiredAssociationsForJSON) {
@@ -25,7 +25,7 @@ function inflateTransactions(db, transactionModels = []) {
       for (const model of models) {
         const tsForId = byObjectIds[model.id];
         if (!tsForId || tsForId.length === 0) { continue; }
-        for (const t of tsForId) { t.object = model; }
+        for (const t of tsForId) { t.attributes = model; }
       }
     })
   })).then(() => transactions)
diff --git a/packages/nylas-core/hook-transaction-log.js b/packages/nylas-core/hook-transaction-log.js
index 537ce51ab..7f6f6d11f 100644
--- a/packages/nylas-core/hook-transaction-log.js
+++ b/packages/nylas-core/hook-transaction-log.js
@@ -3,8 +3,8 @@ const PubsubConnector = require('./pubsub-connector')
 module.exports = (db, sequelize) => {
   const parseHookData = ({dataValues, _changed, $modelOptions}) => {
     return {
+      object: $modelOptions.name.singular,
       objectId: dataValues.id,
-      modelName: $modelOptions.name.singular,
       changedFields: _changed,
     }
   }
@@ -34,11 +34,13 @@ module.exports = (db, sequelize) => {
     return (sequelizeHookData) => {
       if (isSilent(sequelizeHookData)) return;
 
-      const transactionData = Object.assign({type: type},
+      const event = (type === "update" ? "modify" : type)
+
+      const transactionData = Object.assign({event},
         parseHookData(sequelizeHookData)
       );
       db.Transaction.create(transactionData);
-      transactionData.object = sequelizeHookData.dataValues;
+      transactionData.attributes = sequelizeHookData.dataValues;
 
       PubsubConnector.notifyDelta(db.accountId, transactionData);
     }
diff --git a/packages/nylas-core/models/account/transaction.js b/packages/nylas-core/models/account/transaction.js
index b089c7cdb..0ea09d8a0 100644
--- a/packages/nylas-core/models/account/transaction.js
+++ b/packages/nylas-core/models/account/transaction.js
@@ -2,9 +2,9 @@ const {JSONARRAYType} = require('../../database-types');
 
 module.exports = (sequelize, Sequelize) => {
   const Transaction = sequelize.define('transaction', {
-    type: Sequelize.STRING,
+    event: Sequelize.STRING,
+    object: Sequelize.STRING,
     objectId: Sequelize.STRING,
-    modelName: Sequelize.STRING,
     changedFields: JSONARRAYType('changedFields'),
   });
 

From e89975456b6b28a089accecb50df30af70bb480d Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 12 Jul 2016 18:31:55 -0700
Subject: [PATCH 255/800] Fix scenario where sync worker is re-assigned the
 same account before fully releasing it

---
 packages/nylas-sync/sync-process-manager.js | 3 ++-
 packages/nylas-sync/sync-worker.js          | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index 3d205c384..d1ee98c9f 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -194,13 +194,14 @@ class SyncProcessManager {
     const dst = ACCOUNTS_UNCLAIMED;
 
     return PubsubConnector.broadcastClient().lremAsync(src, 1, accountId).then((didRemove) => {
+      this._workers[accountId] = null;
+
       if (didRemove) {
         PubsubConnector.broadcastClient().rpushAsync(dst, accountId)
       } else {
         this._logger.error("Wanted to return item to pool, but didn't have claim on it.")
         return
       }
-      this._workers[accountId] = null;
     });
   }
 }
diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 992ef3a52..1b6c7c801 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -225,6 +225,7 @@ class SyncWorker {
 
     this._account.lastSyncCompletions = lastSyncCompletions
     this._account.save()
+
     this._logger.info('Syncworker: Completed sync cycle')
 
     if (afterSync === 'idle') {

From 17f920fb90d6100a8091e29ed3ee15a073a5b1d3 Mon Sep 17 00:00:00 2001
From: Annie 
Date: Tue, 12 Jul 2016 18:31:30 -0700
Subject: [PATCH 256/800] Ordered inbox messages chronologically

---
 packages/nylas-message-processor/processors/contact.js   | 3 +--
 packages/nylas-message-processor/processors/threading.js | 6 ++----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/packages/nylas-message-processor/processors/contact.js b/packages/nylas-message-processor/processors/contact.js
index 38397e60a..f4a984cbb 100644
--- a/packages/nylas-message-processor/processors/contact.js
+++ b/packages/nylas-message-processor/processors/contact.js
@@ -27,9 +27,8 @@ class ContactProcessor {
   }
 
 
-  processMessage({db, message, logger}) {
+  processMessage({db, message}) {
     const {Contact} = db;
-    this.logger = logger
 
     let allContacts = []
     const fields = ['to', 'from', 'bcc', 'cc']
diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js
index 2f9c847fd..02d6b7ff9 100644
--- a/packages/nylas-message-processor/processors/threading.js
+++ b/packages/nylas-message-processor/processors/threading.js
@@ -77,7 +77,6 @@ class ThreadingProcessor {
       throw new Error("Threading processMessage expects folder value to be present.");
     }
 
-    this.logger = logger
 
     const {Folder, Label} = db;
     let findOrCreateThread = null;
@@ -140,13 +139,12 @@ class ThreadingProcessor {
         isSent = !!message.labels.find(l => l.id === sentLabel.id)
       }
 
-      if (isSent && (message.date > thread.lastMessageSentDate)) {
+      if (isSent && ((message.date > thread.lastMessageSentDate) || !thread.lastMessageSentDate)) {
         thread.lastMessageSentDate = message.date;
       }
-      if (!isSent && (message.date > thread.lastMessageReceivedDate)) {
+      if (!isSent && ((message.date > thread.lastMessageReceivedDate) || !thread.lastMessageReceivedDate)) {
         thread.lastMessageReceivedDate = message.date;
       }
-
       // update folders and labels
       if (!thread.folders.find(f => f.id === message.folderId)) {
         thread.addFolder(message.folder)

From 8629b5a055a4439a399b3d2b9c907219d355c66a Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 10:57:05 -0700
Subject: [PATCH 257/800] New function to stub dashboard view

---
 packages/nylas-dashboard/app.js | 102 ++++++++++++++++++++++----------
 1 file changed, 72 insertions(+), 30 deletions(-)

diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js
index dbd02d606..99931e3cc 100644
--- a/packages/nylas-dashboard/app.js
+++ b/packages/nylas-dashboard/app.js
@@ -21,6 +21,76 @@ const attach = (directory) => {
   });
 }
 
+const onAccountsWebsocketConnected = (wss, ws) => {
+  DatabaseConnector.forShared().then(({Account}) => {
+    Account.findAll().then((accounts) => {
+      accounts.forEach((acct) => {
+        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+      });
+    });
+
+    this.observable = PubsubConnector.observeAllAccounts().subscribe((accountId) => {
+      Account.find({where: {id: accountId}}).then((acct) => {
+        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+      });
+    });
+
+    this.pollInterval = setInterval(() => {
+      SchedulerUtils.listActiveAccounts().then((accountIds) => {
+        ws.send(JSON.stringify({ cmd: "ACTIVE", payload: accountIds}))
+      });
+      const assignments = {};
+      SchedulerUtils.forEachAccountList((identity, accountIds) => {
+        for (const accountId of accountIds) {
+          assignments[accountId] = identity;
+        }
+      }).then(() =>
+        ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments}))
+      )
+    }, 1000);
+  });
+};
+
+const onAccountsWebsocketConnectedFake = (wss, ws) => {
+  const accts = [];
+  for (let ii = 0; ii < 300; ii++) {
+    const acct = {
+      id: ii,
+      email_address: `halla+${ii}@nylas.com`,
+      object: "account",
+      organization_unit: "folder",
+      provider: "imap",
+      connection_settings: {
+        imap_host: "imap.mail.me.com",
+        imap_port: 993,
+        smtp_host: "smtp.mail.me.com",
+        smtp_port: 0,
+        ssl_required: true,
+      },
+      sync_policy: {
+        afterSync: "idle",
+        intervals: {
+          active: 30000,
+          inactive: 300000,
+        },
+        folderSyncOptions: {
+          deepFolderScan: 600000,
+        },
+      },
+      sync_error: null,
+      first_sync_completion: 0,
+      last_sync_completions: [],
+      created_at: "2016-07-13T00:49:25.000Z",
+    };
+    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+    accts.push(acct);
+  }
+  setInterval(() => {
+    const acct = accts[Math.floor(Math.random() * accts.length)];
+    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+  }, 250);
+}
+
 server.register([HapiWebSocket, Inert], () => {
   attach('./routes/')
 
@@ -31,36 +101,8 @@ server.register([HapiWebSocket, Inert], () => {
       plugins: {
         websocket: {
           only: true,
-          connect: (wss, ws) => {
-            DatabaseConnector.forShared().then(({Account}) => {
-              Account.findAll().then((accounts) => {
-                accounts.forEach((acct) => {
-                  ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
-                });
-              });
-
-              this.observable = PubsubConnector.observeAllAccounts().subscribe((accountId) => {
-                Account.find({where: {id: accountId}}).then((acct) => {
-                  ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
-                });
-              });
-
-              this.pollInterval = setInterval(() => {
-                SchedulerUtils.listActiveAccounts().then((accountIds) => {
-                  ws.send(JSON.stringify({ cmd: "ACTIVE", payload: accountIds}))
-                });
-                const assignments = {};
-                SchedulerUtils.forEachAccountList((identity, accountIds) => {
-                  for (const accountId of accountIds) {
-                    assignments[accountId] = identity;
-                  }
-                }).then(() =>
-                  ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments}))
-                )
-              }, 1000);
-            });
-          },
-          disconnect: () => {
+          connect: onAccountsWebsocketConnectedFake,
+          disconnect: function disconnect() {
             clearInterval(this.pollInterval);
             this.observable.dispose();
           },

From ecc8bd67d3c69534a79e4650bc1128718d3c0ce5 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 11:11:32 -0700
Subject: [PATCH 258/800] Clean up dashboard websocket code a bit

---
 packages/nylas-dashboard/app.js              | 103 +------------------
 packages/nylas-dashboard/public/js/app.jsx   |   4 +-
 packages/nylas-dashboard/routes/websocket.js | 102 ++++++++++++++++++
 3 files changed, 109 insertions(+), 100 deletions(-)
 create mode 100644 packages/nylas-dashboard/routes/websocket.js

diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js
index 99931e3cc..0284429ae 100644
--- a/packages/nylas-dashboard/app.js
+++ b/packages/nylas-dashboard/app.js
@@ -1,9 +1,9 @@
-const Hapi = require('hapi');
-const HapiWebSocket = require('hapi-plugin-websocket');
-const Inert = require('inert');
-const {DatabaseConnector, PubsubConnector, SchedulerUtils, Logger} = require(`nylas-core`);
 const fs = require('fs');
 const path = require('path');
+const Inert = require('inert');
+const Hapi = require('hapi');
+const HapiWebSocket = require('hapi-plugin-websocket');
+const {Logger} = require(`nylas-core`);
 
 global.Promise = require('bluebird');
 global.Logger = Logger.createLogger('nylas-k2-dashboard')
@@ -21,102 +21,9 @@ const attach = (directory) => {
   });
 }
 
-const onAccountsWebsocketConnected = (wss, ws) => {
-  DatabaseConnector.forShared().then(({Account}) => {
-    Account.findAll().then((accounts) => {
-      accounts.forEach((acct) => {
-        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
-      });
-    });
-
-    this.observable = PubsubConnector.observeAllAccounts().subscribe((accountId) => {
-      Account.find({where: {id: accountId}}).then((acct) => {
-        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
-      });
-    });
-
-    this.pollInterval = setInterval(() => {
-      SchedulerUtils.listActiveAccounts().then((accountIds) => {
-        ws.send(JSON.stringify({ cmd: "ACTIVE", payload: accountIds}))
-      });
-      const assignments = {};
-      SchedulerUtils.forEachAccountList((identity, accountIds) => {
-        for (const accountId of accountIds) {
-          assignments[accountId] = identity;
-        }
-      }).then(() =>
-        ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments}))
-      )
-    }, 1000);
-  });
-};
-
-const onAccountsWebsocketConnectedFake = (wss, ws) => {
-  const accts = [];
-  for (let ii = 0; ii < 300; ii++) {
-    const acct = {
-      id: ii,
-      email_address: `halla+${ii}@nylas.com`,
-      object: "account",
-      organization_unit: "folder",
-      provider: "imap",
-      connection_settings: {
-        imap_host: "imap.mail.me.com",
-        imap_port: 993,
-        smtp_host: "smtp.mail.me.com",
-        smtp_port: 0,
-        ssl_required: true,
-      },
-      sync_policy: {
-        afterSync: "idle",
-        intervals: {
-          active: 30000,
-          inactive: 300000,
-        },
-        folderSyncOptions: {
-          deepFolderScan: 600000,
-        },
-      },
-      sync_error: null,
-      first_sync_completion: 0,
-      last_sync_completions: [],
-      created_at: "2016-07-13T00:49:25.000Z",
-    };
-    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
-    accts.push(acct);
-  }
-  setInterval(() => {
-    const acct = accts[Math.floor(Math.random() * accts.length)];
-    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
-  }, 250);
-}
-
 server.register([HapiWebSocket, Inert], () => {
   attach('./routes/')
 
-  server.route({
-    method: "POST",
-    path: "/accounts",
-    config: {
-      plugins: {
-        websocket: {
-          only: true,
-          connect: onAccountsWebsocketConnectedFake,
-          disconnect: function disconnect() {
-            clearInterval(this.pollInterval);
-            this.observable.dispose();
-          },
-        },
-      },
-    },
-    handler: (request, reply) => {
-      if (request.payload.cmd === "PING") {
-        reply(JSON.stringify({ result: "PONG" }));
-        return;
-      }
-    },
-  });
-
   server.route({
     method: 'GET',
     path: '/ping',
@@ -124,7 +31,7 @@ server.register([HapiWebSocket, Inert], () => {
       auth: false,
     },
     handler: (request, reply) => {
-      console.log("---> Ping!")
+      global.Logger.info("---> Ping!")
       reply("pong")
     },
   });
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index ac04fd317..7d66c075e 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -119,9 +119,9 @@ class Root extends React.Component {
   componentDidMount() {
     let url = null;
     if (window.location.protocol === "https:") {
-      url = `wss://${window.location.host}/accounts`;
+      url = `wss://${window.location.host}/websocket`;
     } else {
-      url = `ws://${window.location.host}/accounts`;
+      url = `ws://${window.location.host}/websocket`;
     }
     this.websocket = new WebSocket(url);
     this.websocket.onopen = () => {
diff --git a/packages/nylas-dashboard/routes/websocket.js b/packages/nylas-dashboard/routes/websocket.js
new file mode 100644
index 000000000..496488d7b
--- /dev/null
+++ b/packages/nylas-dashboard/routes/websocket.js
@@ -0,0 +1,102 @@
+const {
+  DatabaseConnector,
+  PubsubConnector,
+  SchedulerUtils,
+} = require(`nylas-core`);
+
+function onWebsocketConnected(wss, ws) {
+  DatabaseConnector.forShared().then(({Account}) => {
+    Account.findAll().then((accounts) => {
+      accounts.forEach((acct) => {
+        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+      });
+    });
+
+    this.observable = PubsubConnector.observeAllAccounts().subscribe((accountId) => {
+      Account.find({where: {id: accountId}}).then((acct) => {
+        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+      });
+    });
+
+    this.pollInterval = setInterval(() => {
+      SchedulerUtils.listActiveAccounts().then((accountIds) => {
+        ws.send(JSON.stringify({ cmd: "ACTIVE", payload: accountIds}))
+      });
+      const assignments = {};
+      SchedulerUtils.forEachAccountList((identity, accountIds) => {
+        for (const accountId of accountIds) {
+          assignments[accountId] = identity;
+        }
+      }).then(() =>
+        ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments}))
+      )
+    }, 1000);
+  });
+}
+
+function onWebsocketDisconnected() {
+  clearInterval(this.pollInterval);
+  this.observable.dispose();
+}
+
+function onWebsocketConnectedFake(wss, ws) {
+  const accts = [];
+  for (let ii = 0; ii < 300; ii++) {
+    const acct = {
+      id: ii,
+      email_address: `halla+${ii}@nylas.com`,
+      object: "account",
+      organization_unit: "folder",
+      provider: "imap",
+      connection_settings: {
+        imap_host: "imap.mail.me.com",
+        imap_port: 993,
+        smtp_host: "smtp.mail.me.com",
+        smtp_port: 0,
+        ssl_required: true,
+      },
+      sync_policy: {
+        afterSync: "idle",
+        intervals: {
+          active: 30000,
+          inactive: 300000,
+        },
+        folderSyncOptions: {
+          deepFolderScan: 600000,
+        },
+      },
+      sync_error: null,
+      first_sync_completion: 0,
+      last_sync_completions: [],
+      created_at: "2016-07-13T00:49:25.000Z",
+    };
+    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+    accts.push(acct);
+  }
+  setInterval(() => {
+    const acct = accts[Math.floor(Math.random() * accts.length)];
+    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+  }, 250);
+}
+
+module.exports = (server) => {
+  server.route({
+    method: "POST",
+    path: "/websocket",
+    config: {
+      plugins: {
+        websocket: {
+          only: true,
+          connect: onWebsocketConnected,
+          disconnect: onWebsocketDisconnected,
+        },
+      },
+    },
+    handler: (request, reply) => {
+      if (request.payload.cmd === "PING") {
+        reply(JSON.stringify({ result: "PONG" }));
+        return;
+      }
+    },
+  });
+}

From 7056a4b8f487ab5de3dbe6be84662efbafb8fde8 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 11:30:22 -0700
Subject: [PATCH 259/800] Fix bug with accounts that have uidmin=0, not 1

---
 packages/nylas-sync/imap/fetch-messages-in-folder.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js
index 9ce3fe108..038ae281b 100644
--- a/packages/nylas-sync/imap/fetch-messages-in-folder.js
+++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js
@@ -299,7 +299,7 @@ class FetchMessagesInFolder {
 
   _fetchUnsyncedMessages() {
     const savedSyncState = this._category.syncState;
-    const isFirstSync = !savedSyncState.fetchedmax;
+    const isFirstSync = savedSyncState.fetchedmax === undefined;
     const boxUidnext = this._box.uidnext;
     const boxUidvalidity = this._box.uidvalidity;
 
@@ -350,7 +350,7 @@ class FetchMessagesInFolder {
 
   _runScan() {
     const {fetchedmin, fetchedmax} = this._category.syncState;
-    if (!fetchedmin || !fetchedmax) {
+    if ((fetchedmin === undefined) || (fetchedmax === undefined)) {
       throw new Error("Unseen messages must be fetched at least once before the first update/delete scan.")
     }
     return this._shouldRunDeepScan() ? this._runDeepScan() : this._runShallowScan()

From 6e19cd84f5a19392755f3dd4f62c5bfd76de9fbe Mon Sep 17 00:00:00 2001
From: Annie 
Date: Wed, 13 Jul 2016 11:35:07 -0700
Subject: [PATCH 260/800] Added view query param for contact and message routes

---
 packages/nylas-api/routes/contacts.js | 1 +
 packages/nylas-api/routes/messages.js | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/nylas-api/routes/contacts.js b/packages/nylas-api/routes/contacts.js
index c763d5b61..549ce5cb8 100644
--- a/packages/nylas-api/routes/contacts.js
+++ b/packages/nylas-api/routes/contacts.js
@@ -13,6 +13,7 @@ module.exports = (server) => {
         query: {
           name: Joi.string(),
           email: Joi.string().email(),
+          view: Joi.string().valid('expanded', 'count'),
           limit: Joi.number().integer().min(1).max(2000).default(100),
           offset: Joi.number().integer().min(0).default(0),
         },
diff --git a/packages/nylas-api/routes/messages.js b/packages/nylas-api/routes/messages.js
index 474f773fe..c8ea00b1e 100644
--- a/packages/nylas-api/routes/messages.js
+++ b/packages/nylas-api/routes/messages.js
@@ -23,6 +23,7 @@ module.exports = (server) => {
           'in': Joi.string(),
           'limit': Joi.number().integer().min(1).max(2000).default(100),
           'offset': Joi.number().integer().min(0).default(0),
+          'view': Joi.string().valid('expanded', 'count'),
         },
       },
       response: {

From 1c0c4ee61e988d5c6e33fb72fee2796a92019096 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 13 Jul 2016 11:41:29 -0700
Subject: [PATCH 261/800] Add redis-server back into pm2

---
 pm2-dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pm2-dev.yml b/pm2-dev.yml
index 21c1ea226..bacc7248f 100644
--- a/pm2-dev.yml
+++ b/pm2-dev.yml
@@ -32,3 +32,5 @@ apps:
       DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
       DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
       NODE_ENV: 'development'
+  - script   : redis-server
+    name     : redis

From ffc2593c4e09c4630500b7b4f082c00680eb215b Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 11:42:34 -0700
Subject: [PATCH 262/800] Log all box attributes to help track down bug

---
 packages/nylas-sync/imap/fetch-folder-list.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/nylas-sync/imap/fetch-folder-list.js b/packages/nylas-sync/imap/fetch-folder-list.js
index d59d5794e..36675fb16 100644
--- a/packages/nylas-sync/imap/fetch-folder-list.js
+++ b/packages/nylas-sync/imap/fetch-folder-list.js
@@ -57,6 +57,12 @@ class FetchFolderList {
         continue;
       }
 
+      this._logger.info({
+        box_name: boxName,
+        attributes: JSON.stringify(box.attribs),
+      }, `FetchFolderList: Box Information`)
+
+
       if (box.children && box.attribs.includes('\\HasChildren')) {
         Object.keys(box.children).forEach((subname) => {
           stack.push([`${boxName}${box.delimiter}${subname}`, box.children[subname]]);

From 7f72ab7f14ae08137f727fc7006e71c582d44f6d Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 13 Jul 2016 11:58:46 -0700
Subject: [PATCH 263/800] Add loggly logging service to test it out

- Adds env to all logs, cleans up code a bit
---
 package.json                       |  1 +
 packages/nylas-core/log-streams.js | 57 ++++++++++++++++++++++++++++++
 packages/nylas-core/logger.js      | 36 ++-----------------
 3 files changed, 60 insertions(+), 34 deletions(-)
 create mode 100644 packages/nylas-core/log-streams.js

diff --git a/package.json b/package.json
index a6cdce36b..e2dcb9481 100644
--- a/package.json
+++ b/package.json
@@ -7,6 +7,7 @@
     "bluebird": "3.x.x",
     "bunyan": "1.8.0",
     "bunyan-cloudwatch": "2.0.0",
+    "bunyan-loggly": "^1.0.0",
     "bunyan-prettystream": "^0.1.3",
     "lerna": "2.0.0-beta.23",
     "mysql": "^2.11.1",
diff --git a/packages/nylas-core/log-streams.js b/packages/nylas-core/log-streams.js
new file mode 100644
index 000000000..fdd5746b5
--- /dev/null
+++ b/packages/nylas-core/log-streams.js
@@ -0,0 +1,57 @@
+const os = require('os');
+const createCWStream = require('bunyan-cloudwatch')
+const PrettyStream = require('bunyan-prettystream')
+const Bunyan2Loggly = require('bunyan-loggly')
+
+const {LOGGLY_TOKEN} = process.env
+const logglyConfig = (name, env) => ({
+  token: LOGGLY_TOKEN,
+  subdomain: 'nylas',
+  tags: [`${name}-${env}`],
+})
+const cloudwatchConfig = (name, env) => ({
+  logGroupName: `k2-${env}`,
+  logStreamName: `${name}-${env}-${os.hostname()}`,
+  cloudWatchLogsOptions: {
+    region: 'us-east-1',
+  },
+})
+
+const stdoutStream = {
+  level: 'info',
+  stream: process.stdout,
+}
+
+const getLogStreams = (name, env) => {
+  switch (env) {
+    case 'development': {
+      const prettyStdOut = new PrettyStream();
+      prettyStdOut.pipe(process.stdout);
+      return [
+        {
+          type: 'raw',
+          level: 'debug',
+          stream: prettyStdOut,
+          reemitErrorEvents: true,
+        },
+      ]
+    }
+    default: {
+      return [
+        stdoutStream,
+        {
+          type: 'raw',
+          reemitErrorEvents: true,
+          stream: new Bunyan2Loggly(logglyConfig(name, env)),
+        },
+        {
+          type: 'raw',
+          reemitErrorEvents: true,
+          stream: createCWStream(cloudwatchConfig(name, env)),
+        },
+      ]
+    }
+  }
+}
+
+module.exports = {getLogStreams}
diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js
index b77ba2c29..b50e704f9 100644
--- a/packages/nylas-core/logger.js
+++ b/packages/nylas-core/logger.js
@@ -1,44 +1,12 @@
-const os = require('os');
 const bunyan = require('bunyan')
-const createCWStream = require('bunyan-cloudwatch')
-const PrettyStream = require('bunyan-prettystream');
+const {getLogStreams} = require('./log-streams')
 const NODE_ENV = process.env.NODE_ENV || 'unknown'
 
-
-function getLogStreams(name, env) {
-  if (env === 'development') {
-    const prettyStdOut = new PrettyStream();
-    prettyStdOut.pipe(process.stdout);
-    const stdoutStream = {
-      type: 'raw',
-      level: 'debug',
-      stream: prettyStdOut,
-    }
-    return [stdoutStream]
-  }
-
-  const stdoutStream = {
-    stream: process.stdout,
-    level: 'info',
-  }
-  const cloudwatchStream = {
-    stream: createCWStream({
-      logGroupName: `k2-${env}`,
-      logStreamName: `${name}-${env}-${os.hostname()}`,
-      cloudWatchLogsOptions: {
-        region: 'us-east-1',
-      },
-    }),
-    type: 'raw',
-    reemitErrorEvents: true,
-  }
-  return [stdoutStream, cloudwatchStream]
-}
-
 function createLogger(name, env = NODE_ENV) {
   const childLogs = new Map()
   const logger = bunyan.createLogger({
     name,
+    env,
     serializers: bunyan.stdSerializers,
     streams: getLogStreams(name, env),
   })

From f2f6dec35ac7127994824ab6184cdc553cb8814c Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 12:35:30 -0700
Subject: [PATCH 264/800] Add dashboard optimizations

Make account elements absolutely positioned and opaque
Send account updates all at once, at timed intervals
---
 packages/nylas-dashboard/public/css/app.css  |  7 +--
 packages/nylas-dashboard/public/js/app.jsx   | 57 +++++++++++++-------
 packages/nylas-dashboard/routes/websocket.js | 42 +++++++++++----
 3 files changed, 72 insertions(+), 34 deletions(-)

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index 754519847..27c272722 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -18,13 +18,14 @@ pre {
 }
 
 .account {
-  display: inline-block;
+  position: absolute;
   border-radius: 5px;
   width: 300px;
-  background-color: rgba(255, 255, 255, 0.6);
+  height: 500px;
+  background-color: rgb(255, 255, 255);
   padding: 15px;
   margin: 5px;
-  vertical-align: top;
+  overflow: auto;
 }
 
 .account h3 {
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 7d66c075e..e84f6d2f9 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -11,6 +11,21 @@ const {
   SyncbackRequestDetails,
 } = window;
 
+function calcAcctPosition(count) {
+  const width = 340;
+  const height = 540;
+  const marginTop = 100;
+  const marginSide = 0;
+
+  const acctsPerRow = Math.floor((window.innerWidth - 2 * marginSide) / width);
+  const row = Math.floor(count / acctsPerRow)
+  const col = count - (row * acctsPerRow);
+  const top = marginTop + (row * height);
+  const left = marginSide + (width * col);
+
+  return {left: left, top: top};
+}
+
 class Account extends React.Component {
   constructor(props) {
     super(props);
@@ -72,8 +87,13 @@ class Account extends React.Component {
       firstSyncDuration = (new Date(account.first_sync_completion) - new Date(account.created_at)) / 1000;
     }
 
+    const position = calcAcctPosition(this.props.count);
+
     return (
-      

+      

         
{account.email_address} [{account.id}] {active ? '🌕' : '🌑'}

         {assignment}
         
@@ -102,6 +122,7 @@ Account.propTypes = {
   account: React.PropTypes.object,
   active: React.PropTypes.bool,
   assignment: React.PropTypes.string,
+  count: React.PropTypes.number,
 }
 
 class Root extends React.Component {
@@ -130,14 +151,8 @@ class Root extends React.Component {
     this.websocket.onmessage = (evt) => {
       try {
         const msg = JSON.parse(evt.data);
-        if (msg.cmd === 'ACCOUNT') {
-          this.onReceivedAccount(msg.payload);
-        }
-        if (msg.cmd === 'ASSIGNMENTS') {
-          this.onReceivedAssignments(msg.payload);
-        }
-        if (msg.cmd === 'ACTIVE') {
-          this.onReceivedActiveAccountIds(msg.payload);
+        if (msg.cmd === 'UPDATE') {
+          this.onReceivedUpdate(msg.payload);
         }
       } catch (err) {
         console.error(err);
@@ -148,18 +163,17 @@ class Root extends React.Component {
     };
   }
 
-  onReceivedAssignments(assignments) {
-    this.setState({assignments})
-  }
-
-  onReceivedActiveAccountIds(accountIds) {
-    this.setState({activeAccountIds: accountIds})
-  }
-
-  onReceivedAccount(account) {
+  onReceivedUpdate(update) {
     const accounts = Object.assign({}, this.state.accounts);
-    accounts[account.id] = account;
-    this.setState({accounts});
+    for (const account of update.updatedAccounts) {
+      accounts[account.id] = account;
+    }
+
+    this.setState({
+      assignments: update.assignments || this.state.assignments,
+      activeAccountIds: update.activeAccountIds || this.state.activeAccountIds,
+      accounts: accounts,
+    })
   }
 
   onFilter() {
@@ -180,6 +194,8 @@ class Root extends React.Component {
         break;
     }
 
+    let count = 0;
+
     return (
       

          this.onFilter.call(this)} />
@@ -191,6 +207,7 @@ class Root extends React.Component {
               active={this.state.activeAccountIds.includes(id)}
               assignment={this.state.assignments[id]}
               account={this.state.accounts[id]}
+              count={count++}
             />
           )
         }
diff --git a/packages/nylas-dashboard/routes/websocket.js b/packages/nylas-dashboard/routes/websocket.js
index 496488d7b..41b38af51 100644
--- a/packages/nylas-dashboard/routes/websocket.js
+++ b/packages/nylas-dashboard/routes/websocket.js
@@ -5,31 +5,51 @@ const {
 } = require(`nylas-core`);
 
 function onWebsocketConnected(wss, ws) {
+  let toSend;
+  function resetToSend() {
+    toSend = {
+      updatedAccounts: [],
+      activeAccountIds: [],
+      assignments: {},
+    };
+  }
+  resetToSend();
+
+  function sendUpdate() {
+    ws.send(JSON.stringify({cmd: "UPDATE", payload: toSend}));
+    resetToSend();
+  }
+
   DatabaseConnector.forShared().then(({Account}) => {
     Account.findAll().then((accounts) => {
       accounts.forEach((acct) => {
-        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+        toSend.updatedAccounts.push(acct);
+        if (toSend.updatedAccounts.length >= 50) {
+          sendUpdate();
+        }
       });
+      sendUpdate();
     });
 
     this.observable = PubsubConnector.observeAllAccounts().subscribe((accountId) => {
       Account.find({where: {id: accountId}}).then((acct) => {
-        ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+        toSend.updatedAccounts.push(acct);
       });
     });
 
     this.pollInterval = setInterval(() => {
-      SchedulerUtils.listActiveAccounts().then((accountIds) => {
-        ws.send(JSON.stringify({ cmd: "ACTIVE", payload: accountIds}))
+      const getActiveIds = SchedulerUtils.listActiveAccounts().then((accountIds) => {
+        toSend.activeAccountIds = accountIds;
       });
-      const assignments = {};
-      SchedulerUtils.forEachAccountList((identity, accountIds) => {
+      const getAssignments = SchedulerUtils.forEachAccountList((identity, accountIds) => {
         for (const accountId of accountIds) {
-          assignments[accountId] = identity;
+          toSend.assignments[accountId] = identity;
         }
-      }).then(() =>
-        ws.send(JSON.stringify({ cmd: "ASSIGNMENTS", payload: assignments}))
-      )
+      })
+
+      Promise.all([getActiveIds, getAssignments]).then(() => {
+        sendUpdate();
+      })
     }, 1000);
   });
 }
@@ -41,7 +61,7 @@ function onWebsocketDisconnected() {
 
 function onWebsocketConnectedFake(wss, ws) {
   const accts = [];
-  for (let ii = 0; ii < 300; ii++) {
+  for (let ii = 0; ii < 100; ii++) {
     const acct = {
       id: ii,
       email_address: `halla+${ii}@nylas.com`,

From 3d64affc52c495679349af96760e3b59524eae50 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 13 Jul 2016 12:37:24 -0700
Subject: [PATCH 265/800] Delta stream adhere to old Nylas spec

---
 packages/nylas-api/routes/delta.js          | 31 ++++++++++++++-------
 packages/nylas-core/hook-transaction-log.js |  9 ++++--
 2 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js
index fbdd263e0..21f3fc3c3 100644
--- a/packages/nylas-api/routes/delta.js
+++ b/packages/nylas-api/routes/delta.js
@@ -9,6 +9,7 @@ function keepAlive(request) {
 
 function inflateTransactions(db, transactionModels = []) {
   const transactions = _.pluck(transactionModels, "dataValues")
+  transactions.forEach((t) => t.cursor = t.id);
   const byModel = _.groupBy(transactions, "object");
   const byObjectIds = _.groupBy(transactions, "objectId");
 
@@ -23,12 +24,13 @@ function inflateTransactions(db, transactionModels = []) {
     return ModelKlass.findAll({where: {id: ids}, include: includes})
     .then((models = []) => {
       for (const model of models) {
+        model.dataValues.object = object
         const tsForId = byObjectIds[model.id];
         if (!tsForId || tsForId.length === 0) { continue; }
-        for (const t of tsForId) { t.attributes = model; }
+        for (const t of tsForId) { t.attributes = model.dataValues; }
       }
     })
-  })).then(() => transactions)
+  })).then(() => transactions.map(JSON.stringify).join("\n"))
 }
 
 function createOutputStream() {
@@ -41,11 +43,15 @@ function createOutputStream() {
   return outputStream
 }
 
+function lastTransaction(db) {
+  return db.Transaction.findOne({order: [['id', 'DESC']]})
+}
+
 function initialTransactions(db, request) {
-  const getParams = request.query || {}
-  const since = new Date(getParams.since || Date.now())
+  let cursor = (request.query || {}).cursor;
+  const where = cursor ? {id: {$gt: cursor}} : {createdAt: {$gte: new Date()}}
   return db.Transaction
-           .streamAll({where: {createdAt: {$gte: since}}})
+           .streamAll({where})
            .flatMap((objs) => inflateTransactions(db, objs))
 }
 
@@ -55,21 +61,26 @@ module.exports = (server) => {
     path: '/delta/streaming',
     handler: (request, reply) => {
       const outputStream = createOutputStream();
-      const account = request.auth.credentials;
 
       request.getAccountDatabase().then((db) => {
         const source = Rx.Observable.merge(
-          PubsubConnector.observeDeltas(account.id),
+          PubsubConnector.observeDeltas(request.auth.credentials.id),
           initialTransactions(db, request),
           keepAlive(request)
         ).subscribe(outputStream.pushJSON)
 
-        request.on("disconnect", () => {
-          source.dispose()
-        });
+        request.on("disconnect", source.dispose.bind(source));
       });
 
       reply(outputStream)
     },
   });
+
+  server.route({
+    method: 'POST',
+    path: '/delta/latest_cursor',
+    handler: (request, reply) => request.getAccountDatabase().then((db) =>
+      lastTransaction(db).then((t) => reply({cursor: t.id}))
+    ),
+  });
 };
diff --git a/packages/nylas-core/hook-transaction-log.js b/packages/nylas-core/hook-transaction-log.js
index 7f6f6d11f..0e1247d9c 100644
--- a/packages/nylas-core/hook-transaction-log.js
+++ b/packages/nylas-core/hook-transaction-log.js
@@ -39,10 +39,13 @@ module.exports = (db, sequelize) => {
       const transactionData = Object.assign({event},
         parseHookData(sequelizeHookData)
       );
-      db.Transaction.create(transactionData);
-      transactionData.attributes = sequelizeHookData.dataValues;
+      db.Transaction.create(transactionData).then((transaction) => {
+        const dataValues = transaction.dataValues
+        dataValues.attributes = sequelizeHookData.dataValues;
+        dataValues.cursor = transaction.id;
+        PubsubConnector.notifyDelta(db.accountId, dataValues);
+      })
 
-      PubsubConnector.notifyDelta(db.accountId, transactionData);
     }
   }
 

From a11ee2987bacbd0eaf14d194fbf17c3f50553144 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 12:39:07 -0700
Subject: [PATCH 266/800] Change proxy to honor null values, create helper for
 open box name

---
 packages/nylas-core/imap-box.js        | 28 +++++++++++++++++---------
 packages/nylas-core/imap-connection.js |  4 ++++
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/packages/nylas-core/imap-box.js b/packages/nylas-core/imap-box.js
index ec9283749..6c5a21c69 100644
--- a/packages/nylas-core/imap-box.js
+++ b/packages/nylas-core/imap-box.js
@@ -4,23 +4,31 @@ const {
   IMAPConnectionNotReadyError,
 } = require('./imap-errors');
 
+/*
+IMAPBox uses Proxy to wrap the "box" exposed by node-imap. It provides higher-level
+primitives, but you can still call through to properties / methods of the node-imap
+box, ala `imapbox.uidvalidity`
+*/
 class IMAPBox {
   constructor(imapConn, box) {
     this._conn = imapConn
     this._box = box
 
     return new Proxy(this, {
-      get(target, name) {
-        const prop = Reflect.get(target, name)
-        if (!prop) {
-          return Reflect.get(target._box, name)
+      get(obj, prop) {
+        const val = (prop in obj) ? obj[prop] : obj._box[prop];
+
+        if (_.isFunction(val)) {
+          const myBox = obj._box.name;
+          const openBox = obj._conn.getOpenBoxName()
+          if (myBox !== openBox) {
+            return () => {
+              throw new Error(`IMAPBox::${prop} - Mailbox is no longer selected on the IMAPConnection (${myBox} != ${openBox}).`);
+            }
+          }
         }
-        if (_.isFunction(prop) && target._conn._imap._box.name !== target._box.name) {
-          return () => Promise.reject(
-            new Error(`IMAPBox::${name} - Can't operate on a mailbox that is no longer open on the current IMAPConnection.`)
-          )
-        }
-        return prop
+
+        return val;
       },
     })
   }
diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index 0d23f068f..20f52808f 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -184,6 +184,10 @@ class IMAPConnection extends EventEmitter {
     return this._imap.delBoxAsync(folderName)
   }
 
+  getOpenBoxName() {
+    return (this._imap && this._imap._box) ? this._imap._box.name : null;
+  }
+
   runOperation(operation) {
     if (!this._imap) {
       throw new IMAPConnectionNotReadyError(`IMAPConnection::runOperation`)

From b03b8b537d7fd9b2b003451dc9a6491052727a53 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 14:34:03 -0700
Subject: [PATCH 267/800] Optimize dashboard rendering

Add version to local copy of account and only re-render entire account
if the version is different. Create an ElapsedTime component that
re-renders on its own, and update SyncGraph to re-render on its
own as well.
---
 packages/nylas-dashboard/public/index.html    |  1 +
 packages/nylas-dashboard/public/js/app.jsx    | 22 +++++++++++--
 .../public/js/elapsed-time.jsx                | 31 +++++++++++++++++++
 .../nylas-dashboard/public/js/sync-graph.jsx  | 11 ++++++-
 4 files changed, 62 insertions(+), 3 deletions(-)
 create mode 100644 packages/nylas-dashboard/public/js/elapsed-time.jsx

diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html
index 0bc66e9b1..66b51bb93 100644
--- a/packages/nylas-dashboard/public/index.html
+++ b/packages/nylas-dashboard/public/index.html
@@ -3,6 +3,7 @@
   
   
   
+  
   
   
   
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index e84f6d2f9..292075214 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -9,6 +9,7 @@ const {
   AccountFilter,
   SyncGraph,
   SyncbackRequestDetails,
+  ElapsedTime,
 } = window;
 
 function calcAcctPosition(count) {
@@ -26,13 +27,23 @@ function calcAcctPosition(count) {
   return {left: left, top: top};
 }
 
+function formatSyncTimes(timestamp) {
+  return timestamp / 1000;
+}
+
 class Account extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
       accountId: props.account.id,
+      version: null,
     }
   }
+
+  shouldComponentUpdate(nextProps) {
+    return nextProps.account.version !== this.props.account.version;
+  }
+
   clearError() {
     const req = new XMLHttpRequest();
     const url = `${window.location.protocol}/accounts/${this.state.accountId}/clear-sync-error`;
@@ -48,6 +59,7 @@ class Account extends React.Component {
     }
     req.send();
   }
+
   renderError() {
     const {account} = this.props;
 
@@ -80,7 +92,6 @@ class Account extends React.Component {
     const oldestSync = account.last_sync_completions[numStoredSyncs - 1];
     const newestSync = account.last_sync_completions[0];
     const avgBetweenSyncs = (newestSync - oldestSync) / (1000 * numStoredSyncs);
-    const timeSinceLastSync = (Date.now() - newestSync) / 1000;
 
     let firstSyncDuration = "Incomplete";
     if (account.first_sync_completion) {
@@ -108,7 +119,9 @@ class Account extends React.Component {
            Average Time Between Syncs (seconds):
           
{avgBetweenSyncs}

           Time Since Last Sync (seconds):
-          
{timeSinceLastSync}

+          
+            
+          

           Recent Syncs:
           
         

@@ -166,6 +179,11 @@ class Root extends React.Component {
   onReceivedUpdate(update) {
     const accounts = Object.assign({}, this.state.accounts);
     for (const account of update.updatedAccounts) {
+      if (accounts[account.id]) {
+        account.version = accounts[account.id].version + 1;
+      } else {
+        account.version = 0;
+      }
       accounts[account.id] = account;
     }
 
diff --git a/packages/nylas-dashboard/public/js/elapsed-time.jsx b/packages/nylas-dashboard/public/js/elapsed-time.jsx
new file mode 100644
index 000000000..61f351e86
--- /dev/null
+++ b/packages/nylas-dashboard/public/js/elapsed-time.jsx
@@ -0,0 +1,31 @@
+const React = window.React;
+
+class ElapsedTime extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      elapsed: 0,
+    }
+  }
+
+  componentDidMount() {
+    this.interval = setInterval(() => {
+      this.setState({elapsed: Date.now() - this.props.refTimestamp})
+    }, 1000);
+  }
+
+  componentWillUnmount() {
+    clearInterval(this.interval);
+  }
+
+  render() {
+    return {this.props.formatTime(this.state.elapsed)} 
+  }
+}
+
+ElapsedTime.propTypes = {
+  refTimestamp: React.PropTypes.number, // milliseconds
+  formatTime: React.PropTypes.func,
+}
+
+window.ElapsedTime = ElapsedTime;
diff --git a/packages/nylas-dashboard/public/js/sync-graph.jsx b/packages/nylas-dashboard/public/js/sync-graph.jsx
index abde9a400..123ceb5cf 100644
--- a/packages/nylas-dashboard/public/js/sync-graph.jsx
+++ b/packages/nylas-dashboard/public/js/sync-graph.jsx
@@ -2,15 +2,24 @@ const React = window.React;
 const ReactDOM = window.ReactDOM;
 
 class SyncGraph extends React.Component {
-
   componentDidMount() {
     this.drawGraph(true);
+
+    this.interval = setInterval(() => {
+      if (Date.now() - this.props.syncTimestamps[0] > 10000) {
+        this.drawGraph(false);
+      }
+    }, 10000);
   }
 
   componentDidUpdate() {
     this.drawGraph(false);
   }
 
+  componentWillUnmount() {
+    clearInterval(this.interval);
+  }
+
   drawGraph(isInitial) {
     const now = Date.now();
     const config = SyncGraph.config;

From 0b9933b51469611bba0d2773748557efb2be0cf6 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 14:53:56 -0700
Subject: [PATCH 268/800] Re-render accounts in more scenarios and update
 onWebsocketConnectedFake

---
 packages/nylas-dashboard/public/js/app.jsx   |  5 ++++-
 packages/nylas-dashboard/routes/websocket.js | 12 ++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 292075214..2ffa54034 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -41,7 +41,10 @@ class Account extends React.Component {
   }
 
   shouldComponentUpdate(nextProps) {
-    return nextProps.account.version !== this.props.account.version;
+    return nextProps.account.version !== this.props.account.version ||
+      nextProps.active !== this.props.account.active ||
+      nextProps.assignment !== this.props.assignment ||
+      nextProps.count !== this.props.count;
   }
 
   clearError() {
diff --git a/packages/nylas-dashboard/routes/websocket.js b/packages/nylas-dashboard/routes/websocket.js
index 41b38af51..ffadf3dce 100644
--- a/packages/nylas-dashboard/routes/websocket.js
+++ b/packages/nylas-dashboard/routes/websocket.js
@@ -90,12 +90,20 @@ function onWebsocketConnectedFake(wss, ws) {
       last_sync_completions: [],
       created_at: "2016-07-13T00:49:25.000Z",
     };
-    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+    ws.send(JSON.stringify({ cmd: "UPDATE", payload: {
+      updatedAccounts: [acct],
+      activeAccountIds: [],
+      assignments: {},
+    }}));
     accts.push(acct);
   }
   setInterval(() => {
     const acct = accts[Math.floor(Math.random() * accts.length)];
-    ws.send(JSON.stringify({ cmd: "ACCOUNT", payload: acct }));
+    ws.send(JSON.stringify({ cmd: "UPDATE", payload: {
+      updatedAccounts: [acct],
+      activeAccountIds: [],
+      assignments: {},
+    }}));
   }, 250);
 }
 

From 8c33c4976f1fc4e24532912b93dbe56d6f488da4 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 13 Jul 2016 15:57:44 -0700
Subject: [PATCH 269/800] Better IMAP Error handling

---
 packages/nylas-api/routes/auth.js      | 10 +--
 packages/nylas-api/serialization.js    |  6 ++
 packages/nylas-core/imap-connection.js |  3 +-
 packages/nylas-core/imap-errors.js     | 87 ++++++++++++++++++++++++--
 packages/nylas-core/index.js           |  1 +
 packages/nylas-sync/sync-worker.js     |  3 +-
 6 files changed, 100 insertions(+), 10 deletions(-)

diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js
index a1734f8f0..8df72de54 100644
--- a/packages/nylas-api/routes/auth.js
+++ b/packages/nylas-api/routes/auth.js
@@ -9,6 +9,7 @@ const {
   DatabaseConnector,
   SyncPolicy,
   Provider,
+  Errors,
 } = require('nylas-core');
 
 const {GMAIL_CLIENT_ID, GMAIL_CLIENT_SECRET, GMAIL_REDIRECT_URL} = process.env;
@@ -162,7 +163,8 @@ module.exports = (server) => {
         reply(Serialization.jsonStringify(response));
       })
       .catch((err) => {
-        reply({error: err.message}).code(400);
+        const code = err instanceof Errors.IMAPAuthenticationError ? 401 : 400
+        reply({message: err.message, type: "api_error"}).code(code);
       })
     },
   });
@@ -204,13 +206,13 @@ module.exports = (server) => {
       const oauthClient = new OAuth2(GMAIL_CLIENT_ID, GMAIL_CLIENT_SECRET, GMAIL_REDIRECT_URL);
       oauthClient.getToken(request.query.code, (err, tokens) => {
         if (err) {
-          reply({error: err.message}).code(400);
+          reply({message: err.message, type: "api_error"}).code(400);
           return;
         }
         oauthClient.setCredentials(tokens);
         google.oauth2({version: 'v2', auth: oauthClient}).userinfo.get((error, profile) => {
           if (error) {
-            reply({error: error.message}).code(400);
+            reply({message: error.message, type: "api_error"}).code(400);
             return;
           }
 
@@ -247,7 +249,7 @@ module.exports = (server) => {
             reply(Serialization.jsonStringify(response));
           })
           .catch((connectionErr) => {
-            reply({error: connectionErr.message}).code(400);
+            reply({message: connectionErr.message, type: "api_error"}).code(400);
           });
         });
       });
diff --git a/packages/nylas-api/serialization.js b/packages/nylas-api/serialization.js
index f01d9d757..07527b1f6 100644
--- a/packages/nylas-api/serialization.js
+++ b/packages/nylas-api/serialization.js
@@ -11,6 +11,12 @@ function jsonSchema(modelName) {
   if (models.includes(modelName)) {
     return Joi.object();
   }
+  if (modelName === 'Error') {
+    return Joi.object().keys({
+      message: Joi.string(),
+      type: Joi.string(),
+    })
+  }
   if (modelName === 'Account') {
     // Ben: Disabled temporarily because folks keep changing the keys and it's hard
     // to keep in sync. Might need to consider another approach to these.
diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index 20f52808f..628fd27da 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -5,6 +5,7 @@ const EventEmitter = require('events');
 
 const IMAPBox = require('./imap-box');
 const {
+  convertImapError,
   IMAPConnectionNotReadyError,
   IMAPConnectionEndedError,
 } = require('./imap-errors');
@@ -111,7 +112,7 @@ class IMAPConnection extends EventEmitter {
 
       this._imap.once('error', (err) => {
         this.end();
-        reject(err);
+        reject(convertImapError(err));
       });
 
       this._imap.once('end', () => {
diff --git a/packages/nylas-core/imap-errors.js b/packages/nylas-core/imap-errors.js
index 3161deff5..cd9fe9258 100644
--- a/packages/nylas-core/imap-errors.js
+++ b/packages/nylas-core/imap-errors.js
@@ -1,21 +1,100 @@
+/**
+ * Errors may come from:
+ *
+ * 1. Underlying IMAP provider (Fastmail, Yahoo, etc)
+ * 2. Node IMAP
+ * 3. K2 code
+ *
+ * NodeIMAP puts a `source` attribute on `Error` objects to indicate where
+ * a particular error came from. See https://github.com/mscdex/node-imap/blob/master/lib/Connection.js
+ *
+ * These may have the following values:
+ *
+ *   - "socket-timeout": Created by NodeIMAP when `config.socketTimeout`
+ *     expires on the base Node `net.Socket` and socket.on('timeout') fires
+ *     Message: 'Socket timed out while talking to server'
+ *
+ *   - "timeout": Created by NodeIMAP when `config.connTimeout` has been
+ *     reached when trying to connect the socket.
+ *     Message: 'Timed out while connecting to server'
+ *
+ *   - "socket": Created by Node's `net.Socket` on error. See:
+ *     https://nodejs.org/api/net.html#net_event_error_1
+ *     Message: Various from `net.Socket`
+ *
+ *   - "protocol": Created by NodeIMAP when `bad` or `no` types come back
+ *     from the IMAP protocol.
+ *     Message: Various from underlying IMAP protocol
+ *
+ *   - "authentication": Created by underlying IMAP connection or NodeIMAP
+ *     in a few scenarios.
+ *     Message: Various from underlying IMAP connection
+ *              OR: No supported authentication method(s) available. Unable to login.
+ *              OR: Logging in is disabled on this server
+ *
+ *   - "timeout-auth": Created by NodeIMAP when `config.authTimeout` has
+ *     been reached when trying to authenticate
+ *     Message: 'Timed out while authenticating with server'
+ *
+ */
+function convertImapError(imapError) {
+  let error;
+  switch(imapError.source) {
+    case "socket-timeout":
+      error = new IMAPConnectionTimeoutError(imapError); break;
+    case "timeout":
+      error = new IMAPConnectionTimeoutError(imapError); break;
+    case "socket":
+      error = new IMAPSocketError(imapError); break;
+    case "protocol":
+      error = new IMAPProtocolError(imapError); break;
+    case "authentication":
+      error = new IMAPAuthenticationError(imapError); break;
+    case "timeout-auth":
+      error = new IMAPAuthenticationTimeoutError(imapError); break;
+    default:
+      return error
+  }
+  error.source = imapError.source
+  return error
+}
 
-// "Source" is a hack so that the error matches the ones used by node-imap
+/**
+ * An abstract base class that can be used to indicate errors that may fix
+ * themselves when retried
+ */
+class RetryableError extends Error {  }
 
-class IMAPConnectionNotReadyError extends Error {
+/**
+ * Errors that originate from NodeIMAP. See `convertImapError` for
+ * documentation on underlying causes
+ */
+class IMAPSocketError extends RetryableError { }
+class IMAPConnectionTimeoutError extends RetryableError { }
+class IMAPAuthenticationTimeoutError extends RetryableError { }
+class IMAPProtocolError extends Error { }
+class IMAPAuthenticationError extends Error { }
+
+class IMAPConnectionNotReadyError extends RetryableError {
   constructor(funcName) {
     super(`${funcName} - You must call connect() first.`);
-    this.source = 'socket';
   }
 }
 
 class IMAPConnectionEndedError extends Error {
   constructor(msg = "The IMAP Connection was ended.") {
     super(msg);
-    this.source = 'socket';
   }
 }
 
 module.exports = {
+  convertImapError,
+  RetryableError,
+  IMAPSocketError,
+  IMAPConnectionTimeoutError,
+  IMAPAuthenticationTimeoutError,
+  IMAPProtocolError,
+  IMAPAuthenticationError,
   IMAPConnectionNotReadyError,
   IMAPConnectionEndedError,
 };
diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js
index 3282ce9a3..431eed10a 100644
--- a/packages/nylas-core/index.js
+++ b/packages/nylas-core/index.js
@@ -12,4 +12,5 @@ module.exports = {
   SchedulerUtils: require('./scheduler-utils'),
   MessageTypes: require('./message-types'),
   Logger: require('./logger'),
+  Errors: require('./imap-errors'),
 }
diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 1b6c7c801..210dce127 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -4,6 +4,7 @@ const {
   PubsubConnector,
   DatabaseConnector,
   MessageTypes,
+  Errors,
 } = require('nylas-core');
 const {
   jsonError,
@@ -200,7 +201,7 @@ class SyncWorker {
     this.closeConnection()
 
     // Continue to retry if it was a network error
-    if (error.source && (error.source.includes('socket') || error.source.includes('timeout'))) {
+    if (error instanceof Errors.RetryableError) {
       return Promise.resolve()
     }
 

From 93942e792dcf8d54c1a4b544ddfbb0d39e98bb74 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 16:20:43 -0700
Subject: [PATCH 270/800] Also conert imap errors in createConnectionPromise

---
 packages/nylas-core/imap-connection.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index 628fd27da..2b002e048 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -223,7 +223,7 @@ class IMAPConnection extends EventEmitter {
       };
       onErrored = (error) => {
         returned = true;
-        reject(error);
+        reject(convertImapError(error));
       };
 
       this._imap.once('error', onErrored);

From 971b64b4c299758cc942fd55c27c89b073f15900 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 16:08:52 -0700
Subject: [PATCH 271/800] Add other dashboard optimizations

Fix typo in Account's shouldComponentUpdate method
Consolidate intervals into custom window events
Manually update ElapsedTime contents
---
 packages/nylas-dashboard/public/js/app.jsx    |  2 +-
 .../public/js/elapsed-time.jsx                | 19 ++++++++++++++-----
 .../nylas-dashboard/public/js/sync-graph.jsx  | 12 +++++++++---
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 2ffa54034..18bc942f4 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -42,7 +42,7 @@ class Account extends React.Component {
 
   shouldComponentUpdate(nextProps) {
     return nextProps.account.version !== this.props.account.version ||
-      nextProps.active !== this.props.account.active ||
+      nextProps.active !== this.props.active ||
       nextProps.assignment !== this.props.assignment ||
       nextProps.count !== this.props.count;
   }
diff --git a/packages/nylas-dashboard/public/js/elapsed-time.jsx b/packages/nylas-dashboard/public/js/elapsed-time.jsx
index 61f351e86..b94f8ad04 100644
--- a/packages/nylas-dashboard/public/js/elapsed-time.jsx
+++ b/packages/nylas-dashboard/public/js/elapsed-time.jsx
@@ -1,4 +1,10 @@
 const React = window.React;
+const ReactDOM = window.ReactDOM;
+
+setInterval(() => {
+  const event = new Event('tick');
+  window.dispatchEvent(event);
+}, 1000);
 
 class ElapsedTime extends React.Component {
   constructor(props) {
@@ -9,17 +15,20 @@ class ElapsedTime extends React.Component {
   }
 
   componentDidMount() {
-    this.interval = setInterval(() => {
-      this.setState({elapsed: Date.now() - this.props.refTimestamp})
-    }, 1000);
+    this.onTick = () => {
+      ReactDOM.findDOMNode(this.refs.timestamp).innerHTML = this.props.formatTime(
+        Date.now() - this.props.refTimestamp
+      );
+    };
+    window.addEventListener('tick', this.onTick);
   }
 
   componentWillUnmount() {
-    clearInterval(this.interval);
+    window.removeEventListener('tick', this.onTick);
   }
 
   render() {
-    return {this.props.formatTime(this.state.elapsed)} 
+    return 
   }
 }
 
diff --git a/packages/nylas-dashboard/public/js/sync-graph.jsx b/packages/nylas-dashboard/public/js/sync-graph.jsx
index 123ceb5cf..c3eabcf4b 100644
--- a/packages/nylas-dashboard/public/js/sync-graph.jsx
+++ b/packages/nylas-dashboard/public/js/sync-graph.jsx
@@ -1,15 +1,21 @@
 const React = window.React;
 const ReactDOM = window.ReactDOM;
 
+setInterval(() => {
+  const event = new Event('graphtick')
+  window.dispatchEvent(event);
+}, 10000);
+
 class SyncGraph extends React.Component {
   componentDidMount() {
     this.drawGraph(true);
 
-    this.interval = setInterval(() => {
+    this.onGraphTick = () => {
       if (Date.now() - this.props.syncTimestamps[0] > 10000) {
         this.drawGraph(false);
       }
-    }, 10000);
+    }
+    window.addEventListener('graphtick', this.onGraphTick);
   }
 
   componentDidUpdate() {
@@ -17,7 +23,7 @@ class SyncGraph extends React.Component {
   }
 
   componentWillUnmount() {
-    clearInterval(this.interval);
+    window.removeEventListener('graphtick', this.onGraphTick);
   }
 
   drawGraph(isInitial) {

From fcbaac302b49dca3aea813f608133fd901a0fb97 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 16:31:08 -0700
Subject: [PATCH 272/800] Change dashboard error appearances

---
 packages/nylas-dashboard/public/css/app.css | 14 ++++--
 packages/nylas-dashboard/public/js/app.jsx  | 51 +++++++++++----------
 2 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index 27c272722..065c8232a 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -43,13 +43,16 @@ pre {
 .account.errored {
   color: #a94442;
   border-radius: 4px;
-  background-color: rgba(231, 195, 195, 0.6);
+  background-color: rgb(231, 195, 195);
 }
 
-.account .error pre {
-  text-overflow: ellipsis;
-  width: inherit;
-  overflow: hidden;
+.error-link {
+  font-weight: bold;
+}
+
+.error-link:hover {
+  cursor: pointer;
+  color: #702726;
 }
 
 #open-all-sync {
@@ -94,6 +97,7 @@ pre {
   left: 0;
   top: 0;
   background-color: rgba(0, 0, 0, 0.3);
+  z-index: 10;
 }
 
 .modal-close-wrapper {
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 18bc942f4..ee2c0aa50 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -10,6 +10,7 @@ const {
   SyncGraph,
   SyncbackRequestDetails,
   ElapsedTime,
+  Modal,
 } = window;
 
 function calcAcctPosition(count) {
@@ -63,28 +64,32 @@ class Account extends React.Component {
     req.send();
   }
 
-  renderError() {
-    const {account} = this.props;
-
+  renderPolicyOrError() {
+    const account = this.props.account;
     if (account.sync_error != null) {
-      const {message, stack} = account.sync_error
-      const error = {
-        message,
-        stack: stack.slice(0, 4),
-      }
-      return (
-        

-          
Error

-          
 this.clearError()}>Clear Error

-          

-            
-              {JSON.stringify(error, null, 2)}
-            

-          

-        

-      )
+      return this.renderError();
     }
-    return 
+    return (
+      
+    );
+  }
+
+  renderError() {
+    const {message, stack} = this.props.account.sync_error
+    return (
+      

+        
Error

+        
+          
{JSON.stringify(stack, null, 2)}

+        
+        
 this.clearError()}>Clear Error

+      

+    )
   }
 
   render() {
@@ -111,10 +116,6 @@ class Account extends React.Component {
         
{account.email_address} [{account.id}] {active ? '🌕' : '🌑'}

         {assignment}
         
-        
         
Sync Cycles

         

           First Sync Duration (seconds):
@@ -128,7 +129,7 @@ class Account extends React.Component {
           Recent Syncs:
           
         

-        {this.renderError()}
+        {this.renderPolicyOrError()}
       

     );
   }

From 87905d0145c7f0be8379ffb960c513dbb8fca982 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 13 Jul 2016 16:31:11 -0700
Subject: [PATCH 273/800] Fix transaction log to ignore syncState changes

---
 packages/nylas-core/hook-transaction-log.js | 32 ++++++++-------------
 1 file changed, 12 insertions(+), 20 deletions(-)

diff --git a/packages/nylas-core/hook-transaction-log.js b/packages/nylas-core/hook-transaction-log.js
index 0e1247d9c..9ba1c7f2c 100644
--- a/packages/nylas-core/hook-transaction-log.js
+++ b/packages/nylas-core/hook-transaction-log.js
@@ -1,3 +1,4 @@
+const _ = require('underscore')
 const PubsubConnector = require('./pubsub-connector')
 
 module.exports = (db, sequelize) => {
@@ -5,37 +6,28 @@ module.exports = (db, sequelize) => {
     return {
       object: $modelOptions.name.singular,
       objectId: dataValues.id,
-      changedFields: _changed,
+      changedFields: Object.keys(_changed),
     }
   }
 
-  const isSilent = (data) => {
-    data._previousDataValues
-    data._changed
+  const isTransaction = (data) => {
+    return data.$modelOptions.name.singular === "transaction"
+  }
 
-    if (data.$modelOptions.name.singular === "transaction") {
-      return true
-    }
+  const allIgnoredFields = (data) => {
+    const IGNORED_FIELDS = ["syncState", "version"];
+    return _.difference(Object.keys(data._changed), IGNORED_FIELDS).length === 0
+  }
 
-    if (data._changed && data._changed.syncState) {
-      for (const key of Object.keys(data._changed)) {
-        if (key === "syncState") { continue }
-        if (data._changed[key] !== data._previousDataValues[key]) {
-          // SyncState changed, but so did something else
-          return false;
-        }
-      }
-      // Be silent due to nothing but sync state changing
-      return true;
-    }
+  const isIgnored = (data) => {
+    return (isTransaction(data) || allIgnoredFields(data))
   }
 
   const transactionLogger = (type) => {
     return (sequelizeHookData) => {
-      if (isSilent(sequelizeHookData)) return;
+      if (isIgnored(sequelizeHookData)) return;
 
       const event = (type === "update" ? "modify" : type)
-
       const transactionData = Object.assign({event},
         parseHookData(sequelizeHookData)
       );

From ab9c01a249c40ceeedb19f053003b186251c4941 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 13 Jul 2016 16:23:25 -0700
Subject: [PATCH 274/800] Remove bluebird

- Implement `each` and `promisifyAll` + other bluebird fns
---
 package.json                                  |  5 +-
 packages/nylas-api/app.js                     |  1 -
 packages/nylas-core/imap-connection.js        |  3 +-
 packages/nylas-core/index.js                  |  3 +-
 packages/nylas-core/promise-utils.js          | 50 +++++++++++++++++++
 packages/nylas-core/pubsub-connector.js       |  5 +-
 packages/nylas-core/scheduler-utils.js        |  3 +-
 packages/nylas-dashboard/app.js               |  1 -
 packages/nylas-message-processor/app.js       |  1 -
 packages/nylas-sync/app.js                    |  1 -
 .../imap/fetch-messages-in-folder.js          | 19 +++----
 packages/nylas-sync/sync-process-manager.js   |  9 ++--
 12 files changed, 76 insertions(+), 25 deletions(-)
 create mode 100644 packages/nylas-core/promise-utils.js

diff --git a/package.json b/package.json
index e2dcb9481..ed2755689 100644
--- a/package.json
+++ b/package.json
@@ -9,16 +9,17 @@
     "bunyan-cloudwatch": "2.0.0",
     "bunyan-loggly": "^1.0.0",
     "bunyan-prettystream": "^0.1.3",
+    "imap": "0.8.x",
     "lerna": "2.0.0-beta.23",
     "mysql": "^2.11.1",
     "newrelic": "^1.28.1",
     "pm2": "^1.1.3",
+    "promise.prototype.finally": "^1.0.1",
     "redis": "2.x.x",
     "rx": "4.x.x",
     "sequelize": "3.x.x",
     "underscore": "1.x.x",
-    "utf7": "https://github.com/truebit/utf7/archive/1f753bac59b99d93b17a5ef11681e232465e2558.tar.gz",
-    "imap": "0.8.x"
+    "utf7": "https://github.com/truebit/utf7/archive/1f753bac59b99d93b17a5ef11681e232465e2558.tar.gz"
   },
   "devDependencies": {
     "babel-eslint": "6.x",
diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js
index 8199a9687..2dc1e9d28 100644
--- a/packages/nylas-api/app.js
+++ b/packages/nylas-api/app.js
@@ -11,7 +11,6 @@ const fs = require('fs');
 const path = require('path');
 const {DatabaseConnector, SchedulerUtils, Logger} = require(`nylas-core`);
 
-global.Promise = require('bluebird');
 global.Logger = Logger.createLogger('nylas-k2-api')
 
 const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error')
diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index 2b002e048..4132cf140 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -3,6 +3,7 @@ const _ = require('underscore');
 const xoauth2 = require('xoauth2');
 const EventEmitter = require('events');
 
+const PromiseUtils = require('./promise-utils')
 const IMAPBox = require('./imap-box');
 const {
   convertImapError,
@@ -87,7 +88,7 @@ class IMAPConnection extends EventEmitter {
 
   _buildUnderlyingConnection(settings) {
     return new Promise((resolve, reject) => {
-      this._imap = Promise.promisifyAll(new Imap(settings));
+      this._imap = PromiseUtils.promisifyAll(new Imap(settings));
 
       // Emitted when new mail arrives in the currently open mailbox.
       // Fix https://github.com/mscdex/node-imap/issues/445
diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js
index 431eed10a..b6348ddd4 100644
--- a/packages/nylas-core/index.js
+++ b/packages/nylas-core/index.js
@@ -1,5 +1,3 @@
-global.Promise = require('bluebird');
-
 module.exports = {
   Provider: {
     Gmail: 'gmail',
@@ -13,4 +11,5 @@ module.exports = {
   MessageTypes: require('./message-types'),
   Logger: require('./logger'),
   Errors: require('./imap-errors'),
+  PromiseUtils: require('./promise-utils'),
 }
diff --git a/packages/nylas-core/promise-utils.js b/packages/nylas-core/promise-utils.js
new file mode 100644
index 000000000..d0114b34d
--- /dev/null
+++ b/packages/nylas-core/promise-utils.js
@@ -0,0 +1,50 @@
+require('promise.prototype.finally')
+const _ = require('underscore')
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function each(iterable, iterator) {
+  return Promise.resolve(iterable)
+  .then((iter) => Array.from(iter))
+  .then((array) => {
+    return new Promise((resolve, reject) => {
+      array.reduce((prevPromise, item, idx, len) => (
+        prevPromise.then(() => Promise.resolve(iterator(item, idx, len)))
+      ), Promise.resolve())
+      .then(() => resolve(iterable))
+      .catch((err) => reject(err))
+    })
+  })
+}
+
+function promisify(nodeFn) {
+  return function wrapper(...fnArgs) {
+    return new Promise((resolve, reject) => {
+      nodeFn.call(this, ...fnArgs, (err, ...results) => {
+        if (err) {
+          reject(err)
+          return
+        }
+        resolve(...results)
+      });
+    })
+  }
+}
+
+function promisifyAll(obj) {
+  for(const key in obj) {
+    if (!key.endsWith('Async') && _.isFunction(obj[key])) {
+      obj[`${key}Async`] = promisify(obj[key])
+    }
+  }
+  return obj
+}
+
+module.exports = {
+  each,
+  sleep,
+  promisify,
+  promisifyAll,
+}
diff --git a/packages/nylas-core/pubsub-connector.js b/packages/nylas-core/pubsub-connector.js
index 36121688a..fc5ff8bba 100644
--- a/packages/nylas-core/pubsub-connector.js
+++ b/packages/nylas-core/pubsub-connector.js
@@ -1,9 +1,10 @@
 const Rx = require('rx')
 const redis = require("redis");
+const PromiseUtils = require('./promise-utils')
 const log = global.Logger || console
 
-Promise.promisifyAll(redis.RedisClient.prototype);
-Promise.promisifyAll(redis.Multi.prototype);
+PromiseUtils.promisifyAll(redis.RedisClient.prototype);
+PromiseUtils.promisifyAll(redis.Multi.prototype);
 
 
 class PubsubConnector {
diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js
index 9b719f0ba..41711ca71 100644
--- a/packages/nylas-core/scheduler-utils.js
+++ b/packages/nylas-core/scheduler-utils.js
@@ -8,12 +8,13 @@ const HEARTBEAT_EXPIRES = 30; // 2 min in prod?
 
 const CLAIM_DURATION = 10 * 60 * 1000; // 2 hours on prod?
 
+const PromiseUtils = require('./promise-utils')
 const PubsubConnector = require('./pubsub-connector');
 const MessageTypes = require('./message-types')
 
 const forEachAccountList = (forEachCallback) => {
   const client = PubsubConnector.broadcastClient();
-  return Promise.each(client.keysAsync(`accounts:*`), (key) => {
+  return PromiseUtils.each(client.keysAsync(`accounts:*`), (key) => {
     const processId = key.replace('accounts:', '');
     return client.lrangeAsync(key, 0, 20000).then((foundIds) =>
       forEachCallback(processId, foundIds)
diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js
index 0284429ae..95eddfb30 100644
--- a/packages/nylas-dashboard/app.js
+++ b/packages/nylas-dashboard/app.js
@@ -5,7 +5,6 @@ const Hapi = require('hapi');
 const HapiWebSocket = require('hapi-plugin-websocket');
 const {Logger} = require(`nylas-core`);
 
-global.Promise = require('bluebird');
 global.Logger = Logger.createLogger('nylas-k2-dashboard')
 
 const server = new Hapi.Server();
diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js
index 4ad7aee14..73e4950f5 100644
--- a/packages/nylas-message-processor/app.js
+++ b/packages/nylas-message-processor/app.js
@@ -1,7 +1,6 @@
 const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`)
 const {processors} = require('./processors')
 
-global.Promise = require('bluebird');
 global.Logger = Logger.createLogger('nylas-k2-message-processor')
 
 // List of the attributes of Message that the processor should be allowed to change.
diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js
index 16bf4509f..03b04d92d 100644
--- a/packages/nylas-sync/app.js
+++ b/packages/nylas-sync/app.js
@@ -1,5 +1,4 @@
 // require('newrelic');
-global.Promise = require('bluebird');
 const {DatabaseConnector, Logger} = require(`nylas-core`)
 const SyncProcessManager = require('./sync-process-manager');
 
diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js
index 038ae281b..2f3be4cbe 100644
--- a/packages/nylas-sync/imap/fetch-messages-in-folder.js
+++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js
@@ -1,7 +1,7 @@
 const _ = require('underscore');
 const Imap = require('imap');
 
-const {IMAPConnection, PubsubConnector} = require('nylas-core');
+const {IMAPConnection, PubsubConnector, PromiseUtils} = require('nylas-core');
 const {Capabilities} = IMAPConnection;
 
 const MessageFlagAttributes = ['id', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels']
@@ -113,12 +113,12 @@ class FetchMessagesInFolder {
     const {Message} = this._db;
 
     const removedUIDs = localMessageAttributes
-      .filter(msg => !remoteUIDAttributes[msg.folderImapUID])
-      .map(msg => msg.folderImapUID)
+    .filter(msg => !remoteUIDAttributes[msg.folderImapUID])
+    .map(msg => msg.folderImapUID)
 
-      this._logger.info({
-        removed_messages: removedUIDs.length,
-      }, `FetchMessagesInFolder: found messages no longer in the folder`)
+    this._logger.info({
+      removed_messages: removedUIDs.length,
+    }, `FetchMessagesInFolder: found messages no longer in the folder`)
 
     if (removedUIDs.length === 0) {
       return Promise.resolve();
@@ -176,7 +176,7 @@ class FetchMessagesInFolder {
       uidsByPart[key].push(attributes.uid);
     })
     .then(() => {
-      return Promise.each(Object.keys(uidsByPart), (key) => {
+      return PromiseUtils.each(Object.keys(uidsByPart), (key) => {
         const uids = uidsByPart[key];
         const desiredParts = JSON.parse(key);
         const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
@@ -329,7 +329,7 @@ class FetchMessagesInFolder {
       }
     }
 
-    return Promise.each(desiredRanges, ({min, max}) => {
+    return PromiseUtils.each(desiredRanges, ({min, max}) => {
       this._logger.info({
         range: `${min}:${max}`,
       }, `FetchMessagesInFolder: Fetching range`);
@@ -343,7 +343,8 @@ class FetchMessagesInFolder {
           timeFetchedUnseen: Date.now(),
         });
       })
-    }).then(() => {
+    })
+    .then(() => {
       this._logger.info(`FetchMessagesInFolder: Fetching messages finished`);
     });
   }
diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index d1ee98c9f..c7eb423cc 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -1,6 +1,6 @@
 const os = require('os');
 const SyncWorker = require('./sync-worker');
-const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`)
+const {DatabaseConnector, PubsubConnector, SchedulerUtils, PromiseUtils} = require(`nylas-core`)
 
 const IDENTITY = `${os.hostname()}-${process.pid}`;
 
@@ -110,12 +110,13 @@ class SyncProcessManager {
 
     this._logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.")
 
-    Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
+    PromiseUtils.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
       const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, '');
       return client.existsAsync(HEARTBEAT_FOR(id)).then((exists) =>
         (exists ? Promise.resolve() : this.unassignAccountsAssignedTo(id))
       )
-    }).finally(() => {
+    })
+    .finally(() => {
       const delay = HEARTBEAT_EXPIRES * 1000;
       setTimeout(() => this.unassignAccountsMissingHeartbeats(), delay);
     });
@@ -165,7 +166,7 @@ class SyncProcessManager {
 
       // If we've added an account, wait a second before asking for another one.
       // Spacing them out is probably healthy.
-      return Promise.delay(2000);
+      return PromiseUtils.sleep(2000);
     });
   }
 

From 35c32a3645a5e11e6b647b895a510e051dd0ac75 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 16:42:13 -0700
Subject: [PATCH 275/800] After authing an account, close IMAP connection

---
 packages/nylas-api/routes/auth.js      | 14 ++++++++++----
 packages/nylas-core/imap-connection.js |  8 ++++----
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/packages/nylas-api/routes/auth.js b/packages/nylas-api/routes/auth.js
index 8df72de54..25f1e1bd4 100644
--- a/packages/nylas-api/routes/auth.js
+++ b/packages/nylas-api/routes/auth.js
@@ -148,7 +148,10 @@ module.exports = (server) => {
         db: dbStub,
       }));
 
-      Promise.all(connectionChecks).then(() => {
+      Promise.all(connectionChecks).then((conns) => {
+        for (const conn of conns) {
+          if (conn) { conn.end(); }
+        }
         return buildAccountWith({
           name: name,
           email: email,
@@ -234,15 +237,18 @@ module.exports = (server) => {
               db: {},
             }),
           ])
-          .then(() =>
-            buildAccountWith({
+          .then((conns) => {
+            for (const conn of conns) {
+              if (conn) { conn.end(); }
+            }
+            return buildAccountWith({
               name: profile.name,
               email: profile.email,
               provider: Provider.Gmail,
               settings,
               credentials,
             })
-          )
+          })
           .then(({account, token}) => {
             const response = account.toJSON();
             response.token = token.value;
diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index 2b002e048..373f4d947 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -46,9 +46,9 @@ class IMAPConnection extends EventEmitter {
 
   connect() {
     if (!this._connectPromise) {
-      this._connectPromise = this._resolveIMAPSettings().then((settings) =>
-        this._buildUnderlyingConnection(settings)
-      );
+      this._connectPromise = this._resolveIMAPSettings()
+      .then((settings) => this._buildUnderlyingConnection(settings))
+      .thenReturn(this);
     }
     return this._connectPromise;
   }
@@ -107,7 +107,7 @@ class IMAPConnection extends EventEmitter {
       this._imap.on('update', () => this.emit('update'))
 
       this._imap.once('ready', () => {
-        resolve(this)
+        resolve()
       });
 
       this._imap.once('error', (err) => {

From b43b623ca66f9f3d84000aa954312c428f830b7d Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 17:22:02 -0700
Subject: [PATCH 276/800] Expand promise utils

---
 package.json                                  |  1 +
 packages/nylas-core/imap-connection.js        |  8 +++---
 packages/nylas-core/promise-utils.js          | 27 +++++++++++--------
 packages/nylas-core/scheduler-utils.js        |  3 +--
 .../imap/fetch-messages-in-folder.js          |  6 ++---
 packages/nylas-sync/sync-process-manager.js   |  6 ++---
 6 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/package.json b/package.json
index ed2755689..5404711fc 100644
--- a/package.json
+++ b/package.json
@@ -14,6 +14,7 @@
     "mysql": "^2.11.1",
     "newrelic": "^1.28.1",
     "pm2": "^1.1.3",
+    "promise-props": "^1.0.0",
     "promise.prototype.finally": "^1.0.1",
     "redis": "2.x.x",
     "rx": "4.x.x",
diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index d70dba97b..4132cf140 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -47,9 +47,9 @@ class IMAPConnection extends EventEmitter {
 
   connect() {
     if (!this._connectPromise) {
-      this._connectPromise = this._resolveIMAPSettings()
-      .then((settings) => this._buildUnderlyingConnection(settings))
-      .thenReturn(this);
+      this._connectPromise = this._resolveIMAPSettings().then((settings) =>
+        this._buildUnderlyingConnection(settings)
+      );
     }
     return this._connectPromise;
   }
@@ -108,7 +108,7 @@ class IMAPConnection extends EventEmitter {
       this._imap.on('update', () => this.emit('update'))
 
       this._imap.once('ready', () => {
-        resolve()
+        resolve(this)
       });
 
       this._imap.once('error', (err) => {
diff --git a/packages/nylas-core/promise-utils.js b/packages/nylas-core/promise-utils.js
index d0114b34d..6b0db666e 100644
--- a/packages/nylas-core/promise-utils.js
+++ b/packages/nylas-core/promise-utils.js
@@ -1,16 +1,15 @@
+/* eslint no-restricted-syntax: 0 */
+
 require('promise.prototype.finally')
+
 const _ = require('underscore')
 
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms))
-}
+global.Promise.props = require('promise-props');
 
-function each(iterable, iterator) {
-  return Promise.resolve(iterable)
-  .then((iter) => Array.from(iter))
-  .then((array) => {
+global.Promise.each = function each(iterable, iterator) {
+  return Promise.resolve(iterable).then((array) => {
     return new Promise((resolve, reject) => {
-      array.reduce((prevPromise, item, idx, len) => (
+      Array.from(array).reduce((prevPromise, item, idx, len) => (
         prevPromise.then(() => Promise.resolve(iterator(item, idx, len)))
       ), Promise.resolve())
       .then(() => resolve(iterable))
@@ -19,6 +18,14 @@ function each(iterable, iterator) {
   })
 }
 
+global.Promise.sleep = function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+global.Promise.prototype.thenReturn = function thenReturnPolyfill(value) {
+  this.then(function then() { return Promise.resolve(value); })
+}
+
 function promisify(nodeFn) {
   return function wrapper(...fnArgs) {
     return new Promise((resolve, reject) => {
@@ -34,7 +41,7 @@ function promisify(nodeFn) {
 }
 
 function promisifyAll(obj) {
-  for(const key in obj) {
+  for (const key in obj) {
     if (!key.endsWith('Async') && _.isFunction(obj[key])) {
       obj[`${key}Async`] = promisify(obj[key])
     }
@@ -43,8 +50,6 @@ function promisifyAll(obj) {
 }
 
 module.exports = {
-  each,
-  sleep,
   promisify,
   promisifyAll,
 }
diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js
index 41711ca71..9b719f0ba 100644
--- a/packages/nylas-core/scheduler-utils.js
+++ b/packages/nylas-core/scheduler-utils.js
@@ -8,13 +8,12 @@ const HEARTBEAT_EXPIRES = 30; // 2 min in prod?
 
 const CLAIM_DURATION = 10 * 60 * 1000; // 2 hours on prod?
 
-const PromiseUtils = require('./promise-utils')
 const PubsubConnector = require('./pubsub-connector');
 const MessageTypes = require('./message-types')
 
 const forEachAccountList = (forEachCallback) => {
   const client = PubsubConnector.broadcastClient();
-  return PromiseUtils.each(client.keysAsync(`accounts:*`), (key) => {
+  return Promise.each(client.keysAsync(`accounts:*`), (key) => {
     const processId = key.replace('accounts:', '');
     return client.lrangeAsync(key, 0, 20000).then((foundIds) =>
       forEachCallback(processId, foundIds)
diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js
index 2f3be4cbe..8e5531f1e 100644
--- a/packages/nylas-sync/imap/fetch-messages-in-folder.js
+++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js
@@ -1,7 +1,7 @@
 const _ = require('underscore');
 const Imap = require('imap');
 
-const {IMAPConnection, PubsubConnector, PromiseUtils} = require('nylas-core');
+const {IMAPConnection, PubsubConnector} = require('nylas-core');
 const {Capabilities} = IMAPConnection;
 
 const MessageFlagAttributes = ['id', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels']
@@ -176,7 +176,7 @@ class FetchMessagesInFolder {
       uidsByPart[key].push(attributes.uid);
     })
     .then(() => {
-      return PromiseUtils.each(Object.keys(uidsByPart), (key) => {
+      return Promise.each(Object.keys(uidsByPart), (key) => {
         const uids = uidsByPart[key];
         const desiredParts = JSON.parse(key);
         const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
@@ -329,7 +329,7 @@ class FetchMessagesInFolder {
       }
     }
 
-    return PromiseUtils.each(desiredRanges, ({min, max}) => {
+    return Promise.each(desiredRanges, ({min, max}) => {
       this._logger.info({
         range: `${min}:${max}`,
       }, `FetchMessagesInFolder: Fetching range`);
diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index c7eb423cc..732dfac20 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -1,6 +1,6 @@
 const os = require('os');
 const SyncWorker = require('./sync-worker');
-const {DatabaseConnector, PubsubConnector, SchedulerUtils, PromiseUtils} = require(`nylas-core`)
+const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`)
 
 const IDENTITY = `${os.hostname()}-${process.pid}`;
 
@@ -110,7 +110,7 @@ class SyncProcessManager {
 
     this._logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.")
 
-    PromiseUtils.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
+    Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
       const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, '');
       return client.existsAsync(HEARTBEAT_FOR(id)).then((exists) =>
         (exists ? Promise.resolve() : this.unassignAccountsAssignedTo(id))
@@ -166,7 +166,7 @@ class SyncProcessManager {
 
       // If we've added an account, wait a second before asking for another one.
       // Spacing them out is probably healthy.
-      return PromiseUtils.sleep(2000);
+      return Promise.sleep(2000);
     });
   }
 

From 2a05e69e4c43fb79fce68524a36bc691272913da Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 17:25:39 -0700
Subject: [PATCH 277/800] Sort IDs as numbers not strings

---
 packages/nylas-dashboard/public/js/app.jsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index ee2c0aa50..85a9484fd 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -223,7 +223,7 @@ class Root extends React.Component {
          this.onFilter.call(this)} />
          parseInt(id, 10))} />
         {
-          ids.sort((a, b) => a.localeCompare(b)).map((id) =>
+          ids.sort((a, b) => a / 1 - b / 1).map((id) =>
             
Date: Wed, 13 Jul 2016 17:34:42 -0700
Subject: [PATCH 278/800] Make dashboard boxes slightly smaller

---
 packages/nylas-dashboard/public/css/app.css       | 12 ++++++------
 packages/nylas-dashboard/public/js/app.jsx        | 11 +++++------
 packages/nylas-dashboard/public/js/sync-graph.jsx |  2 +-
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index 065c8232a..e2263bc90 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -5,7 +5,7 @@ body {
   background-size: 100vw auto;
   background-attachment: fixed;
   font-family: Roboto, sans-serif;
-  font-size: 12px;
+  font-size: 11px;
 }
 
 h2 {
@@ -20,22 +20,22 @@ pre {
 .account {
   position: absolute;
   border-radius: 5px;
-  width: 300px;
-  height: 500px;
+  width: 240px;
+  height: 450px;
   background-color: rgb(255, 255, 255);
   padding: 15px;
   margin: 5px;
-  overflow: auto;
+  overflow: hidden;
 }
 
 .account h3 {
-  font-size: 16px;
+  font-size: 13px;
   margin: 0;
   padding: 0;
 }
 
 .account .section {
-  font-size: 14px;
+  font-size: 12px;
   padding: 10px 0;
   text-align: center;
 }
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 85a9484fd..0d198a53b 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -14,8 +14,8 @@ const {
 } = window;
 
 function calcAcctPosition(count) {
-  const width = 340;
-  const height = 540;
+  const width = 280;
+  const height = 490;
   const marginTop = 100;
   const marginSide = 0;
 
@@ -116,13 +116,12 @@ class Account extends React.Component {
         
{account.email_address} [{account.id}] {active ? '🌕' : '🌑'}

         {assignment}
         
-        
Sync Cycles

         

-          First Sync Duration (seconds):
+          First Sync Duration (sec):
           
{firstSyncDuration}

-           Average Time Between Syncs (seconds):
+           Average Time Between Syncs (sec):
           
{avgBetweenSyncs}

-          Time Since Last Sync (seconds):
+          Time Since Last Sync (sec):
           
             
           

diff --git a/packages/nylas-dashboard/public/js/sync-graph.jsx b/packages/nylas-dashboard/public/js/sync-graph.jsx
index c3eabcf4b..72815bad6 100644
--- a/packages/nylas-dashboard/public/js/sync-graph.jsx
+++ b/packages/nylas-dashboard/public/js/sync-graph.jsx
@@ -95,7 +95,7 @@ class SyncGraph extends React.Component {
 
 SyncGraph.config = {
   height: 50, // Doesn't include labels
-  width: 300,
+  width: 240,
   // timeLength is 30 minutes in seconds. If you change this, be sure to update
   // syncGraphTimeLength in sync-worker.js and the axis labels in drawGraph()!
   timeLength: 60 * 30,

From 5c1704ac8efe1e02fc85d36f1850cff7b3c31d23 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 13 Jul 2016 17:40:50 -0700
Subject: [PATCH 279/800] Fix thenReturn + misc logger fixes

---
 packages/nylas-core/imap-connection.js                       | 3 ++-
 packages/nylas-core/promise-utils.js                         | 4 ++--
 packages/nylas-dashboard/public/js/set-all-sync-policies.jsx | 1 -
 packages/nylas-dashboard/public/js/sync-policy.jsx           | 1 -
 packages/nylas-sync/imap/fetch-folder-list.js                | 2 +-
 packages/nylas-sync/sync-worker.js                           | 2 +-
 6 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/nylas-core/imap-connection.js b/packages/nylas-core/imap-connection.js
index 4132cf140..6c44cf0a1 100644
--- a/packages/nylas-core/imap-connection.js
+++ b/packages/nylas-core/imap-connection.js
@@ -233,7 +233,8 @@ class IMAPConnection extends EventEmitter {
       const cresolve = (...args) => (!returned ? resolve(...args) : null)
       const creject = (...args) => (!returned ? reject(...args) : null)
       return callback(cresolve, creject)
-    }).finally(() => {
+    })
+    .finally(() => {
       if (this._imap) {
         this._imap.removeListener('error', onErrored);
         this._imap.removeListener('end', onEnded);
diff --git a/packages/nylas-core/promise-utils.js b/packages/nylas-core/promise-utils.js
index 6b0db666e..f6951ff65 100644
--- a/packages/nylas-core/promise-utils.js
+++ b/packages/nylas-core/promise-utils.js
@@ -22,8 +22,8 @@ global.Promise.sleep = function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
-global.Promise.prototype.thenReturn = function thenReturnPolyfill(value) {
-  this.then(function then() { return Promise.resolve(value); })
+global.Promise.prototype.thenReturn = function thenReturn(value) {
+  return this.then(function then() { return Promise.resolve(value); })
 }
 
 function promisify(nodeFn) {
diff --git a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
index 3e8e3451d..64be249e0 100644
--- a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
+++ b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
@@ -10,7 +10,6 @@ class SetAllSyncPolicies extends React.Component {
     req.setRequestHeader("Content-type", "application/json");
     req.onreadystatechange = () => {
       if (req.readyState === XMLHttpRequest.DONE) {
-        console.log(req.responseText);
         if (req.status === 200) {
           this.setState({editMode: false});
         }
diff --git a/packages/nylas-dashboard/public/js/sync-policy.jsx b/packages/nylas-dashboard/public/js/sync-policy.jsx
index a0dd05315..3c19ce5d5 100644
--- a/packages/nylas-dashboard/public/js/sync-policy.jsx
+++ b/packages/nylas-dashboard/public/js/sync-policy.jsx
@@ -18,7 +18,6 @@ class SyncPolicy extends React.Component {
     req.setRequestHeader("Content-type", "application/json");
     req.onreadystatechange = () => {
       if (req.readyState === XMLHttpRequest.DONE) {
-        console.log(req.responseText);
         if (req.status === 200) {
           this.setState({editMode: false});
         }
diff --git a/packages/nylas-sync/imap/fetch-folder-list.js b/packages/nylas-sync/imap/fetch-folder-list.js
index 36675fb16..6c37d58fd 100644
--- a/packages/nylas-sync/imap/fetch-folder-list.js
+++ b/packages/nylas-sync/imap/fetch-folder-list.js
@@ -3,7 +3,7 @@ const {Provider} = require('nylas-core');
 const GMAIL_ROLES_WITH_FOLDERS = ['all', 'trash', 'junk'];
 
 class FetchFolderList {
-  constructor(provider, logger = console) {
+  constructor(provider, logger) {
     this._provider = provider;
     this._logger = logger;
   }
diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 210dce127..1a992c5ad 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -185,7 +185,7 @@ class SyncWorker {
       return this._account.update({syncError: null})
       .then(() => this.ensureConnection())
       .then(() => this.syncbackMessageActions())
-      .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider)))
+      .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider, this._logger)))
       .then(() => this.syncAllCategories())
       .then(() => this.onSyncDidComplete())
       .catch((error) => this.onSyncError(error))

From 49f475cdd1fd6c0e8be1b243e125e04e08d32429 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 17:42:49 -0700
Subject: [PATCH 280/800] Add collapsed dashboard view

---
 packages/nylas-dashboard/public/css/app.css   | 21 +++++++++
 packages/nylas-dashboard/public/index.html    |  1 +
 packages/nylas-dashboard/public/js/app.jsx    | 20 +++++++-
 .../public/js/mini-account.jsx                | 46 +++++++++++++++++++
 4 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 packages/nylas-dashboard/public/js/mini-account.jsx

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index e2263bc90..f1d039511 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -198,3 +198,24 @@ pre {
   cursor: pointer;
   font-weight: normal;
 }
+
+.mini-account::after {
+  display: inline-block;
+  position: relative;
+  height: 10px;
+  width: 10px;
+  background-color: #666666;
+  content: "";
+  z-index: -1;
+}
+
+.mini-account {
+  height: 10px;
+  width: 10px;
+  background-color: rgb(0, 255, 157);
+  display: inline-block;
+}
+
+.mini-account.errored {
+  background-color: rgb(255, 38, 0);
+}
diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html
index 66b51bb93..3e0a66ba7 100644
--- a/packages/nylas-dashboard/public/index.html
+++ b/packages/nylas-dashboard/public/index.html
@@ -4,6 +4,7 @@
   
   
   
+  
   
   
   
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 0d198a53b..83ef037f5 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -11,6 +11,7 @@ const {
   SyncbackRequestDetails,
   ElapsedTime,
   Modal,
+  MiniAccount,
 } = window;
 
 function calcAcctPosition(count) {
@@ -215,6 +216,7 @@ class Root extends React.Component {
         break;
     }
 
+    const AccountType = this.props.collapsed ? MiniAccount : Account;
     let count = 0;
 
     return (
@@ -223,7 +225,7 @@ class Root extends React.Component {
          parseInt(id, 10))} />
         {
           ids.sort((a, b) => a / 1 - b / 1).map((id) =>
-            = 0) {
+  const value = window.location.search.substring(index + collapsedStr.length + 1);
+  if (value.startsWith("true")) {
+    collapsed = true;
+  }
+}
+
 ReactDOM.render(
-  ,
+  ,
   document.getElementById('root')
 );
diff --git a/packages/nylas-dashboard/public/js/mini-account.jsx b/packages/nylas-dashboard/public/js/mini-account.jsx
new file mode 100644
index 000000000..8d6d5a785
--- /dev/null
+++ b/packages/nylas-dashboard/public/js/mini-account.jsx
@@ -0,0 +1,46 @@
+const React = window.React;
+
+class MiniAccount extends React.Component {
+
+  calculateColor() {
+    // in milliseconds
+    const grayAfter = 10000;
+    const elapsedTime = Date.now() - this.props.account.last_sync_completions[0];
+    let opacity = 0;
+    if (elapsedTime < grayAfter) {
+      opacity = 1.0 - elapsedTime / grayAfter;
+    }
+
+    return `rgba(0, 255, 157, ${opacity})`;
+  }
+
+  render() {
+    const {account, assignment, active} = this.props;
+
+    let errorClass;
+    let style;
+    if (account.sync_error) {
+      errorClass = 'errored';
+      style = {};
+    } else {
+      errorClass = '';
+      style = {backgroundColor: this.calculateColor()};
+    }
+
+    return (
+      

+    )
+  }
+}
+
+MiniAccount.propTypes = {
+  account: React.PropTypes.object,
+  active: React.PropTypes.bool,
+  assignment: React.PropTypes.string,
+  count: React.PropTypes.number,
+};
+
+window.MiniAccount = MiniAccount;

From 070670137874422b0b917655c36ed435364cf34d Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 13 Jul 2016 18:00:17 -0700
Subject: [PATCH 281/800] Make the grayAfter time more appropriate for
 production, 10 mins

---
 packages/nylas-dashboard/public/js/mini-account.jsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/nylas-dashboard/public/js/mini-account.jsx b/packages/nylas-dashboard/public/js/mini-account.jsx
index 8d6d5a785..cd3539470 100644
--- a/packages/nylas-dashboard/public/js/mini-account.jsx
+++ b/packages/nylas-dashboard/public/js/mini-account.jsx
@@ -4,7 +4,7 @@ class MiniAccount extends React.Component {
 
   calculateColor() {
     // in milliseconds
-    const grayAfter = 10000;
+    const grayAfter = 1000 * 60 * 10; // 10 minutes
     const elapsedTime = Date.now() - this.props.account.last_sync_completions[0];
     let opacity = 0;
     if (elapsedTime < grayAfter) {

From a07174cc4e4678d0900085ad47e1e38dacfc4623 Mon Sep 17 00:00:00 2001
From: Annie 
Date: Wed, 13 Jul 2016 18:18:59 -0700
Subject: [PATCH 282/800] fixed json parsing of message

---
 packages/nylas-core/models/account/message.js | 24 ++++---------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js
index 4db6f4d24..024e36fdf 100644
--- a/packages/nylas-core/models/account/message.js
+++ b/packages/nylas-core/models/account/message.js
@@ -92,25 +92,11 @@ module.exports = (sequelize, Sequelize) => {
         if (this.folder_id && !this.folder) {
           throw new Error("Message.toJSON called on a message where folder were not eagerly loaded.")
         }
-
-        return {
-          id: this.id,
-          account_id: this.accountId,
-          object: 'message',
-          body: this.body,
-          subject: this.subject,
-          snippet: this.snippet,
-          to: this.to,
-          from: this.from,
-          cc: this.cc,
-          bcc: this.bcc,
-          reply_to: this.replyTo,
-          date: this.date.getTime() / 1000.0,
-          unread: this.unread,
-          starred: this.starred,
-          folder: this.folder,
-          thread_id: this.threadId,
-        };
+        const json = Object.assign({object: 'message'}, this.dataValues)
+        if (json.date) {
+          json.date = json.date.getTime() / 1000.0
+        }
+        return json
       },
     },
   });

From 66951b4b1a675a2a8c57c40afe7a43706f40fa83 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 13 Jul 2016 18:26:44 -0700
Subject: [PATCH 283/800] Add more logging to process claiming

---
 packages/nylas-sync/sync-process-manager.js | 36 +++++++++++++--------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index 732dfac20..935a499b2 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -171,22 +171,27 @@ class SyncProcessManager {
   }
 
   addWorkerForAccountId(accountId) {
-    DatabaseConnector.forShared().then(({Account}) => {
+    DatabaseConnector.forShared().then(({Account}) =>
       Account.find({where: {id: accountId}}).then((account) => {
-        if (!account || this._workers[account.id]) {
-          return;
+        if (!account) {
+          return Promise.reject(new Error("Could not find account"));
         }
-        DatabaseConnector.forAccount(account.id).then((db) => {
-          if (this._exiting || this._workers[account.id]) {
-            return;
+        return DatabaseConnector.forAccount(accountId).then((db) => {
+          if (this._exiting || this._workers[accountId]) {
+            return Promise.reject(new Error("Exiting or local worker already exists"));
           }
-          this._logger.info({account_id: accountId}, `ProcessManager: Starting worker for Account`)
-
           this._workers[account.id] = new SyncWorker(account, db, () => {
             this.removeWorkerForAccountId(accountId)
           });
+          return Promise.resolve();
         });
-      });
+      })
+    )
+    .then(() => {
+      this._logger.info({account_id: accountId}, `ProcessManager: Claiming Account Succeeded`)
+    })
+    .catch((err) => {
+      this._logger.error({account_id: accountId, reason: err.message}, `ProcessManager: Claiming Account Failed`)
     });
   }
 
@@ -196,13 +201,16 @@ class SyncProcessManager {
 
     return PubsubConnector.broadcastClient().lremAsync(src, 1, accountId).then((didRemove) => {
       this._workers[accountId] = null;
-
       if (didRemove) {
-        PubsubConnector.broadcastClient().rpushAsync(dst, accountId)
-      } else {
-        this._logger.error("Wanted to return item to pool, but didn't have claim on it.")
-        return
+        return PubsubConnector.broadcastClient().rpushAsync(dst, accountId)
       }
+      return Promise.reject(new Error("Did not own account."));
+    })
+    .then(() => {
+      this._logger.info({account_id: accountId}, `ProcessManager: Relinquishing Account Succeeded`)
+    })
+    .catch((err) => {
+      this._logger.error({account_id: accountId, reason: err.message}, `ProcessManager: Relinquishing Account Failed`)
     });
   }
 }

From 0c900b072d26911ede866a65dc3753179bb1a3bb Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 13 Jul 2016 19:07:24 -0700
Subject: [PATCH 284/800] Add new relic to all pkgs + Revert to PromiseUtils
 methods

- PromiseUtils does not conflict with newrelics changes to Promise
- Other misc fixes
---
 packages/nylas-api/app.js                     |  9 +++++--
 packages/nylas-api/newrelic.js                |  7 ++---
 packages/nylas-api/route-helpers.js           |  4 +--
 packages/nylas-core/index.js                  |  1 +
 packages/nylas-core/metrics.js                | 26 +++++++++++++++++++
 packages/nylas-core/models/account/file.js    |  3 ++-
 packages/nylas-core/models/account/message.js |  3 ++-
 packages/nylas-core/promise-utils.js          | 21 ++++++++-------
 packages/nylas-core/scheduler-utils.js        |  3 ++-
 packages/nylas-dashboard/app.js               | 11 ++++++++
 packages/nylas-dashboard/newrelic.js          | 21 +++++++++++++++
 packages/nylas-message-processor/app.js       | 11 ++++++++
 packages/nylas-message-processor/newrelic.js  | 22 ++++++++++++++++
 .../processors/threading.js                   |  3 ++-
 packages/nylas-sync/app.js                    | 12 ++++++---
 packages/nylas-sync/imap/fetch-folder-list.js |  7 +++--
 .../imap/fetch-messages-in-folder.js          | 13 ++++++----
 packages/nylas-sync/newrelic.js               |  7 ++---
 packages/nylas-sync/sync-process-manager.js   |  6 ++---
 19 files changed, 149 insertions(+), 41 deletions(-)
 create mode 100644 packages/nylas-core/metrics.js
 create mode 100644 packages/nylas-dashboard/newrelic.js
 create mode 100644 packages/nylas-message-processor/newrelic.js

diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js
index 2dc1e9d28..7513ac616 100644
--- a/packages/nylas-api/app.js
+++ b/packages/nylas-api/app.js
@@ -1,4 +1,5 @@
-// require('newrelic');
+const {Metrics} = require(`nylas-core`)
+Metrics.startCapturing()
 
 const Hapi = require('hapi');
 const HapiSwagger = require('hapi-swagger');
@@ -11,9 +12,13 @@ const fs = require('fs');
 const path = require('path');
 const {DatabaseConnector, SchedulerUtils, Logger} = require(`nylas-core`);
 
+global.Metrics = Metrics
 global.Logger = Logger.createLogger('nylas-k2-api')
 
-const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error')
+const onUnhandledError = (err) => {
+  global.Logger.fatal(err, 'Unhandled error')
+  global.Metrics.reportError(err)
+}
 process.on('uncaughtException', onUnhandledError)
 process.on('unhandledRejection', onUnhandledError)
 
diff --git a/packages/nylas-api/newrelic.js b/packages/nylas-api/newrelic.js
index 00882097e..2b0f06474 100644
--- a/packages/nylas-api/newrelic.js
+++ b/packages/nylas-api/newrelic.js
@@ -1,3 +1,4 @@
+const {NODE_ENV} = process.env
 /**
  * New Relic agent configuration.
  *
@@ -8,11 +9,7 @@ exports.config = {
   /**
    * Array of application names.
    */
-  app_name: ['Nylas K2 API'],
-  /**
-   * Your New Relic license key.
-   */
-  license_key: 'e232d6ccc786bd87aa72b86782439710162e3739',
+  app_name: [`k2-api-${NODE_ENV}`],
   logging: {
     /**
      * Level at which to log. 'trace' is most useful to New Relic when diagnosing
diff --git a/packages/nylas-api/route-helpers.js b/packages/nylas-api/route-helpers.js
index c22ed0d42..9ed1cac0d 100644
--- a/packages/nylas-api/route-helpers.js
+++ b/packages/nylas-api/route-helpers.js
@@ -1,5 +1,5 @@
 const Serialization = require('./serialization');
-const {PubsubConnector, MessageTypes} = require('nylas-core')
+const {PromiseUtils, PubsubConnector, MessageTypes} = require('nylas-core')
 
 module.exports = {
   createSyncbackRequest: function createSyncbackRequest(request, reply, syncRequestArgs) {
@@ -14,7 +14,7 @@ module.exports = {
     })
   },
   findFolderOrLabel: function findFolderOrLabel({Folder, Label}, str) {
-    return Promise.props({
+    return PromiseUtils.props({
       folder: Folder.find({
         where: { $or: [
           { id: str },
diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js
index b6348ddd4..9cff4c9f7 100644
--- a/packages/nylas-core/index.js
+++ b/packages/nylas-core/index.js
@@ -11,5 +11,6 @@ module.exports = {
   MessageTypes: require('./message-types'),
   Logger: require('./logger'),
   Errors: require('./imap-errors'),
+  Metrics: require('./metrics'),
   PromiseUtils: require('./promise-utils'),
 }
diff --git a/packages/nylas-core/metrics.js b/packages/nylas-core/metrics.js
new file mode 100644
index 000000000..0931cf879
--- /dev/null
+++ b/packages/nylas-core/metrics.js
@@ -0,0 +1,26 @@
+const {NODE_ENV} = process.env
+
+class Metrics {
+  constructor() {
+    this.newrelic = null
+    this.shouldReport = NODE_ENV && NODE_ENV !== 'development'
+  }
+
+  startCapturing() {
+    if (this.shouldReport) {
+      this.newrelic = require('newrelic')
+    }
+  }
+
+  reportError(error) {
+    if (this.newrelic && this.shouldReport) {
+      this.newrelic.noticeError(error)
+    }
+  }
+
+  reportMetric() {
+
+  }
+}
+
+module.exports = new Metrics()
diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js
index ba52ed4d2..6dff982db 100644
--- a/packages/nylas-core/models/account/file.js
+++ b/packages/nylas-core/models/account/file.js
@@ -1,3 +1,4 @@
+const PromiseUtils = require('../../promise-utils')
 const IMAPConnection = require('../../imap-connection')
 
 module.exports = (sequelize, Sequelize) => {
@@ -17,7 +18,7 @@ module.exports = (sequelize, Sequelize) => {
     instanceMethods: {
       fetch: function fetch({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
-        return Promise.props({
+        return PromiseUtils.props({
           message: this.getMessage(),
           connection: IMAPConnection.connect({db, settings, logger}),
         })
diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js
index 024e36fdf..d9553a8ec 100644
--- a/packages/nylas-core/models/account/message.js
+++ b/packages/nylas-core/models/account/message.js
@@ -1,4 +1,5 @@
 const crypto = require('crypto');
+const PromiseUtils = require('../../promise-utils')
 const IMAPConnection = require('../../imap-connection')
 const {JSONType, JSONARRAYType} = require('../../database-types');
 
@@ -71,7 +72,7 @@ module.exports = (sequelize, Sequelize) => {
 
       fetchRaw: function fetchRaw({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
-        return Promise.props({
+        return PromiseUtils.props({
           folder: this.getFolder(),
           connection: IMAPConnection.connect({db, settings, logger}),
         })
diff --git a/packages/nylas-core/promise-utils.js b/packages/nylas-core/promise-utils.js
index f6951ff65..6490c5019 100644
--- a/packages/nylas-core/promise-utils.js
+++ b/packages/nylas-core/promise-utils.js
@@ -4,9 +4,15 @@ require('promise.prototype.finally')
 
 const _ = require('underscore')
 
-global.Promise.props = require('promise-props');
+global.Promise.prototype.thenReturn = function thenReturn(value) {
+  return this.then(function then() { return Promise.resolve(value); })
+}
 
-global.Promise.each = function each(iterable, iterator) {
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function each(iterable, iterator) {
   return Promise.resolve(iterable).then((array) => {
     return new Promise((resolve, reject) => {
       Array.from(array).reduce((prevPromise, item, idx, len) => (
@@ -18,14 +24,6 @@ global.Promise.each = function each(iterable, iterator) {
   })
 }
 
-global.Promise.sleep = function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms))
-}
-
-global.Promise.prototype.thenReturn = function thenReturn(value) {
-  return this.then(function then() { return Promise.resolve(value); })
-}
-
 function promisify(nodeFn) {
   return function wrapper(...fnArgs) {
     return new Promise((resolve, reject) => {
@@ -50,6 +48,9 @@ function promisifyAll(obj) {
 }
 
 module.exports = {
+  each,
+  sleep,
   promisify,
   promisifyAll,
+  props: require('promise-props'),
 }
diff --git a/packages/nylas-core/scheduler-utils.js b/packages/nylas-core/scheduler-utils.js
index 9b719f0ba..41711ca71 100644
--- a/packages/nylas-core/scheduler-utils.js
+++ b/packages/nylas-core/scheduler-utils.js
@@ -8,12 +8,13 @@ const HEARTBEAT_EXPIRES = 30; // 2 min in prod?
 
 const CLAIM_DURATION = 10 * 60 * 1000; // 2 hours on prod?
 
+const PromiseUtils = require('./promise-utils')
 const PubsubConnector = require('./pubsub-connector');
 const MessageTypes = require('./message-types')
 
 const forEachAccountList = (forEachCallback) => {
   const client = PubsubConnector.broadcastClient();
-  return Promise.each(client.keysAsync(`accounts:*`), (key) => {
+  return PromiseUtils.each(client.keysAsync(`accounts:*`), (key) => {
     const processId = key.replace('accounts:', '');
     return client.lrangeAsync(key, 0, 20000).then((foundIds) =>
       forEachCallback(processId, foundIds)
diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js
index 95eddfb30..0968db821 100644
--- a/packages/nylas-dashboard/app.js
+++ b/packages/nylas-dashboard/app.js
@@ -1,3 +1,6 @@
+const {Metrics} = require(`nylas-core`)
+Metrics.startCapturing()
+
 const fs = require('fs');
 const path = require('path');
 const Inert = require('inert');
@@ -5,8 +8,16 @@ const Hapi = require('hapi');
 const HapiWebSocket = require('hapi-plugin-websocket');
 const {Logger} = require(`nylas-core`);
 
+global.Metrics = Metrics
 global.Logger = Logger.createLogger('nylas-k2-dashboard')
 
+const onUnhandledError = (err) => {
+  global.Logger.fatal(err, 'Unhandled error')
+  global.Metrics.reportError(err)
+}
+process.on('uncaughtException', onUnhandledError)
+process.on('unhandledRejection', onUnhandledError)
+
 const server = new Hapi.Server();
 server.connection({ port: process.env.PORT });
 
diff --git a/packages/nylas-dashboard/newrelic.js b/packages/nylas-dashboard/newrelic.js
new file mode 100644
index 000000000..97cd18e90
--- /dev/null
+++ b/packages/nylas-dashboard/newrelic.js
@@ -0,0 +1,21 @@
+const {NODE_ENV} = process.env
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+  /**
+   * Array of application names.
+   */
+  app_name: [`k2-dash-${NODE_ENV}`],
+  logging: {
+    /**
+     * Level at which to log. 'trace' is most useful to New Relic when diagnosing
+     * issues with the agent, 'info' and higher will impose the least overhead on
+     * production applications.
+     */
+    level: 'info',
+  },
+}
diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js
index 73e4950f5..dba5ee509 100644
--- a/packages/nylas-message-processor/app.js
+++ b/packages/nylas-message-processor/app.js
@@ -1,8 +1,19 @@
+const {Metrics} = require(`nylas-core`)
+Metrics.startCapturing()
+
 const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`)
 const {processors} = require('./processors')
 
+global.Metrics = Metrics
 global.Logger = Logger.createLogger('nylas-k2-message-processor')
 
+const onUnhandledError = (err) => {
+  global.Logger.fatal(err, 'Unhandled error')
+  global.Metrics.reportError(err)
+}
+process.on('uncaughtException', onUnhandledError)
+process.on('unhandledRejection', onUnhandledError)
+
 // List of the attributes of Message that the processor should be allowed to change.
 // The message may move between folders, get starred, etc. while it's being
 // processed, and it shouldn't overwrite changes to those fields.
diff --git a/packages/nylas-message-processor/newrelic.js b/packages/nylas-message-processor/newrelic.js
new file mode 100644
index 000000000..e74c412de
--- /dev/null
+++ b/packages/nylas-message-processor/newrelic.js
@@ -0,0 +1,22 @@
+const {NODE_ENV} = process.env
+/**
+ * New Relic agent configuration.
+ *
+ * See lib/config.defaults.js in the agent distribution for a more complete
+ * description of configuration variables and their potential values.
+ */
+exports.config = {
+  /**
+   * Array of application names.
+   */
+  app_name: [`k2-message-processor-${NODE_ENV}`],
+  logging: {
+    /**
+     * Level at which to log. 'trace' is most useful to New Relic when diagnosing
+     * issues with the agent, 'info' and higher will impose the least overhead on
+     * production applications.
+     */
+    level: 'info',
+  },
+}
+
diff --git a/packages/nylas-message-processor/processors/threading.js b/packages/nylas-message-processor/processors/threading.js
index 02d6b7ff9..a8b228aaf 100644
--- a/packages/nylas-message-processor/processors/threading.js
+++ b/packages/nylas-message-processor/processors/threading.js
@@ -1,3 +1,4 @@
+const {PromiseUtils} = require('nylas-core')
 // const _ = require('underscore');
 
 class ThreadingProcessor {
@@ -86,7 +87,7 @@ class ThreadingProcessor {
       findOrCreateThread = this.findOrCreateByMatching(db, message)
     }
 
-    return Promise.props({
+    return PromiseUtils.props({
       thread: findOrCreateThread,
       sentFolder: Folder.find({where: {role: 'sent'}}),
       sentLabel: Label.find({where: {role: 'sent'}}),
diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js
index 03b04d92d..39b9cdee9 100644
--- a/packages/nylas-sync/app.js
+++ b/packages/nylas-sync/app.js
@@ -1,10 +1,16 @@
-// require('newrelic');
-const {DatabaseConnector, Logger} = require(`nylas-core`)
+const {Metrics} = require(`nylas-core`)
+Metrics.startCapturing()
+
+const {DatabaseConnector, Logger} = require('nylas-core')
 const SyncProcessManager = require('./sync-process-manager');
 
+global.Metrics = Metrics
 global.Logger = Logger.createLogger('nylas-k2-sync')
 
-const onUnhandledError = (err) => global.Logger.fatal(err, 'Unhandled error')
+const onUnhandledError = (err) => {
+  global.Logger.fatal(err, 'Unhandled error')
+  global.Metrics.reportError(err)
+}
 process.on('uncaughtException', onUnhandledError)
 process.on('unhandledRejection', onUnhandledError)
 
diff --git a/packages/nylas-sync/imap/fetch-folder-list.js b/packages/nylas-sync/imap/fetch-folder-list.js
index 6c37d58fd..3ca310c6a 100644
--- a/packages/nylas-sync/imap/fetch-folder-list.js
+++ b/packages/nylas-sync/imap/fetch-folder-list.js
@@ -1,4 +1,4 @@
-const {Provider} = require('nylas-core');
+const {Provider, PromiseUtils} = require('nylas-core');
 
 const GMAIL_ROLES_WITH_FOLDERS = ['all', 'trash', 'junk'];
 
@@ -6,6 +6,9 @@ class FetchFolderList {
   constructor(provider, logger) {
     this._provider = provider;
     this._logger = logger;
+    if (!this._logger) {
+      throw new Error("FetchFolderList requires a logger")
+    }
   }
 
   description() {
@@ -96,7 +99,7 @@ class FetchFolderList {
       const {Folder, Label, sequelize} = this._db;
 
       return sequelize.transaction((transaction) => {
-        return Promise.props({
+        return PromiseUtils.props({
           folders: Folder.findAll({transaction}),
           labels: Label.findAll({transaction}),
         }).then(({folders, labels}) => {
diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js
index 8e5531f1e..89d1f8a9e 100644
--- a/packages/nylas-sync/imap/fetch-messages-in-folder.js
+++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js
@@ -1,7 +1,7 @@
 const _ = require('underscore');
 const Imap = require('imap');
 
-const {IMAPConnection, PubsubConnector} = require('nylas-core');
+const {PromiseUtils, IMAPConnection, PubsubConnector} = require('nylas-core');
 const {Capabilities} = IMAPConnection;
 
 const MessageFlagAttributes = ['id', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels']
@@ -11,13 +11,16 @@ const FETCH_MESSAGES_FIRST_COUNT = 100;
 const FETCH_MESSAGES_COUNT = 200;
 
 class FetchMessagesInFolder {
-  constructor(category, options, logger = console) {
+  constructor(category, options, logger) {
     this._imap = null
     this._box = null
     this._db = null
     this._category = category;
     this._options = options;
     this._logger = logger;
+    if (!this._logger) {
+      throw new Error("FetchMessagesInFolder requires a logger")
+    }
     if (!this._category) {
       throw new Error("FetchMessagesInFolder requires a category")
     }
@@ -176,7 +179,7 @@ class FetchMessagesInFolder {
       uidsByPart[key].push(attributes.uid);
     })
     .then(() => {
-      return Promise.each(Object.keys(uidsByPart), (key) => {
+      return PromiseUtils.each(Object.keys(uidsByPart), (key) => {
         const uids = uidsByPart[key];
         const desiredParts = JSON.parse(key);
         const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
@@ -329,7 +332,7 @@ class FetchMessagesInFolder {
       }
     }
 
-    return Promise.each(desiredRanges, ({min, max}) => {
+    return PromiseUtils.each(desiredRanges, ({min, max}) => {
       this._logger.info({
         range: `${min}:${max}`,
       }, `FetchMessagesInFolder: Fetching range`);
@@ -414,7 +417,7 @@ class FetchMessagesInFolder {
         attributes: MessageFlagAttributes,
       })
       .then((localMessageAttributes) => (
-        Promise.props({
+        PromiseUtils.props({
           updates: this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes),
           deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes),
         })
diff --git a/packages/nylas-sync/newrelic.js b/packages/nylas-sync/newrelic.js
index a85120e87..ff04075f6 100644
--- a/packages/nylas-sync/newrelic.js
+++ b/packages/nylas-sync/newrelic.js
@@ -1,3 +1,4 @@
+const {NODE_ENV} = process.env
 /**
  * New Relic agent configuration.
  *
@@ -8,11 +9,7 @@ exports.config = {
   /**
    * Array of application names.
    */
-  app_name: ['Nylas K2 Sync'],
-  /**
-   * Your New Relic license key.
-   */
-  license_key: 'e232d6ccc786bd87aa72b86782439710162e3739',
+  app_name: [`k2-sync-${NODE_ENV}`],
   logging: {
     /**
      * Level at which to log. 'trace' is most useful to New Relic when diagnosing
diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index 935a499b2..074beba90 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -1,6 +1,6 @@
 const os = require('os');
 const SyncWorker = require('./sync-worker');
-const {DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`)
+const {PromiseUtils, DatabaseConnector, PubsubConnector, SchedulerUtils} = require(`nylas-core`)
 
 const IDENTITY = `${os.hostname()}-${process.pid}`;
 
@@ -110,7 +110,7 @@ class SyncProcessManager {
 
     this._logger.info("ProcessManager: Starting unassignment for processes missing heartbeats.")
 
-    Promise.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
+    PromiseUtils.each(client.keysAsync(`${ACCOUNTS_CLAIMED_PREFIX}*`), (key) => {
       const id = key.replace(ACCOUNTS_CLAIMED_PREFIX, '');
       return client.existsAsync(HEARTBEAT_FOR(id)).then((exists) =>
         (exists ? Promise.resolve() : this.unassignAccountsAssignedTo(id))
@@ -166,7 +166,7 @@ class SyncProcessManager {
 
       // If we've added an account, wait a second before asking for another one.
       // Spacing them out is probably healthy.
-      return Promise.sleep(2000);
+      return PromiseUtils.sleep(2000);
     });
   }
 

From 16b91ea3941a9f7473f118002b8c802f43688e29 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 14 Jul 2016 10:39:05 -0700
Subject: [PATCH 285/800] Update metrics with signalfx - report # of syncing
 accts per host

---
 package.json                                |  1 +
 packages/nylas-api/app.js                   |  2 +-
 packages/nylas-core/metrics.js              | 66 +++++++++++++++------
 packages/nylas-dashboard/app.js             |  2 +-
 packages/nylas-message-processor/app.js     |  2 +-
 packages/nylas-sync/app.js                  |  2 +-
 packages/nylas-sync/sync-process-manager.js |  8 ++-
 7 files changed, 59 insertions(+), 24 deletions(-)

diff --git a/package.json b/package.json
index 5404711fc..b102ba807 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
     "redis": "2.x.x",
     "rx": "4.x.x",
     "sequelize": "3.x.x",
+    "signalfx": "^3.0.1",
     "underscore": "1.x.x",
     "utf7": "https://github.com/truebit/utf7/archive/1f753bac59b99d93b17a5ef11681e232465e2558.tar.gz"
   },
diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js
index 7513ac616..26180ae27 100644
--- a/packages/nylas-api/app.js
+++ b/packages/nylas-api/app.js
@@ -1,5 +1,5 @@
 const {Metrics} = require(`nylas-core`)
-Metrics.startCapturing()
+Metrics.startCapturing('nylas-k2-api')
 
 const Hapi = require('hapi');
 const HapiSwagger = require('hapi-swagger');
diff --git a/packages/nylas-core/metrics.js b/packages/nylas-core/metrics.js
index 0931cf879..1dbc70fa7 100644
--- a/packages/nylas-core/metrics.js
+++ b/packages/nylas-core/metrics.js
@@ -1,26 +1,54 @@
-const {NODE_ENV} = process.env
+const {env: {NODE_ENV, SIGNALFX_TOKEN}, pid} = process
+const os = require('os')
+const signalfx = require('signalfx')
 
-class Metrics {
-  constructor() {
-    this.newrelic = null
-    this.shouldReport = NODE_ENV && NODE_ENV !== 'development'
-  }
+let newrelicClient = null
+let signalfxClient = null
 
-  startCapturing() {
-    if (this.shouldReport) {
-      this.newrelic = require('newrelic')
-    }
-  }
+const MetricTypes = {
+  Gauge: 'gauges',
+  Counter: 'counters',
+  CumulativeCounter: 'cumulative_counters',
+}
+const shouldReport = NODE_ENV && NODE_ENV !== 'development'
+
+
+const Metrics = {
+
+  MetricTypes,
+
+  startCapturing(name) {
+    if (!shouldReport) { return }
+    newrelicClient = require('newrelic')
+    signalfxClient = new signalfx.Ingest(SIGNALFX_TOKEN, {
+      dimensions: {
+        name,
+        host: os.hostname(),
+        pid: pid.toString(),
+        env: NODE_ENV,
+      },
+    })
+  },
 
   reportError(error) {
-    if (this.newrelic && this.shouldReport) {
-      this.newrelic.noticeError(error)
+    if (!newrelicClient || !shouldReport) { return }
+    newrelicClient.noticeError(error)
+  },
+
+  reportMetric({name, value, type, dimensions = {}} = {}) {
+    if (!signalfxClient || !shouldReport) { return }
+    if (!name) {
+      throw new Error('Metrics.reportMetric requires a metric.name')
     }
-  }
-
-  reportMetric() {
-
-  }
+    if (value == null) {
+      throw new Error('Metrics.reportMetric requires a metric.value')
+    }
+    if (!type) {
+      throw new Error('Metrics.reportMetric requires a metric.type from Metrics.MetricTypes')
+    }
+    const metric = {metric: name, value, timestamp: Date.now(), dimensions}
+    signalfxClient.send({[type]: [metric]})
+  },
 }
 
-module.exports = new Metrics()
+module.exports = Metrics
diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js
index 0968db821..6e520d0e6 100644
--- a/packages/nylas-dashboard/app.js
+++ b/packages/nylas-dashboard/app.js
@@ -1,5 +1,5 @@
 const {Metrics} = require(`nylas-core`)
-Metrics.startCapturing()
+Metrics.startCapturing('nylas-k2-dashboard')
 
 const fs = require('fs');
 const path = require('path');
diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js
index dba5ee509..ccbf51eb8 100644
--- a/packages/nylas-message-processor/app.js
+++ b/packages/nylas-message-processor/app.js
@@ -1,5 +1,5 @@
 const {Metrics} = require(`nylas-core`)
-Metrics.startCapturing()
+Metrics.startCapturing('nylas-k2-message-processor')
 
 const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`)
 const {processors} = require('./processors')
diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js
index 39b9cdee9..d043707c7 100644
--- a/packages/nylas-sync/app.js
+++ b/packages/nylas-sync/app.js
@@ -1,5 +1,5 @@
 const {Metrics} = require(`nylas-core`)
-Metrics.startCapturing()
+Metrics.startCapturing('nylas-k2-sync')
 
 const {DatabaseConnector, Logger} = require('nylas-core')
 const SyncProcessManager = require('./sync-process-manager');
diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index 074beba90..f1bcbeeaa 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -64,9 +64,15 @@ class SyncProcessManager {
     client.setAsync(key, Date.now())
     .then(() => client.expireAsync(key, HEARTBEAT_EXPIRES))
     .then(() => {
+      const accountsSyncing = Object.keys(this._workers).length
       this._logger.info({
-        accounts_syncing_count: Object.keys(this._workers).length,
+        accounts_syncing_count: accountsSyncing,
       }, "ProcessManager: 💘")
+      global.Metrics.reportMetric({
+        name: 'accounts_syncing_count',
+        value: accountsSyncing,
+        type: global.Metrics.MetricTypes.CumulativeCounter,
+      })
     })
   }
 

From 592b2af986c687d885bae1d54ca7ed4af65377e0 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 14 Jul 2016 10:49:40 -0700
Subject: [PATCH 286/800] Update metric type for `accounts_syncing_count`

---
 packages/nylas-sync/sync-process-manager.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/nylas-sync/sync-process-manager.js b/packages/nylas-sync/sync-process-manager.js
index f1bcbeeaa..0c2e79b3a 100644
--- a/packages/nylas-sync/sync-process-manager.js
+++ b/packages/nylas-sync/sync-process-manager.js
@@ -71,7 +71,7 @@ class SyncProcessManager {
       global.Metrics.reportMetric({
         name: 'accounts_syncing_count',
         value: accountsSyncing,
-        type: global.Metrics.MetricTypes.CumulativeCounter,
+        type: global.Metrics.MetricTypes.Gauge,
       })
     })
   }

From b58c11605cb6223b6657ef02d3143281b42e3daf Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 14 Jul 2016 11:20:58 -0700
Subject: [PATCH 287/800] Moves metrics into its own pkg to be able to
 instrument other services

- With the Metrics module inside nylas-core, and bc of our current lerna
setup, we required other modules like sequelize and redis before requiring
newrelic, thus preventing them from being properly instrumented
---
 package.json                                         |  2 --
 packages/nylas-api/app.js                            |  2 +-
 packages/nylas-api/package.json                      |  1 +
 packages/nylas-core/index.js                         |  1 -
 packages/nylas-dashboard/app.js                      |  2 +-
 packages/nylas-dashboard/package.json                |  3 ++-
 packages/nylas-message-processor/app.js              |  2 +-
 packages/nylas-message-processor/package.json        |  3 ++-
 .../metrics.js => nylas-metrics/index.js}            |  0
 packages/nylas-metrics/package.json                  | 12 ++++++++++++
 packages/nylas-sync/app.js                           |  2 +-
 packages/nylas-sync/package.json                     |  1 +
 12 files changed, 22 insertions(+), 9 deletions(-)
 rename packages/{nylas-core/metrics.js => nylas-metrics/index.js} (100%)
 create mode 100644 packages/nylas-metrics/package.json

diff --git a/package.json b/package.json
index b102ba807..b67104175 100644
--- a/package.json
+++ b/package.json
@@ -12,14 +12,12 @@
     "imap": "0.8.x",
     "lerna": "2.0.0-beta.23",
     "mysql": "^2.11.1",
-    "newrelic": "^1.28.1",
     "pm2": "^1.1.3",
     "promise-props": "^1.0.0",
     "promise.prototype.finally": "^1.0.1",
     "redis": "2.x.x",
     "rx": "4.x.x",
     "sequelize": "3.x.x",
-    "signalfx": "^3.0.1",
     "underscore": "1.x.x",
     "utf7": "https://github.com/truebit/utf7/archive/1f753bac59b99d93b17a5ef11681e232465e2558.tar.gz"
   },
diff --git a/packages/nylas-api/app.js b/packages/nylas-api/app.js
index 26180ae27..f4979eb95 100644
--- a/packages/nylas-api/app.js
+++ b/packages/nylas-api/app.js
@@ -1,4 +1,4 @@
-const {Metrics} = require(`nylas-core`)
+const Metrics = require(`nylas-metrics`)
 Metrics.startCapturing('nylas-k2-api')
 
 const Hapi = require('hapi');
diff --git a/packages/nylas-api/package.json b/packages/nylas-api/package.json
index 7d9f53b24..137f9e340 100644
--- a/packages/nylas-api/package.json
+++ b/packages/nylas-api/package.json
@@ -17,6 +17,7 @@
     "joi": "8.4.2",
     "nylas-core": "0.x.x",
     "nylas-sync": "0.x.x",
+    "nylas-metrics": "0.x.x",
     "vision": "4.1.0"
   }
 }
diff --git a/packages/nylas-core/index.js b/packages/nylas-core/index.js
index 9cff4c9f7..b6348ddd4 100644
--- a/packages/nylas-core/index.js
+++ b/packages/nylas-core/index.js
@@ -11,6 +11,5 @@ module.exports = {
   MessageTypes: require('./message-types'),
   Logger: require('./logger'),
   Errors: require('./imap-errors'),
-  Metrics: require('./metrics'),
   PromiseUtils: require('./promise-utils'),
 }
diff --git a/packages/nylas-dashboard/app.js b/packages/nylas-dashboard/app.js
index 6e520d0e6..dbe069770 100644
--- a/packages/nylas-dashboard/app.js
+++ b/packages/nylas-dashboard/app.js
@@ -1,4 +1,4 @@
-const {Metrics} = require(`nylas-core`)
+const Metrics = require(`nylas-metrics`)
 Metrics.startCapturing('nylas-k2-dashboard')
 
 const fs = require('fs');
diff --git a/packages/nylas-dashboard/package.json b/packages/nylas-dashboard/package.json
index f41efb12a..c1d2bc8e0 100644
--- a/packages/nylas-dashboard/package.json
+++ b/packages/nylas-dashboard/package.json
@@ -12,6 +12,7 @@
     "hapi": "13.4.1",
     "hapi-plugin-websocket": "0.9.2",
     "inert": "4.0.0",
-    "nylas-core": "0.x.x"
+    "nylas-core": "0.x.x",
+    "nylas-metrics": "0.x.x"
   }
 }
diff --git a/packages/nylas-message-processor/app.js b/packages/nylas-message-processor/app.js
index ccbf51eb8..b2ff487d0 100644
--- a/packages/nylas-message-processor/app.js
+++ b/packages/nylas-message-processor/app.js
@@ -1,4 +1,4 @@
-const {Metrics} = require(`nylas-core`)
+const Metrics = require(`nylas-metrics`)
 Metrics.startCapturing('nylas-k2-message-processor')
 
 const {PubsubConnector, DatabaseConnector, Logger} = require(`nylas-core`)
diff --git a/packages/nylas-message-processor/package.json b/packages/nylas-message-processor/package.json
index 74ac74f2d..a50b1ccf3 100644
--- a/packages/nylas-message-processor/package.json
+++ b/packages/nylas-message-processor/package.json
@@ -14,7 +14,8 @@
     "jasmine": "2.4.1",
     "mailparser": "0.6.0",
     "mimelib": "0.2.19",
-    "nylas-core": "0.x.x"
+    "nylas-core": "0.x.x",
+    "nylas-metrics": "0.x.x"
   },
   "devDependencies": {
     "babel-cli": "^6.10.1",
diff --git a/packages/nylas-core/metrics.js b/packages/nylas-metrics/index.js
similarity index 100%
rename from packages/nylas-core/metrics.js
rename to packages/nylas-metrics/index.js
diff --git a/packages/nylas-metrics/package.json b/packages/nylas-metrics/package.json
new file mode 100644
index 000000000..923793087
--- /dev/null
+++ b/packages/nylas-metrics/package.json
@@ -0,0 +1,12 @@
+{
+  "name": "nylas-metrics",
+  "version": "0.0.1",
+  "description": "Metrics package",
+  "main": "index.js",
+  "dependencies": {
+    "newrelic": "^1.28.1",
+    "signalfx": "^3.0.1"
+  },
+  "author": "Nylas",
+  "license": "ISC"
+}
diff --git a/packages/nylas-sync/app.js b/packages/nylas-sync/app.js
index d043707c7..1d8ccd647 100644
--- a/packages/nylas-sync/app.js
+++ b/packages/nylas-sync/app.js
@@ -1,4 +1,4 @@
-const {Metrics} = require(`nylas-core`)
+const Metrics = require(`nylas-metrics`)
 Metrics.startCapturing('nylas-k2-sync')
 
 const {DatabaseConnector, Logger} = require('nylas-core')
diff --git a/packages/nylas-sync/package.json b/packages/nylas-sync/package.json
index 17f307081..1ac52b60b 100644
--- a/packages/nylas-sync/package.json
+++ b/packages/nylas-sync/package.json
@@ -5,6 +5,7 @@
   "dependencies": {
     "nylas-core": "0.x.x",
     "nylas-message-processor": "0.x.x",
+    "nylas-metrics": "0.x.x",
     "xoauth2": "1.1.0"
   },
   "scripts": {

From 6ce54c2a34de9358599f8f564a01fd71589b7ec8 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Thu, 14 Jul 2016 11:34:23 -0700
Subject: [PATCH 288/800] Add send endpoint

---
 packages/nylas-api/package.json              |  1 +
 packages/nylas-api/routes/delta.js           |  3 +-
 packages/nylas-api/routes/send.js            | 53 ++++++++++++++++++++
 packages/nylas-core/models/shared/account.js | 15 ++++++
 4 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 packages/nylas-api/routes/send.js

diff --git a/packages/nylas-api/package.json b/packages/nylas-api/package.json
index 137f9e340..3f0e6e7c7 100644
--- a/packages/nylas-api/package.json
+++ b/packages/nylas-api/package.json
@@ -15,6 +15,7 @@
     "hapi-swagger": "6.1.0",
     "inert": "4.0.0",
     "joi": "8.4.2",
+    "nodemailer": "2.5.0",
     "nylas-core": "0.x.x",
     "nylas-sync": "0.x.x",
     "nylas-metrics": "0.x.x",
diff --git a/packages/nylas-api/routes/delta.js b/packages/nylas-api/routes/delta.js
index 21f3fc3c3..3f379e38d 100644
--- a/packages/nylas-api/routes/delta.js
+++ b/packages/nylas-api/routes/delta.js
@@ -24,10 +24,9 @@ function inflateTransactions(db, transactionModels = []) {
     return ModelKlass.findAll({where: {id: ids}, include: includes})
     .then((models = []) => {
       for (const model of models) {
-        model.dataValues.object = object
         const tsForId = byObjectIds[model.id];
         if (!tsForId || tsForId.length === 0) { continue; }
-        for (const t of tsForId) { t.attributes = model.dataValues; }
+        for (const t of tsForId) { t.attributes = model.toJSON(); }
       }
     })
   })).then(() => transactions.map(JSON.stringify).join("\n"))
diff --git a/packages/nylas-api/routes/send.js b/packages/nylas-api/routes/send.js
new file mode 100644
index 000000000..e987bcbfa
--- /dev/null
+++ b/packages/nylas-api/routes/send.js
@@ -0,0 +1,53 @@
+const Joi = require('joi');
+const nodemailer = require('nodemailer');
+const {DatabaseConnector} = require('nylas-core');
+
+function toParticipant(payload) {
+  return payload.map((p) => `${p.name} <${p.email}>`).join(',')
+}
+
+module.exports = (server) => {
+  server.route({
+    method: 'POST',
+    path: '/send',
+    config: {
+      validate: {
+        payload: {
+          subject: Joi.string(),
+          reply_to_message_id: Joi.number().integer(),
+          from: Joi.array(),
+          reply_to: Joi.array(),
+          to: Joi.array(),
+          cc: Joi.array(),
+          bcc: Joi.array(),
+          body: Joi.string(),
+          file_ids: Joi.array(),
+        },
+      },
+    },
+    handler: (request, reply) => { DatabaseConnector.forShared().then((db) => {
+      const accountId = request.auth.credentials.id;
+      db.Account.findById(accountId).then((account) => {
+        const sender = nodemailer.createTransport(account.smtpConfig());
+        const data = request.payload;
+
+        const msg = {}
+        for (key of ['from', 'to', 'cc', 'bcc']) {
+          if (data[key]) msg[key] = toParticipant(data[key])
+        }
+        msg.subject = data.subject,
+        msg.html = data.body,
+
+        console.log("------------------------------------------------")
+        console.log(msg)
+        sender.sendMail(msg, (error, info) => {
+          console.log("DONE ===========================================");
+          console.log(error)
+          console.log(info)
+          if (error) { reply(error).code(400) }
+          else { reply(info.response) }
+        });
+      })
+    })},
+  });
+};
diff --git a/packages/nylas-core/models/shared/account.js b/packages/nylas-core/models/shared/account.js
index 4e1297025..f5ad50f26 100644
--- a/packages/nylas-core/models/shared/account.js
+++ b/packages/nylas-core/models/shared/account.js
@@ -67,6 +67,21 @@ module.exports = (sequelize, Sequelize) => {
           return null;
         }
       },
+
+      smtpConfig: function smtpConfig() {
+        if (this.provider !== "imap") {
+          throw new Error("Non IMAP not yet supported")
+        }
+        const {smtp_username, smtp_password} = this.decryptedCredentials();
+        const {smtp_host, smtp_port, ssl_required} = this.connectionSettings;
+
+        return {
+          port: smtp_port, host: smtp_host, secure: ssl_required,
+          auth: { user: smtp_username, pass: smtp_password, },
+        }
+
+      }
+
     },
   });
 

From 0921df432b9af62e313f2238de23fb9702691c03 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Thu, 14 Jul 2016 11:40:46 -0700
Subject: [PATCH 289/800] Uses default from address

---
 packages/nylas-api/routes/send.js | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/packages/nylas-api/routes/send.js b/packages/nylas-api/routes/send.js
index e987bcbfa..90725786c 100644
--- a/packages/nylas-api/routes/send.js
+++ b/packages/nylas-api/routes/send.js
@@ -35,15 +35,13 @@ module.exports = (server) => {
         for (key of ['from', 'to', 'cc', 'bcc']) {
           if (data[key]) msg[key] = toParticipant(data[key])
         }
+        if (!msg.from || msg.from.length === 0) {
+          msg.from = `${account.name} <${account.emailAddress}>`
+        }
         msg.subject = data.subject,
         msg.html = data.body,
 
-        console.log("------------------------------------------------")
-        console.log(msg)
         sender.sendMail(msg, (error, info) => {
-          console.log("DONE ===========================================");
-          console.log(error)
-          console.log(info)
           if (error) { reply(error).code(400) }
           else { reply(info.response) }
         });

From 37b2323cc716b663246bf3227d8b424e6727d192 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 14 Jul 2016 11:59:45 -0700
Subject: [PATCH 290/800] Add metric for sync errors

---
 packages/nylas-sync/sync-worker.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 1a992c5ad..aeadcb213 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -205,6 +205,11 @@ class SyncWorker {
       return Promise.resolve()
     }
 
+    global.Metrics.reportMetric({
+      name: 'sync_error',
+      value: 1,
+      type: global.Metrics.Counter,
+    })
     this._account.syncError = jsonError(error)
     return this._account.save()
   }

From 099e200ec5d40e01af722c99db205298336b960a Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 14 Jul 2016 13:44:43 -0700
Subject: [PATCH 291/800] Only log sync errors if they are permanent errors

---
 packages/nylas-sync/sync-worker.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index aeadcb213..388060445 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -197,14 +197,13 @@ class SyncWorker {
   }
 
   onSyncError(error) {
-    this._logger.error(error, `SyncWorker: Error while syncing account`)
     this.closeConnection()
-
     // Continue to retry if it was a network error
     if (error instanceof Errors.RetryableError) {
       return Promise.resolve()
     }
 
+    this._logger.error(error, `SyncWorker: Error while syncing account`)
     global.Metrics.reportMetric({
       name: 'sync_error',
       value: 1,

From b14d5f7e8dd2f99b7aeb82628c104284f76d4a7b Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Thu, 14 Jul 2016 14:51:48 -0700
Subject: [PATCH 292/800] Add process load counts to the dashboard

---
 packages/nylas-dashboard/public/css/app.css   | 23 +++++++++++++
 packages/nylas-dashboard/public/index.html    |  1 +
 .../public/js/account-filter.jsx              |  2 +-
 packages/nylas-dashboard/public/js/app.jsx    | 29 +++++++++-------
 .../public/js/process-loads.jsx               | 34 +++++++++++++++++++
 packages/nylas-dashboard/routes/websocket.js  |  2 ++
 6 files changed, 78 insertions(+), 13 deletions(-)
 create mode 100644 packages/nylas-dashboard/public/js/process-loads.jsx

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index f1d039511..2cfe5fe04 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -17,6 +17,10 @@ pre {
   margin: 0;
 }
 
+#accounts-wrapper {
+  position: relative;
+}
+
 .account {
   position: absolute;
   border-radius: 5px;
@@ -57,6 +61,7 @@ pre {
 
 #open-all-sync {
   color: #ffffff;
+  padding-left: 5px;
 }
 
 .right-action {
@@ -219,3 +224,21 @@ pre {
 .mini-account.errored {
   background-color: rgb(255, 38, 0);
 }
+
+.process-loads {
+  display: inline-block;
+  padding: 15px;
+  width: 250px;
+  margin: 15px 0;
+  background-color: white;
+}
+
+.process-loads .section {
+  text-decoration: underline;
+  margin-bottom: 10px;
+  font-size: 12px;
+}
+
+.account-filter {
+  padding-left: 5px;
+}
diff --git a/packages/nylas-dashboard/public/index.html b/packages/nylas-dashboard/public/index.html
index 3e0a66ba7..6b012db66 100644
--- a/packages/nylas-dashboard/public/index.html
+++ b/packages/nylas-dashboard/public/index.html
@@ -4,6 +4,7 @@
   
   
   
+  
   
   
   
diff --git a/packages/nylas-dashboard/public/js/account-filter.jsx b/packages/nylas-dashboard/public/js/account-filter.jsx
index 2d7a37b67..4c13b2462 100644
--- a/packages/nylas-dashboard/public/js/account-filter.jsx
+++ b/packages/nylas-dashboard/public/js/account-filter.jsx
@@ -2,7 +2,7 @@ const React = window.React;
 
 function AccountFilter(props) {
   return (
-    

+    

       Display:  this.onGroupChange()}
+          />
+          Group Accounts By Process
+        

+      )
 
-    return (
-      

-        
-         this.onFilter.call(this)} />
-         parseInt(id, 10))} />
+      if (this.state.groupByProcess) {
+        const accountsById = _.groupBy(this.state.accounts, 'id');
+        const processes = [];
+
+        for (const processName of Object.keys(this.state.processLoads)) {
+          const accounts = []
+
+          for (const accountId of this.state.processLoads[processName]) {
+            const account = accountsById[accountId][0];
+            accounts.push((
+              
+            ))
+          }
+          processes.push((
+            

+              {accounts}
+            

+          ))
+        }
+        content = (
+          

+            {groupByProcess}
+            

+              {processes}
+            

+          

+        )
+      } else {
+        content = (
+          

+            {groupByProcess}
+            

+              {
+                ids.sort((a, b) => a / 1 - b / 1).map((id) =>
+                  
+                )
+              }
+            

+          

+        )
+      }
+    } else {
+      let count = 0;
+      content = (
         

           {
             ids.sort((a, b) => a / 1 - b / 1).map((id) =>
-              
+      )
+    }
+
+    return (
+      

+        
+         this.onFilter.call(this)} />
+         parseInt(id, 10))} />
+        {content}
       

     )
   }
diff --git a/packages/nylas-dashboard/public/js/mini-account.jsx b/packages/nylas-dashboard/public/js/mini-account.jsx
index cd3539470..ef6ceca8b 100644
--- a/packages/nylas-dashboard/public/js/mini-account.jsx
+++ b/packages/nylas-dashboard/public/js/mini-account.jsx
@@ -15,11 +15,9 @@ class MiniAccount extends React.Component {
   }
 
   render() {
-    const {account, assignment, active} = this.props;
-
     let errorClass;
     let style;
-    if (account.sync_error) {
+    if (this.props.account.sync_error) {
       errorClass = 'errored';
       style = {};
     } else {
@@ -38,9 +36,6 @@ class MiniAccount extends React.Component {
 
 MiniAccount.propTypes = {
   account: React.PropTypes.object,
-  active: React.PropTypes.bool,
-  assignment: React.PropTypes.string,
-  count: React.PropTypes.number,
 };
 
 window.MiniAccount = MiniAccount;
diff --git a/packages/nylas-dashboard/public/js/process-loads.jsx b/packages/nylas-dashboard/public/js/process-loads.jsx
index 4e05d0725..e8917fd52 100644
--- a/packages/nylas-dashboard/public/js/process-loads.jsx
+++ b/packages/nylas-dashboard/public/js/process-loads.jsx
@@ -3,17 +3,17 @@ const React = window.React;
 function ProcessLoads(props) {
   let entries;
   let sumElem;
-  if (props.counts == null || Object.keys(props.counts).length === 0) {
+  if (props.loads == null || Object.keys(props.loads).length === 0) {
     entries = "No Data";
     sumElem = "";
   } else {
     entries = [];
     let sum = 0;
-    for (const processName of Object.keys(props.counts).sort()) {
-      const count = props.counts[processName];
+    for (const processName of Object.keys(props.loads).sort()) {
+      const count = props.loads[processName].length;
       sum += count;
       entries.push(
-        

+        

           {processName}: {count} accounts
         

       );
@@ -31,7 +31,7 @@ function ProcessLoads(props) {
 }
 
 ProcessLoads.propTypes = {
-  counts: React.PropTypes.object,
+  loads: React.PropTypes.object,
 }
 
 window.ProcessLoads = ProcessLoads;
diff --git a/packages/nylas-dashboard/routes/websocket.js b/packages/nylas-dashboard/routes/websocket.js
index f3aff4ccb..5bfb8e419 100644
--- a/packages/nylas-dashboard/routes/websocket.js
+++ b/packages/nylas-dashboard/routes/websocket.js
@@ -11,7 +11,7 @@ function onWebsocketConnected(wss, ws) {
       updatedAccounts: [],
       activeAccountIds: [],
       assignments: {},
-      processLoadCounts: {},
+      processLoads: {},
     };
   }
   resetToSend();
@@ -43,7 +43,7 @@ function onWebsocketConnected(wss, ws) {
         toSend.activeAccountIds = accountIds;
       });
       const getAssignments = SchedulerUtils.forEachAccountList((identity, accountIds) => {
-        toSend.processLoadCounts[identity] = accountIds.length;
+        toSend.processLoads[identity] = accountIds;
         for (const accountId of accountIds) {
           toSend.assignments[accountId] = identity;
         }

From e13097045ba72782abbc82788a7d6d0b50eac736 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Fri, 15 Jul 2016 12:39:28 -0700
Subject: [PATCH 313/800] Log boxname and category name as well

---
 packages/nylas-sync/imap/fetch-messages-in-folder.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/nylas-sync/imap/fetch-messages-in-folder.js b/packages/nylas-sync/imap/fetch-messages-in-folder.js
index 116c83409..5715664bb 100644
--- a/packages/nylas-sync/imap/fetch-messages-in-folder.js
+++ b/packages/nylas-sync/imap/fetch-messages-in-folder.js
@@ -299,6 +299,8 @@ class FetchMessagesInFolder {
 
       if (lastUIDValidity && (box.uidvalidity !== lastUIDValidity)) {
         this._logger.info({
+          boxname: box.name,
+          categoryname: this._category.name,
           remoteuidvalidity: box.uidvalidity,
           localuidvalidity: lastUIDValidity,
         }, `FetchMessagesInFolder: Recovering from UIDInvalidity`);

From 5e0bcc5effdb487f79f68e5656fc16e4846c0338 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Thu, 14 Jul 2016 18:36:21 -0700
Subject: [PATCH 314/800] Quieter logging on dev

---
 package.json                       |  2 +-
 packages/nylas-core/log-streams.js | 15 ++++++++++++++-
 packages/nylas-core/package.json   |  1 -
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index 313d7604e..522b53df2 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,7 @@
     "bunyan": "1.8.0",
     "bunyan-cloudwatch": "2.0.0",
     "bunyan-loggly": "^1.0.0",
-    "bunyan-prettystream": "^0.1.3",
+    "bunyan-prettystream": "emorikawa/node-bunyan-prettystream",
     "imap": "0.8.x",
     "lerna": "2.0.0-beta.23",
     "mysql": "^2.11.1",
diff --git a/packages/nylas-core/log-streams.js b/packages/nylas-core/log-streams.js
index fdd5746b5..64682ab08 100644
--- a/packages/nylas-core/log-streams.js
+++ b/packages/nylas-core/log-streams.js
@@ -25,9 +25,22 @@ const stdoutStream = {
 const getLogStreams = (name, env) => {
   switch (env) {
     case 'development': {
-      const prettyStdOut = new PrettyStream();
+      const prettyStdOut = new PrettyStream({
+        mode: 'pm2',
+        lessThan: 'error',
+      });
+      const prettyStdErr = new PrettyStream({
+        mode: 'pm2'
+      });
       prettyStdOut.pipe(process.stdout);
+      prettyStdErr.pipe(process.stderr);
       return [
+        {
+          type: 'raw',
+          level: 'error',
+          stream: prettyStdErr,
+          reemitErrorEvents: true,
+        },
         {
           type: 'raw',
           level: 'debug',
diff --git a/packages/nylas-core/package.json b/packages/nylas-core/package.json
index ac3938cb1..58d35ebf4 100644
--- a/packages/nylas-core/package.json
+++ b/packages/nylas-core/package.json
@@ -4,7 +4,6 @@
   "description": "Core shared packages",
   "main": "index.js",
   "dependencies": {
-    "bunyan": "^1.8.1",
     "xoauth2": "1.x.x"
   },
   "author": "Nylas",

From dffb87bd4af7ed9ae02a58c4c6dd94e096ae8534 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Fri, 15 Jul 2016 13:15:26 -0700
Subject: [PATCH 315/800] Report errors earlier

---
 packages/nylas-sync/sync-worker.js | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/nylas-sync/sync-worker.js b/packages/nylas-sync/sync-worker.js
index 35d71597c..e5fbdc0ff 100644
--- a/packages/nylas-sync/sync-worker.js
+++ b/packages/nylas-sync/sync-worker.js
@@ -202,10 +202,6 @@ class SyncWorker {
 
   onSyncError(error) {
     this.closeConnection()
-    // Continue to retry if it was a network error
-    if (error instanceof Errors.RetryableError) {
-      return Promise.resolve()
-    }
 
     this._logger.error(error, `SyncWorker: Error while syncing account`)
     global.Metrics.reportMetric({
@@ -213,6 +209,12 @@ class SyncWorker {
       value: 1,
       type: global.Metrics.Counter,
     })
+
+    // Continue to retry if it was a network error
+    if (error instanceof Errors.RetryableError) {
+      return Promise.resolve()
+    }
+
     this._account.syncError = jsonError(error)
     return this._account.save()
   }

From 0dad9bf7fa19cd76c195b897ac9a269cd9dbeb92 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Fri, 15 Jul 2016 13:19:59 -0700
Subject: [PATCH 316/800] Auto-scale MiniAccounts in the collapsed, ungrouped
 dashboard view

---
 packages/nylas-dashboard/public/css/app.css         |  6 ++----
 packages/nylas-dashboard/public/js/app.jsx          | 10 +++++++---
 packages/nylas-dashboard/public/js/mini-account.jsx |  9 ++++++---
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index a144daaf2..18eb7371f 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -207,16 +207,14 @@ pre {
 .mini-account::after {
   display: inline-block;
   position: relative;
-  height: 10px;
-  width: 10px;
+  height: 100%;
+  width: 100%;
   background-color: #666666;
   content: "";
   z-index: -1;
 }
 
 .mini-account {
-  height: 10px;
-  width: 10px;
   background-color: rgb(0, 255, 157);
   display: inline-block;
 }
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 7c458f8d9..13a2c86f5 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -208,7 +208,7 @@ class Root extends React.Component {
 
   onGroupChange() {
     this.setState({
-      groupByProcess: document.getElementById('group-by-proccess').checked,
+      groupByProcess: document.getElementById('group-by-process').checked,
     });
   }
 
@@ -232,7 +232,7 @@ class Root extends React.Component {
         

            this.onGroupChange()}
           />
           Group Accounts By Process
@@ -249,7 +249,7 @@ class Root extends React.Component {
           for (const accountId of this.state.processLoads[processName]) {
             const account = accountsById[accountId][0];
             accounts.push((
-              
+              
             ))
           }
           processes.push((
@@ -267,6 +267,9 @@ class Root extends React.Component {
           

         )
       } else {
+        const area = window.innerWidth * window.innerHeight * 0.75;
+        const singleAcctArea = area / Object.keys(this.state.accounts).length;
+        const acctSideDimension = Math.sqrt(singleAcctArea);
         content = (
           

             {groupByProcess}
@@ -276,6 +279,7 @@ class Root extends React.Component {
                   
                 )
               }
diff --git a/packages/nylas-dashboard/public/js/mini-account.jsx b/packages/nylas-dashboard/public/js/mini-account.jsx
index ef6ceca8b..3625ad489 100644
--- a/packages/nylas-dashboard/public/js/mini-account.jsx
+++ b/packages/nylas-dashboard/public/js/mini-account.jsx
@@ -16,13 +16,15 @@ class MiniAccount extends React.Component {
 
   render() {
     let errorClass;
-    let style;
+    let style = {
+      width: `${this.props.sideDimension}px`,
+      height: `${this.props.sideDimension}px`,
+    }
     if (this.props.account.sync_error) {
       errorClass = 'errored';
-      style = {};
     } else {
       errorClass = '';
-      style = {backgroundColor: this.calculateColor()};
+      style.backgroundColor = this.calculateColor();
     }
 
     return (
@@ -36,6 +38,7 @@ class MiniAccount extends React.Component {
 
 MiniAccount.propTypes = {
   account: React.PropTypes.object,
+  sideDimension: React.PropTypes.number,
 };
 
 window.MiniAccount = MiniAccount;

From 8140076e17391271d6ae106d8ed2f64aaffa6e9d Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Fri, 15 Jul 2016 16:09:10 -0700
Subject: [PATCH 317/800] Remove auto-scaling for the collapsed dashboard view,
 at least for now.

---
 packages/nylas-dashboard/public/css/app.css         | 2 ++
 packages/nylas-dashboard/public/js/app.jsx          | 6 +-----
 packages/nylas-dashboard/public/js/mini-account.jsx | 6 +-----
 3 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/packages/nylas-dashboard/public/css/app.css b/packages/nylas-dashboard/public/css/app.css
index 18eb7371f..c77a55c1c 100644
--- a/packages/nylas-dashboard/public/css/app.css
+++ b/packages/nylas-dashboard/public/css/app.css
@@ -217,6 +217,8 @@ pre {
 .mini-account {
   background-color: rgb(0, 255, 157);
   display: inline-block;
+  width: 10px;
+  height: 10px;
 }
 
 .mini-account.errored {
diff --git a/packages/nylas-dashboard/public/js/app.jsx b/packages/nylas-dashboard/public/js/app.jsx
index 13a2c86f5..1b03ab9f6 100644
--- a/packages/nylas-dashboard/public/js/app.jsx
+++ b/packages/nylas-dashboard/public/js/app.jsx
@@ -249,7 +249,7 @@ class Root extends React.Component {
           for (const accountId of this.state.processLoads[processName]) {
             const account = accountsById[accountId][0];
             accounts.push((
-              
+              
             ))
           }
           processes.push((
@@ -267,9 +267,6 @@ class Root extends React.Component {
           

         )
       } else {
-        const area = window.innerWidth * window.innerHeight * 0.75;
-        const singleAcctArea = area / Object.keys(this.state.accounts).length;
-        const acctSideDimension = Math.sqrt(singleAcctArea);
         content = (
           

             {groupByProcess}
@@ -279,7 +276,6 @@ class Root extends React.Component {
                   
                 )
               }
diff --git a/packages/nylas-dashboard/public/js/mini-account.jsx b/packages/nylas-dashboard/public/js/mini-account.jsx
index 3625ad489..cde2a8d74 100644
--- a/packages/nylas-dashboard/public/js/mini-account.jsx
+++ b/packages/nylas-dashboard/public/js/mini-account.jsx
@@ -16,10 +16,7 @@ class MiniAccount extends React.Component {
 
   render() {
     let errorClass;
-    let style = {
-      width: `${this.props.sideDimension}px`,
-      height: `${this.props.sideDimension}px`,
-    }
+    let style = {};
     if (this.props.account.sync_error) {
       errorClass = 'errored';
     } else {
@@ -38,7 +35,6 @@ class MiniAccount extends React.Component {
 
 MiniAccount.propTypes = {
   account: React.PropTypes.object,
-  sideDimension: React.PropTypes.number,
 };
 
 window.MiniAccount = MiniAccount;

From 7efe5db7e941c7781bdeefd89f83b9e627037914 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Fri, 15 Jul 2016 16:25:18 -0700
Subject: [PATCH 318/800] USE_CONSOLE_LOG for simple console logs

---
 packages/nylas-core/logger.js | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js
index b50e704f9..84ee0742d 100644
--- a/packages/nylas-core/logger.js
+++ b/packages/nylas-core/logger.js
@@ -3,6 +3,16 @@ const {getLogStreams} = require('./log-streams')
 const NODE_ENV = process.env.NODE_ENV || 'unknown'
 
 function createLogger(name, env = NODE_ENV) {
+  if (process.env.USE_CONSOLE_LOG) {
+    console.forAccount = () => {
+      return console;
+    }
+    console.child = () => {
+      return console;
+    }
+    return console;
+  }
+
   const childLogs = new Map()
   const logger = bunyan.createLogger({
     name,

From 549ced612825ff681f7927db4e17812d9620e853 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Fri, 15 Jul 2016 16:28:13 -0700
Subject: [PATCH 319/800] =?UTF-8?q?Revert=20change,=20@evan=E2=80=99s=20fi?=
 =?UTF-8?q?x=20is=20better?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 packages/nylas-core/logger.js | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/packages/nylas-core/logger.js b/packages/nylas-core/logger.js
index 84ee0742d..b50e704f9 100644
--- a/packages/nylas-core/logger.js
+++ b/packages/nylas-core/logger.js
@@ -3,16 +3,6 @@ const {getLogStreams} = require('./log-streams')
 const NODE_ENV = process.env.NODE_ENV || 'unknown'
 
 function createLogger(name, env = NODE_ENV) {
-  if (process.env.USE_CONSOLE_LOG) {
-    console.forAccount = () => {
-      return console;
-    }
-    console.child = () => {
-      return console;
-    }
-    return console;
-  }
-
   const childLogs = new Map()
   const logger = bunyan.createLogger({
     name,

From f9f78968b15896a496624e12e4609750cd2421f5 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Fri, 15 Jul 2016 19:01:50 -0700
Subject: [PATCH 320/800] Use defaultValue as a template, fix critical sync
 issue

---
 packages/nylas-core/database-types.js | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/nylas-core/database-types.js b/packages/nylas-core/database-types.js
index 8216e8d0f..7a59c5577 100644
--- a/packages/nylas-core/database-types.js
+++ b/packages/nylas-core/database-types.js
@@ -5,18 +5,22 @@ module.exports = {
     type: Sequelize.TEXT,
     get: function get() {
       const val = this.getDataValue(fieldName);
-      if (!val) { return defaultValue }
+      if (!val) {
+        return defaultValue ? Object.assign({}, defaultValue) : null;
+      }
       return JSON.parse(val);
     },
     set: function set(val) {
       this.setDataValue(fieldName, JSON.stringify(val));
     },
   }),
-  JSONARRAYType: (fieldName, {defaultValue = []} = {}) => ({
+  JSONARRAYType: (fieldName) => ({
     type: Sequelize.TEXT,
     get: function get() {
       const val = this.getDataValue(fieldName);
-      if (!val) { return defaultValue }
+      if (!val) {
+        return [];
+      }
       return JSON.parse(val);
     },
     set: function set(val) {

From e10e51ab5d2029cd260ed9c8c5e8e33d74a39e05 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 11 Oct 2016 00:44:10 -0700
Subject: [PATCH 321/800] A few changes to support linking to K2 via the new
 GUI

---
 packages/nylas-api/routes/accounts.js         | 30 +++++++++++++++++++
 packages/nylas-core/models/account/contact.js |  2 +-
 packages/nylas-core/models/account/file.js    |  2 +-
 packages/nylas-core/models/account/folder.js  |  2 +-
 packages/nylas-core/models/account/label.js   |  2 +-
 packages/nylas-core/models/account/message.js |  2 +-
 .../models/account/syncbackRequest.js         |  2 +-
 packages/nylas-core/models/account/thread.js  |  2 +-
 .../nylas-core/models/account/transaction.js  |  2 +-
 pm2-dev.yml                                   |  1 +
 10 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/packages/nylas-api/routes/accounts.js b/packages/nylas-api/routes/accounts.js
index 856bd5a66..62a227d11 100644
--- a/packages/nylas-api/routes/accounts.js
+++ b/packages/nylas-api/routes/accounts.js
@@ -2,6 +2,36 @@ const Serialization = require('../serialization');
 const {DatabaseConnector} = require('nylas-core');
 
 module.exports = (server) => {
+  if (process.env.ALLOW_LIST_ACCOUNTS) {
+    server.route({
+      method: 'GET',
+      path: '/accounts',
+      config: {
+        auth: false,
+        description: 'Returns all accounts, only in dev mode. Only intended to easily link N1.',
+        tags: ['accounts'],
+      },
+      handler: (request, reply) => {
+        DatabaseConnector.forShared().then((db) => {
+          const {Account, AccountToken} = db;
+
+          // N1 assumes that the local sync engine uses the account IDs as the
+          // auth tokens. K2 supports real auth tokens out of the box, but we
+          // create ones that have value = accountId.
+          Account.all().then((accounts) => {
+            Promise.all(
+              accounts.map(({id}) => AccountToken.create({accountId: id, value: id})
+            )).finally(() =>
+              reply(accounts.map((account) =>
+                Object.assign(account.toJSON(), {id: `${account.id}`, auth_token: `${account.id}`})
+              ))
+            )
+          });
+        });
+      },
+    });
+  }
+
   server.route({
     method: 'GET',
     path: '/account',
diff --git a/packages/nylas-core/models/account/contact.js b/packages/nylas-core/models/account/contact.js
index 425933234..a526732f6 100644
--- a/packages/nylas-core/models/account/contact.js
+++ b/packages/nylas-core/models/account/contact.js
@@ -14,7 +14,7 @@ module.exports = (sequelize, Sequelize) => {
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: this.id,
+          id: `${this.id}`,
           account_id: this.accountId,
           object: 'contact',
           email: this.email,
diff --git a/packages/nylas-core/models/account/file.js b/packages/nylas-core/models/account/file.js
index e9f22e043..287a7a6ab 100644
--- a/packages/nylas-core/models/account/file.js
+++ b/packages/nylas-core/models/account/file.js
@@ -40,7 +40,7 @@ module.exports = (sequelize, Sequelize) => {
       },
       toJSON: function toJSON() {
         return {
-          id: this.id,
+          id: `${this.id}`,
           object: 'file',
           account_id: this.accountId,
           message_id: this.messageId,
diff --git a/packages/nylas-core/models/account/folder.js b/packages/nylas-core/models/account/folder.js
index f3875bff4..fa724000b 100644
--- a/packages/nylas-core/models/account/folder.js
+++ b/packages/nylas-core/models/account/folder.js
@@ -23,7 +23,7 @@ module.exports = (sequelize, Sequelize) => {
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: this.id,
+          id: `${this.id}`,
           account_id: this.accountId,
           object: 'folder',
           name: this.role,
diff --git a/packages/nylas-core/models/account/label.js b/packages/nylas-core/models/account/label.js
index f21902e5c..d943bc160 100644
--- a/packages/nylas-core/models/account/label.js
+++ b/packages/nylas-core/models/account/label.js
@@ -20,7 +20,7 @@ module.exports = (sequelize, Sequelize) => {
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: this.id,
+          id: `${this.id}`,
           account_id: this.accountId,
           object: 'label',
           name: this.role,
diff --git a/packages/nylas-core/models/account/message.js b/packages/nylas-core/models/account/message.js
index 2634947d8..1ed3302fe 100644
--- a/packages/nylas-core/models/account/message.js
+++ b/packages/nylas-core/models/account/message.js
@@ -98,7 +98,7 @@ module.exports = (sequelize, Sequelize) => {
         // Message though and need to protect `this.date` from null
         // errors.
         return {
-          id: this.id,
+          id: `${this.id}`,
           account_id: this.accountId,
           object: 'message',
           body: this.body,
diff --git a/packages/nylas-core/models/account/syncbackRequest.js b/packages/nylas-core/models/account/syncbackRequest.js
index 720461ea0..34ce0215a 100644
--- a/packages/nylas-core/models/account/syncbackRequest.js
+++ b/packages/nylas-core/models/account/syncbackRequest.js
@@ -14,7 +14,7 @@ module.exports = (sequelize, Sequelize) => {
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: this.id,
+          id: `${this.id}`,
           type: this.type,
           status: this.status,
           error: this.error,
diff --git a/packages/nylas-core/models/account/thread.js b/packages/nylas-core/models/account/thread.js
index 75f303208..eae4e632e 100644
--- a/packages/nylas-core/models/account/thread.js
+++ b/packages/nylas-core/models/account/thread.js
@@ -49,7 +49,7 @@ module.exports = (sequelize, Sequelize) => {
         }
 
         const response = {
-          id: this.id,
+          id: `${this.id}`,
           object: 'thread',
           folders: this.folders,
           labels: this.labels,
diff --git a/packages/nylas-core/models/account/transaction.js b/packages/nylas-core/models/account/transaction.js
index 6f22cb15c..a1dfeaa25 100644
--- a/packages/nylas-core/models/account/transaction.js
+++ b/packages/nylas-core/models/account/transaction.js
@@ -10,7 +10,7 @@ module.exports = (sequelize, Sequelize) => {
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: this.id,
+          id: `${this.id}`,
           event: this.event,
           object: this.object,
           objectId: this.objectId,
diff --git a/pm2-dev.yml b/pm2-dev.yml
index 4805cbdae..5401f2564 100644
--- a/pm2-dev.yml
+++ b/pm2-dev.yml
@@ -4,6 +4,7 @@ apps:
     name     : api
     env      :
       PORT: 5100
+      ALLOW_LIST_ACCOUNTS : true
       DB_ENCRYPTION_ALGORITHM : "aes-256-ctr"
       DB_ENCRYPTION_PASSWORD : "d6F3Efeq"
       GMAIL_CLIENT_ID : "271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com"

From 0fe6343137dfccdf4be88ea0ccea92e60b6e6e20 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Mon, 21 Nov 2016 11:07:57 -0800
Subject: [PATCH 322/800] Remove filtering from the collection APIs.

Affected collections were categories, contacts, files, messages, and threads.
---
 packages/nylas-api/route-helpers.js     |  22 +---
 packages/nylas-api/routes/categories.js |   1 -
 packages/nylas-api/routes/contacts.js   |  14 ---
 packages/nylas-api/routes/files.js      |  17 ---
 packages/nylas-api/routes/messages.js   |  76 ++----------
 packages/nylas-api/routes/threads.js    | 149 +++---------------------
 6 files changed, 29 insertions(+), 250 deletions(-)

diff --git a/packages/nylas-api/route-helpers.js b/packages/nylas-api/route-helpers.js
index 9ed1cac0d..319ba8630 100644
--- a/packages/nylas-api/route-helpers.js
+++ b/packages/nylas-api/route-helpers.js
@@ -1,5 +1,5 @@
 const Serialization = require('./serialization');
-const {PromiseUtils, PubsubConnector, MessageTypes} = require('nylas-core')
+const {PubsubConnector, MessageTypes} = require('nylas-core')
 
 module.exports = {
   createSyncbackRequest: function createSyncbackRequest(request, reply, syncRequestArgs) {
@@ -13,24 +13,4 @@ module.exports = {
       })
     })
   },
-  findFolderOrLabel: function findFolderOrLabel({Folder, Label}, str) {
-    return PromiseUtils.props({
-      folder: Folder.find({
-        where: { $or: [
-          { id: str },
-          { name: str },
-          { role: str },
-        ]},
-      }),
-      label: Label.find({
-        where: { $or: [
-          { id: str },
-          { name: str },
-          { role: str },
-        ]},
-      }),
-    }).then(({folder, label}) =>
-      folder || label || null
-    )
-  },
 }
diff --git a/packages/nylas-api/routes/categories.js b/packages/nylas-api/routes/categories.js
index d0143cda6..7b62141f8 100644
--- a/packages/nylas-api/routes/categories.js
+++ b/packages/nylas-api/routes/categories.js
@@ -17,7 +17,6 @@ module.exports = (server) => {
           query: {
             limit: Joi.number().integer().min(1).max(2000).default(100),
             offset: Joi.number().integer().min(0).default(0),
-            view: Joi.string().valid('expanded', 'count'),
           },
         },
         response: {
diff --git a/packages/nylas-api/routes/contacts.js b/packages/nylas-api/routes/contacts.js
index 549ce5cb8..cc5e2833b 100644
--- a/packages/nylas-api/routes/contacts.js
+++ b/packages/nylas-api/routes/contacts.js
@@ -11,9 +11,6 @@ module.exports = (server) => {
       tags: ['contacts'],
       validate: {
         query: {
-          name: Joi.string(),
-          email: Joi.string().email(),
-          view: Joi.string().valid('expanded', 'count'),
           limit: Joi.number().integer().min(1).max(2000).default(100),
           offset: Joi.number().integer().min(0).default(0),
         },
@@ -27,18 +24,7 @@ module.exports = (server) => {
     handler: (request, reply) => {
       request.getAccountDatabase().then((db) => {
         const {Contact} = db;
-        const query = request.query;
-        const where = {};
-
-        if (query.name) {
-          where.name = {like: query.name};
-        }
-        if (query.email) {
-          where.email = query.email;
-        }
-
         Contact.findAll({
-          where: where,
           limit: request.query.limit,
           offset: request.query.offset,
         }).then((contacts) => {
diff --git a/packages/nylas-api/routes/files.js b/packages/nylas-api/routes/files.js
index 46f7ee980..ec8db02fa 100644
--- a/packages/nylas-api/routes/files.js
+++ b/packages/nylas-api/routes/files.js
@@ -12,9 +12,6 @@ module.exports = (server) => {
       tags: ['files'],
       validate: {
         query: {
-          filename: Joi.string(),
-          message_id: Joi.number().integer().min(0),
-          content_type: Joi.string(),
           limit: Joi.number().integer().min(1).max(2000).default(100),
           offset: Joi.number().integer().min(0).default(0),
         },
@@ -28,21 +25,7 @@ module.exports = (server) => {
     handler: (request, reply) => {
       request.getAccountDatabase().then((db) => {
         const {File} = db;
-        const query = request.query;
-        const where = {};
-
-        if (query.filename) {
-          where.filename = query.filename;
-        }
-        if (query.message_id) {
-          where.messageId = query.message_id;
-        }
-        if (query.content_type) {
-          where.contentType = query.content_type;
-        }
-
         File.findAll({
-          where: where,
           limit: request.query.limit,
           offset: request.query.offset,
         }).then((files) => {
diff --git a/packages/nylas-api/routes/messages.js b/packages/nylas-api/routes/messages.js
index c8ea00b1e..58b82c54a 100644
--- a/packages/nylas-api/routes/messages.js
+++ b/packages/nylas-api/routes/messages.js
@@ -1,6 +1,6 @@
 const Joi = require('joi');
 const Serialization = require('../serialization');
-const {createSyncbackRequest, findFolderOrLabel} = require('../route-helpers');
+const {createSyncbackRequest} = require('../route-helpers');
 
 
 module.exports = (server) => {
@@ -13,17 +13,8 @@ module.exports = (server) => {
       tags: ['messages'],
       validate: {
         query: {
-          'unread': Joi.boolean(),
-          'starred': Joi.boolean(),
-          'subject': Joi.string(),
-          'thread_id': Joi.number().integer().min(0),
-          'received_before': Joi.date(),
-          'received_after': Joi.date(),
-          'filename': Joi.string(),
-          'in': Joi.string(),
-          'limit': Joi.number().integer().min(1).max(2000).default(100),
-          'offset': Joi.number().integer().min(0).default(0),
-          'view': Joi.string().valid('expanded', 'count'),
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          offset: Joi.number().integer().min(0).default(0),
         },
       },
       response: {
@@ -34,60 +25,13 @@ module.exports = (server) => {
     },
     handler: (request, reply) => {
       request.getAccountDatabase().then((db) => {
-        const {Message, Folder, Label, File} = db;
-        const query = request.query;
-        const where = {};
-        const include = [];
-
-        if (query.unread != null) {
-          where.unread = query.unread;
-        }
-        if (query.starred != null) {
-          where.starred = query.starred;
-        }
-        if (query.subject) {
-          where.subject = query.subject;
-        }
-        if (query.thread_id != null) {
-          where.threadId = query.thread_id;
-        }
-        if (query.received_before) {
-          where.date = {lt: query.received_before};
-        }
-        if (query.received_after) {
-          if (where.date) {
-            where.date.gt = query.received_after;
-          } else {
-            where.date = {gt: query.received_after};
-          }
-        }
-        if (query.filename) {
-          include.push({
-            model: File,
-            where: {filename: query.filename},
-          })
-        }
-
-        let loadAssociatedModels = Promise.resolve();
-        if (query.in) {
-          loadAssociatedModels = findFolderOrLabel({Folder, Label}, query.in)
-          .then((container) => {
-            include.push({
-              model: container.Model,
-              where: {id: container.id},
-            })
-          })
-        }
-
-        loadAssociatedModels.then(() => {
-          Message.findAll({
-            where: where,
-            limit: query.limit,
-            offset: query.offset,
-            include: include,
-          }).then((messages) => {
-            reply(Serialization.jsonStringify(messages));
-          })
+        const {Message, Folder, Label} = db;
+        Message.findAll({
+          limit: request.query.limit,
+          offset: request.query.offset,
+          include: [{model: Folder}, {model: Label}],
+        }).then((messages) => {
+          reply(Serialization.jsonStringify(messages));
         })
       })
     },
diff --git a/packages/nylas-api/routes/threads.js b/packages/nylas-api/routes/threads.js
index aa6d8102d..b8a058cf1 100644
--- a/packages/nylas-api/routes/threads.js
+++ b/packages/nylas-api/routes/threads.js
@@ -1,7 +1,7 @@
 const Joi = require('joi');
 const _ = require('underscore');
 const Serialization = require('../serialization');
-const {createSyncbackRequest, findFolderOrLabel} = require('../route-helpers')
+const {createSyncbackRequest} = require('../route-helpers')
 
 module.exports = (server) => {
   server.route({
@@ -13,19 +13,8 @@ module.exports = (server) => {
       tags: ['threads'],
       validate: {
         query: {
-          'id': Joi.number().integer().min(0),
-          'view': Joi.string().valid('expanded', 'count'),
-          'subject': Joi.string(),
-          'unread': Joi.boolean(),
-          'starred': Joi.boolean(),
-          'started_before': Joi.date().timestamp(),
-          'started_after': Joi.date().timestamp(),
-          'last_message_before': Joi.date().timestamp(),
-          'last_message_after': Joi.date().timestamp(),
-          'in': Joi.string().allow(Joi.number()),
-          'filename': Joi.string(),
-          'limit': Joi.number().integer().min(1).max(2000).default(100),
-          'offset': Joi.number().integer().min(0).default(0),
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          offset: Joi.number().integer().min(0).default(0),
         },
       },
       response: {
@@ -41,123 +30,21 @@ module.exports = (server) => {
     },
     handler: (request, reply) => {
       request.getAccountDatabase().then((db) => {
-        const {Thread, Folder, Label, Message, File} = db;
-        const query = request.query;
-        const where = {};
-        const include = [];
-
-        if (query.id) {
-          where.id = query.id;
-        }
-        if (query.subject) {
-          // the 'like' operator is case-insenstive in sequelite and for
-          // non-binary strings in mysql
-          where.subject = {like: query.subject};
-        }
-
-        // Boolean queries
-        if (query.unread) {
-          where.unreadCount = {gt: 0};
-        } else if (query.unread !== undefined) {
-          where.unreadCount = 0;
-        }
-        if (query.starred) {
-          where.starredCount = {gt: 0};
-        } else if (query.starred !== undefined) {
-          where.starredCount = 0;
-        }
-
-        // Timestamp queries
-        if (query.last_message_before) {
-          where.lastMessageReceivedDate = {lt: query.last_message_before};
-        }
-        if (query.last_message_after) {
-          if (where.lastMessageReceivedDate) {
-            where.lastMessageReceivedDate.gt = query.last_message_after;
-          } else {
-            where.lastMessageReceivedDate = {gt: query.last_message_after};
-          }
-        }
-        if (query.started_before) {
-          where.firstMessageDate = {lt: query.started_before};
-        }
-        if (query.started_after) {
-          if (where.firstMessageDate) {
-            where.firstMessageDate.gt = query.started_after;
-          } else {
-            where.firstMessageDate = {gt: query.started_after};
-          }
-        }
-
-        // Association queries
-        let loadAssociatedModels = Promise.resolve();
-        if (query.in) {
-          loadAssociatedModels = findFolderOrLabel({Folder, Label}, query.in)
-          .then((container) => {
-            include.push({
-              model: container.Model,
-              where: {id: container.id},
-            })
-            include.push({model: container.Model === Folder ? Label : Folder})
-          })
-        } else {
-          include.push({model: Folder})
-          include.push({model: Label})
-        }
-
-        const messagesInclude = [];
-        if (query.filename) {
-          messagesInclude.push({
-            model: File,
-            where: {filename: query.filename},
-          })
-        }
-        if (query.view === 'expanded') {
-          include.push({
-            model: Message,
-            as: 'messages',
-            attributes: _.without(Object.keys(Message.attributes), 'body'),
-            include: messagesInclude,
-          })
-        } else {
-          include.push({
-            model: Message,
-            as: 'messages',
-            attributes: ['id'],
-            include: messagesInclude,
-          })
-        }
-
-        if (query.view === 'count') {
-          loadAssociatedModels.then(() => {
-            return Thread.count({
-              where: where,
-              include: include,
-            }).then((count) => {
-              reply(Serialization.jsonStringify({count: count}));
-            });
-          })
-          return;
-        }
-
-        loadAssociatedModels.then(() => {
-          Thread.findAll({
-            limit: request.query.limit,
-            offset: request.query.offset,
-            where: where,
-            include: include,
-          }).then((threads) => {
-            // if the user requested the expanded viw, fill message.folder using
-            // thread.folders, since it must be a superset.
-            if (query.view === 'expanded') {
-              for (const thread of threads) {
-                for (const msg of thread.messages) {
-                  msg.folder = thread.folders.find(c => c.id === msg.folderId);
-                }
-              }
-            }
-            reply(Serialization.jsonStringify(threads));
-          })
+        const {Thread, Folder, Label, Message} = db;
+        Thread.findAll({
+          limit: request.query.limit,
+          offset: request.query.offset,
+          include: [
+            {model: Folder},
+            {model: Label},
+            {
+              model: Message,
+              as: 'messages',
+              attributes: _.without(Object.keys(Message.attributes), 'body'),
+            },
+          ],
+        }).then((threads) => {
+          reply(Serialization.jsonStringify(threads));
         })
       })
     },

From 7906e3303dbb8b8ff81c54e9643d4928d1d51f8a Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Mon, 21 Nov 2016 13:29:24 -0800
Subject: [PATCH 323/800] Add placeholders for various routes that N1 uses.

---
 packages/nylas-api/routes/calendars.js | 27 ++++++++++++++++++++++++++
 packages/nylas-api/routes/contacts.js  | 17 ++++++++++++++++
 packages/nylas-api/routes/drafts.js    | 27 ++++++++++++++++++++++++++
 packages/nylas-api/routes/events.js    | 27 ++++++++++++++++++++++++++
 packages/nylas-api/routes/metadata.js  | 27 ++++++++++++++++++++++++++
 5 files changed, 125 insertions(+)
 create mode 100644 packages/nylas-api/routes/calendars.js
 create mode 100644 packages/nylas-api/routes/drafts.js
 create mode 100644 packages/nylas-api/routes/events.js
 create mode 100644 packages/nylas-api/routes/metadata.js

diff --git a/packages/nylas-api/routes/calendars.js b/packages/nylas-api/routes/calendars.js
new file mode 100644
index 000000000..20e49acde
--- /dev/null
+++ b/packages/nylas-api/routes/calendars.js
@@ -0,0 +1,27 @@
+const Joi = require('joi');
+
+// TODO: This is a placeholder
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/calendars',
+    config: {
+      description: 'Returns calendars.',
+      notes: 'Notes go here',
+      tags: ['metadata'],
+      validate: {
+        query: {
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          offset: Joi.number().integer().min(0).default(0),
+          view: Joi.string().valid('count'),
+        },
+      },
+      response: {
+        schema: Joi.array(),
+      },
+    },
+    handler: (request, reply) => {
+      reply('[]');
+    },
+  });
+}
diff --git a/packages/nylas-api/routes/contacts.js b/packages/nylas-api/routes/contacts.js
index cc5e2833b..f8cc38f34 100644
--- a/packages/nylas-api/routes/contacts.js
+++ b/packages/nylas-api/routes/contacts.js
@@ -66,4 +66,21 @@ module.exports = (server) => {
       })
     },
   })
+
+  // TODO: This is a placeholder
+  server.route({
+    method: 'GET',
+    path: '/contacts/rankings',
+    config: {
+      description: 'Returns contact rankings.',
+      notes: 'Notes go here',
+      tags: ['contacts'],
+      response: {
+        schema: Serialization.jsonSchema('Contact'),
+      },
+    },
+    handler: (request, reply) => {
+      reply('{}');
+    },
+  })
 }
diff --git a/packages/nylas-api/routes/drafts.js b/packages/nylas-api/routes/drafts.js
new file mode 100644
index 000000000..63c2b88a2
--- /dev/null
+++ b/packages/nylas-api/routes/drafts.js
@@ -0,0 +1,27 @@
+const Joi = require('joi');
+
+// TODO: This is a placeholder
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/drafts',
+    config: {
+      description: 'Returns drafts.',
+      notes: 'Notes go here',
+      tags: ['drafts'],
+      validate: {
+        query: {
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          offset: Joi.number().integer().min(0).default(0),
+          view: Joi.string().valid('count'),
+        },
+      },
+      response: {
+        schema: Joi.array(),
+      },
+    },
+    handler: (request, reply) => {
+      reply('[]');
+    },
+  });
+}
diff --git a/packages/nylas-api/routes/events.js b/packages/nylas-api/routes/events.js
new file mode 100644
index 000000000..cf7149730
--- /dev/null
+++ b/packages/nylas-api/routes/events.js
@@ -0,0 +1,27 @@
+const Joi = require('joi');
+
+// TODO: This is a placeholder
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/events',
+    config: {
+      description: 'Returns events.',
+      notes: 'Notes go here',
+      tags: ['events'],
+      validate: {
+        query: {
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          offset: Joi.number().integer().min(0).default(0),
+          view: Joi.string().valid('count'),
+        },
+      },
+      response: {
+        schema: Joi.array(),
+      },
+    },
+    handler: (request, reply) => {
+      reply('[]');
+    },
+  });
+}
diff --git a/packages/nylas-api/routes/metadata.js b/packages/nylas-api/routes/metadata.js
new file mode 100644
index 000000000..67ea9ebd7
--- /dev/null
+++ b/packages/nylas-api/routes/metadata.js
@@ -0,0 +1,27 @@
+const Joi = require('joi');
+
+// TODO: This is a placeholder
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/metadata',
+    config: {
+      description: 'Returns metadata.',
+      notes: 'Notes go here',
+      tags: ['metadata'],
+      validate: {
+        query: {
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          offset: Joi.number().integer().min(0).default(0),
+          view: Joi.string().valid('count'),
+        },
+      },
+      response: {
+        schema: Joi.array(),
+      },
+    },
+    handler: (request, reply) => {
+      reply('[]');
+    },
+  });
+}

From e87e67cea35cdc8aa2d654bf850003c4fd0b2ff5 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Mon, 21 Nov 2016 14:00:32 -0800
Subject: [PATCH 324/800] Fix linter issues

---
 .eslintrc                                     | 44 +++++++++++++++++--
 package.json                                  | 14 +++---
 .../nylas-dashboard/public/js/dropdown.jsx    |  2 +-
 .../public/js/elapsed-time.jsx                |  2 +-
 .../public/js/mini-account.jsx                |  4 +-
 packages/nylas-dashboard/public/js/modal.jsx  |  4 +-
 .../public/js/set-all-sync-policies.jsx       |  2 +-
 .../nylas-dashboard/public/js/sync-graph.jsx  |  2 +-
 .../public/js/syncback-request-details.jsx    |  4 +-
 9 files changed, 57 insertions(+), 21 deletions(-)

diff --git a/.eslintrc b/.eslintrc
index d712529ce..f3d506040 100644
--- a/.eslintrc
+++ b/.eslintrc
@@ -2,7 +2,14 @@
   "parser": "babel-eslint",
   "extends": "airbnb",
   "globals": {
-    "NylasError": false,
+    "NylasEnv": false,
+    "$n": false,
+    "waitsForPromise": false,
+    "advanceClock": false,
+    "TEST_ACCOUNT_ID": false,
+    "TEST_ACCOUNT_NAME": false,
+    "TEST_ACCOUNT_EMAIL": false,
+    "TEST_ACCOUNT_ALIAS_EMAIL": false
   },
   "env": {
     "browser": true,
@@ -11,28 +18,57 @@
   },
   "rules": {
     "arrow-body-style": "off",
+    "arrow-parens": "off",
+    "class-methods-use-this": "off",
     "prefer-arrow-callback": ["error", {"allowNamedFunctions": true}],
     "eqeqeq": ["error", "smart"],
     "id-length": "off",
     "object-curly-spacing": "off",
     "max-len": "off",
     "new-cap": ["error", {"capIsNew": false}],
+    "newline-per-chained-call": "off",
+    "no-bitwise": "off",
+    "no-lonely-if": "off",
+    "no-console": "off",
+    "no-continue": "off",
     "no-constant-condition": "off",
     "no-loop-func": "off",
+    "no-plusplus": "off",
     "no-shadow": "error",
     "no-underscore-dangle": "off",
     "object-shorthand": "off",
     "quotes": "off",
-    "global-require": "off",
     "quote-props": ["error", "consistent-as-needed", { "keywords": true }],
     "no-param-reassign": ["error", { "props": false }],
     "semi": "off",
-    "import/no-unresolved": ["error", {"ignore": ["nylas-exports", "nylas-component-kit", "electron", "nylas-store", "react-dom/server", "nylas-observables", "windows-shortcuts", "moment-round", "chrono-node", "event-kit", "enzyme"]}],
+    "no-mixed-operators": "off",
+    "import/extensions": ["error", "never", { "json": "always" }],
+    "import/no-unresolved": ["error", {"ignore": ["nylas-exports", "nylas-component-kit", "electron", "nylas-store", "react-dom/server", "nylas-observables", "windows-shortcuts", "moment-round", "better-sqlite3", "chrono-node", "event-kit", "enzyme"]}],
+    "import/no-extraneous-dependencies": "off",
+    "import/newline-after-import": "off",
+    "import/prefer-default-export": "off",
     "react/no-multi-comp": "off",
+    "react/no-find-dom-node": "off",
+    "react/no-string-refs": "off",
+    "react/no-unused-prop-types": "off",
+    "react/forbid-prop-types": "off",
+    "jsx-a11y/no-static-element-interactions": "off",
     "react/prop-types": ["error", {"ignore": ["children"]}],
-    "react/sort-comp": "error"
+    "react/sort-comp": "error",
+    "no-restricted-syntax": [
+      "error", "ForInStatement", "LabeledStatement", "WithStatement"
+    ],
+    "comma-dangle": ["error", {
+      "arrays": "always-multiline",
+      "objects": "always-multiline",
+      "imports": "always-multiline",
+      "exports": "always-multiline",
+      "functions": "ignore"
+    }],
+    "no-useless-return": "off"
   },
   "settings": {
+    "import/core-modules": [ "nylas-exports", "nylas-component-kit", "electron", "nylas-store", "nylas-observables" ],
     "import/resolver": {"node": {"extensions": [".es6", ".jsx", ".coffee", ".json", ".cjsx", ".js"]}}
   }
 }
diff --git a/package.json b/package.json
index 522b53df2..9638289e6 100644
--- a/package.json
+++ b/package.json
@@ -22,13 +22,13 @@
     "utf7": "https://github.com/truebit/utf7/archive/1f753bac59b99d93b17a5ef11681e232465e2558.tar.gz"
   },
   "devDependencies": {
-    "babel-eslint": "6.x",
-    "eslint": "2.x",
-    "eslint-config-airbnb": "8.x",
-    "eslint-plugin-import": "1.x",
-    "eslint-plugin-jsx-a11y": "1.x",
-    "eslint-plugin-react": "5.x",
-    "eslint_d": "3.x",
+    "babel-eslint": "7.1.0",
+    "eslint": "3.10.1",
+    "eslint_d": "4.2.0",
+    "eslint-config-airbnb": "13.0.0",
+    "eslint-plugin-import": "2.2.0",
+    "eslint-plugin-jsx-a11y": "2.2.3",
+    "eslint-plugin-react": "6.7.1",
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz"
   },
   "scripts": {
diff --git a/packages/nylas-dashboard/public/js/dropdown.jsx b/packages/nylas-dashboard/public/js/dropdown.jsx
index 1c9e5dd9c..c09f78d64 100644
--- a/packages/nylas-dashboard/public/js/dropdown.jsx
+++ b/packages/nylas-dashboard/public/js/dropdown.jsx
@@ -37,7 +37,7 @@ class Dropdown extends React.Component {
     );
 
     // All options, not shown if dropdown is closed
-    let options = [];
+    const options = [];
     let optionsWrapper = ;
     if (!this.state.closed) {
       for (const opt of this.props.options) {
diff --git a/packages/nylas-dashboard/public/js/elapsed-time.jsx b/packages/nylas-dashboard/public/js/elapsed-time.jsx
index b94f8ad04..ecad76b01 100644
--- a/packages/nylas-dashboard/public/js/elapsed-time.jsx
+++ b/packages/nylas-dashboard/public/js/elapsed-time.jsx
@@ -28,7 +28,7 @@ class ElapsedTime extends React.Component {
   }
 
   render() {
-    return 
+    return 
   }
 }
 
diff --git a/packages/nylas-dashboard/public/js/mini-account.jsx b/packages/nylas-dashboard/public/js/mini-account.jsx
index cde2a8d74..aba4b4d22 100644
--- a/packages/nylas-dashboard/public/js/mini-account.jsx
+++ b/packages/nylas-dashboard/public/js/mini-account.jsx
@@ -16,7 +16,7 @@ class MiniAccount extends React.Component {
 
   render() {
     let errorClass;
-    let style = {};
+    const style = {};
     if (this.props.account.sync_error) {
       errorClass = 'errored';
     } else {
@@ -28,7 +28,7 @@ class MiniAccount extends React.Component {
       

+      />
     )
   }
 }
diff --git a/packages/nylas-dashboard/public/js/modal.jsx b/packages/nylas-dashboard/public/js/modal.jsx
index 1c5a3f032..384078f5d 100644
--- a/packages/nylas-dashboard/public/js/modal.jsx
+++ b/packages/nylas-dashboard/public/js/modal.jsx
@@ -5,8 +5,8 @@ class Modal extends React.Component {
     super(props);
     this.state = {
       open: false,
-      onOpen: props.onOpen || () => {},
-      onClose: props.onClose || () => {},
+      onOpen: props.onOpen || (() => {}),
+      onClose: props.onClose || (() => {}),
     }
   }
 
diff --git a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
index 64be249e0..f11cd00e6 100644
--- a/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
+++ b/packages/nylas-dashboard/public/js/set-all-sync-policies.jsx
@@ -46,7 +46,7 @@ class SetAllSyncPolicies extends React.Component {
         ]}
       >
         
Sync Policy

-        
+        
-          
-          
 this.cancel.call(this)}> Cancel 

-        

-
-      )
-    }
-    return (
-      

-        
Sync Policy

-        
{this.props.stringifiedSyncPolicy}

-        
 this.edit.call(this)}> Edit 

-      

-    )
-  }
-}
-
-SyncPolicy.propTypes = {
-  accountId: React.PropTypes.number,
-  stringifiedSyncPolicy: React.PropTypes.string,
-}
diff --git a/packages/local-sync/src/shared/hook-transaction-log.js b/packages/local-sync/src/shared/hook-transaction-log.js
index 19636b0c1..13b826d2e 100644
--- a/packages/local-sync/src/shared/hook-transaction-log.js
+++ b/packages/local-sync/src/shared/hook-transaction-log.js
@@ -2,6 +2,9 @@ const _ = require('underscore')
 const TransactionConnector = require('./transaction-connector')
 
 module.exports = (db, sequelize) => {
+  if (!db.Transaction) {
+    throw new Error("Cannot enable transaction logging, there is no Transaction model class in this database.")
+  }
   const isTransaction = ($modelOptions) => {
     return $modelOptions.name.singular === "transaction"
   }

From ac818f30fecf0257fe4042386dbc286655730890 Mon Sep 17 00:00:00 2001
From: Karim Hamidou 
Date: Tue, 29 Nov 2016 14:59:37 -0800
Subject: [PATCH 415/800] Add a shortcut to restart the sync for an account
 from scratch.

---
 .../local-sync/src/local-api/routes/auth.js   | 17 +++++++--------
 .../src/local-sync-dashboard/root.jsx         |  9 +++++++-
 .../local-sync-worker/sync-process-manager.js | 21 ++++++++++++-------
 .../src/shared/local-database-connector.js    |  7 ++++++-
 4 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/packages/local-sync/src/local-api/routes/auth.js b/packages/local-sync/src/local-api/routes/auth.js
index 609b97118..53d8d256d 100644
--- a/packages/local-sync/src/local-api/routes/auth.js
+++ b/packages/local-sync/src/local-api/routes/auth.js
@@ -55,16 +55,13 @@ const buildAccountWith = ({name, email, provider, settings, credentials}) => {
       account.setCredentials(credentials);
 
       return account.save().then((saved) =>
-        AccountToken.create({accountId: saved.id}).then((token) =>
-          LocalDatabaseConnector.ensureAccountDatabase(saved.id).then(() => {
-            SyncProcessManager.addWorkerForAccount(saved);
-
-            return Promise.resolve({
-              account: saved,
-              token: token,
-            });
-          })
-        )
+        AccountToken.create({accountId: saved.id}).then((token) => {
+          SyncProcessManager.addWorkerForAccount(saved);
+          return Promise.resolve({
+            account: saved,
+            token: token,
+          });
+        })
       );
     });
   });
diff --git a/packages/local-sync/src/local-sync-dashboard/root.jsx b/packages/local-sync/src/local-sync-dashboard/root.jsx
index c96d11a2c..0106d1657 100644
--- a/packages/local-sync/src/local-sync-dashboard/root.jsx
+++ b/packages/local-sync/src/local-sync-dashboard/root.jsx
@@ -45,7 +45,13 @@ class AccountCard extends React.Component {
           SyncProcessManager.wakeWorkerForAccount(account);
         });
       })
-    })
+    });
+  }
+
+  onResetSync = () => {
+    SyncProcessManager.removeWorkerForAccountId(this.props.account.id);
+    LocalDatabaseConnector.destroyAccountDatabase(this.props.account.id);
+    SyncProcessManager.addWorkerForAccount(this.props.account);
   }
 
   renderError() {
@@ -89,6 +95,7 @@ class AccountCard extends React.Component {
         style={{top: `${position.top}px`, left: `${position.left}px`}}
       >
         
{account.emailAddress} [{account.id}]

+        
         
         

           First Sync Duration (sec):
diff --git a/packages/local-sync/src/local-sync-worker/sync-process-manager.js b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
index 9e2346cbb..556ec62cb 100644
--- a/packages/local-sync/src/local-sync-worker/sync-process-manager.js
+++ b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
@@ -50,15 +50,18 @@ class SyncProcessManager {
   }
 
   addWorkerForAccount(account) {
-    return LocalDatabaseConnector.forAccount(account.id).then((db) => {
-      if (this._workers[account.id]) {
-        return Promise.reject(new Error("Local worker already exists"));
-      }
+    return LocalDatabaseConnector.ensureAccountDatabase(account.id)
+    .then(() => {
+      return LocalDatabaseConnector.forAccount(account.id).then((db) => {
+        if (this._workers[account.id]) {
+          return Promise.reject(new Error("Local worker already exists"));
+        }
 
-      this._workers[account.id] = new SyncWorker(account, db, () => {
-        this.removeWorkerForAccountId(account.id)
-      });
-      return Promise.resolve();
+        this._workers[account.id] = new SyncWorker(account, db, () => {
+          this.removeWorkerForAccountId(account.id)
+        });
+        return Promise.resolve();
+      })
     })
     .then(() => {
       this._logger.info({account_id: account.id}, `ProcessManager: Claiming Account Succeeded`)
@@ -69,6 +72,8 @@ class SyncProcessManager {
   }
 
   removeWorkerForAccountId(accountId) {
+    const worker = this._workers[accountId];
+    worker.cleanup();
     this._workers[accountId] = null;
   }
 
diff --git a/packages/local-sync/src/shared/local-database-connector.js b/packages/local-sync/src/shared/local-database-connector.js
index 8248aff06..7354f464f 100644
--- a/packages/local-sync/src/shared/local-database-connector.js
+++ b/packages/local-sync/src/shared/local-database-connector.js
@@ -60,7 +60,12 @@ class LocalDatabaseConnector {
 
   destroyAccountDatabase(accountId) {
     const dbname = `a-${accountId}`;
-    fs.removeFileSync(path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`));
+    fs.access(dbname, fs.F_OK, (err) => {
+      if (!err) {
+        fs.unlinkSync(path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`));
+      }
+    });
+
     return Promise.resolve()
   }
 

From 20be7cc513adfa67ee613a6a6f27ca1c3277374f Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Tue, 29 Nov 2016 15:04:31 -0800
Subject: [PATCH 416/800] [local-sync] properly save folders when detecting
 thread

---
 .../local-sync/src/new-message-processor/detect-thread.js     | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/local-sync/src/new-message-processor/detect-thread.js b/packages/local-sync/src/new-message-processor/detect-thread.js
index f34c24335..b98abeb16 100644
--- a/packages/local-sync/src/new-message-processor/detect-thread.js
+++ b/packages/local-sync/src/new-message-processor/detect-thread.js
@@ -145,9 +145,7 @@ function detectThread({db, message}) {
     if (!isSent && ((message.date > thread.lastMessageReceivedDate) || !thread.lastMessageReceivedDate)) {
       thread.lastMessageReceivedDate = message.date;
     }
-    if (!thread.folders.find(f => f.id === message.folderId)) {
-      thread.folders.push(message.folder)
-    }
+
     return thread.save()
     .then((saved) => {
       const promises = []

From 0879b6390e7446b598ae41b3d00729cafdc652cc Mon Sep 17 00:00:00 2001
From: Karim Hamidou 
Date: Tue, 29 Nov 2016 15:13:31 -0800
Subject: [PATCH 417/800] Make db reset synchronous.

---
 .../src/shared/local-database-connector.js           | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/local-sync/src/shared/local-database-connector.js b/packages/local-sync/src/shared/local-database-connector.js
index 7354f464f..a08c6d8c4 100644
--- a/packages/local-sync/src/shared/local-database-connector.js
+++ b/packages/local-sync/src/shared/local-database-connector.js
@@ -60,12 +60,14 @@ class LocalDatabaseConnector {
 
   destroyAccountDatabase(accountId) {
     const dbname = `a-${accountId}`;
-    fs.access(dbname, fs.F_OK, (err) => {
-      if (!err) {
-        fs.unlinkSync(path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`));
-      }
-    });
+    const dbpath = path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`);
 
+    const err = fs.accessSync(dbpath, fs.F_OK);
+    if (!err) {
+      fs.unlinkSync(dbpath);
+    }
+
+    delete this._cache[accountId];
     return Promise.resolve()
   }
 

From 0c79ebb86a5d1e5df2c55aade50932a40abd338f Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 29 Nov 2016 16:07:33 -0800
Subject: [PATCH 418/800] [local-sync] Synchronize K2 accounts with N1 accounts

---
 .../local-sync/src/local-sync-worker/app.js   | 41 +++++++++++----
 .../local-sync-worker/sync-process-manager.js | 52 ++++++++-----------
 .../src/local-sync-worker/sync-worker.js      | 15 +++---
 3 files changed, 60 insertions(+), 48 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/app.js b/packages/local-sync/src/local-sync-worker/app.js
index 140575e26..94d83cf9b 100644
--- a/packages/local-sync/src/local-sync-worker/app.js
+++ b/packages/local-sync/src/local-sync-worker/app.js
@@ -1,18 +1,37 @@
-const LocalDatabaseConnector = require('../shared/local-database-connector')
-const os = require('os')
-global.instanceId = os.hostname();
+const {AccountStore} = require('nylas-exports');
 
+const LocalDatabaseConnector = require('../shared/local-database-connector')
 const manager = require('./sync-process-manager')
 
-LocalDatabaseConnector.forShared().then((db) => {
-  const {Account} = db;
-  Account.findAll().then((accounts) => {
-    if (accounts.length === 0) {
-      global.Logger.info(`Couldn't find any accounts to sync. Run this CURL command to auth one!`)
-      global.Logger.info(`curl -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"mail.messagingengine.com","imap_port":993,"smtp_host":"mail.messagingengine.com","smtp_port":0,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"trar2e","imap_password":"trar2e","ssl_required":true}}' "http://localhost:2578/auth?client_id=123"`)
+// Right now, it's a bit confusing because N1 has Account objects, and K2 has
+// Account objects. We want to sync all K2 Accounts, but when an N1 Account is
+// deleted, we want to delete the K2 account too.
+
+async function ensureK2Consistency() {
+  const {Account} = await LocalDatabaseConnector.forShared();
+  const k2Accounts = await Account.findAll();
+  const n1Accounts = AccountStore.accounts();
+  const n1Emails = n1Accounts.map(a => a.emailAddress);
+
+  const deletions = [];
+  for (const k2Account of k2Accounts) {
+    const deleted = !n1Emails.includes(k2Account.emailAddress);
+    if (deleted) {
+      console.warn(`Deleting K2 account ID ${k2Account.id} which could not be matched to an N1 account.`)
+      manager.removeWorkerForAccountId(k2Account.id);
+      LocalDatabaseConnector.destroyAccountDatabase(k2Account.id);
+      deletions.push(k2Account.destroy());
     }
-    manager.start();
-  });
+  }
+  return await Promise.all(deletions)
+}
+
+ensureK2Consistency().then(() => {
+  // Step 1: Start all K2 Accounts
+  manager.start();
 });
 
+// Step 2: Watch N1 Accounts, ensure consistency when they change.
+AccountStore.listen(ensureK2Consistency);
+
 global.manager = manager;
diff --git a/packages/local-sync/src/local-sync-worker/sync-process-manager.js b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
index 556ec62cb..c9004ba9e 100644
--- a/packages/local-sync/src/local-sync-worker/sync-process-manager.js
+++ b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
@@ -1,9 +1,6 @@
 const SyncWorker = require('./sync-worker');
 const LocalDatabaseConnector = require('../shared/local-database-connector')
 
-const IDENTITY = `${global.instanceId}-${process.pid}`;
-
-
 /*
 Accounts ALWAYS exist in either `accounts:unclaimed` or an `accounts:{id}` list.
 They are atomically moved between these sets as they are claimed and returned.
@@ -31,50 +28,43 @@ class SyncProcessManager {
     this._workers = {};
     this._listenForSyncsClient = null;
     this._exiting = false;
-    this._logger = global.Logger.child({identity: IDENTITY});
+    this._logger = global.Logger.child();
   }
 
-  start() {
+  async start() {
     this._logger.info(`ProcessManager: Starting with ID`)
 
-    LocalDatabaseConnector.forShared().then(({Account}) =>
-      Account.findAll().then((accounts) => {
-        for (const account of accounts) {
-          this.addWorkerForAccount(account);
-        }
-      }));
+    const {Account} = await LocalDatabaseConnector.forShared();
+    const accounts = await Account.findAll();
+    for (const account of accounts) {
+      this.addWorkerForAccount(account);
+    }
   }
 
   wakeWorkerForAccount(account) {
     this._workers[account.id].syncNow();
   }
 
-  addWorkerForAccount(account) {
-    return LocalDatabaseConnector.ensureAccountDatabase(account.id)
-    .then(() => {
-      return LocalDatabaseConnector.forAccount(account.id).then((db) => {
-        if (this._workers[account.id]) {
-          return Promise.reject(new Error("Local worker already exists"));
-        }
+  async addWorkerForAccount(account) {
+    await LocalDatabaseConnector.ensureAccountDatabase(account.id);
 
-        this._workers[account.id] = new SyncWorker(account, db, () => {
-          this.removeWorkerForAccountId(account.id)
-        });
-        return Promise.resolve();
-      })
-    })
-    .then(() => {
+    try {
+      const db = await LocalDatabaseConnector.forAccount(account.id);
+      if (this._workers[account.id]) {
+        throw new Error("Local worker already exists");
+      }
+      this._workers[account.id] = new SyncWorker(account, db, this);
       this._logger.info({account_id: account.id}, `ProcessManager: Claiming Account Succeeded`)
-    })
-    .catch((err) => {
+    } catch (err) {
       this._logger.error({account_id: account.id, reason: err.message}, `ProcessManager: Claiming Account Failed`)
-    });
+    }
   }
 
   removeWorkerForAccountId(accountId) {
-    const worker = this._workers[accountId];
-    worker.cleanup();
-    this._workers[accountId] = null;
+    if (this._workers[accountId]) {
+      this._workers[accountId].cleanup();
+      this._workers[accountId] = null;
+    }
   }
 
 }
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index 698b910c9..4ff198933 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -14,13 +14,13 @@ const SyncbackTaskFactory = require('./syncback-task-factory')
 
 
 class SyncWorker {
-  constructor(account, db, onExpired) {
+  constructor(account, db, parentManager) {
     this._db = db;
+    this._manager = parentManager;
     this._conn = null;
     this._account = account;
     this._startTime = Date.now();
     this._lastSyncTime = null;
-    this._onExpired = onExpired;
     this._logger = global.Logger.forAccount(account)
 
     this._destroyed = false;
@@ -153,10 +153,13 @@ class SyncWorker {
       .then(() => this.syncMessagesInAllFolders())
       .then(() => this.onSyncDidComplete())
       .catch((error) => this.onSyncError(error))
-    })
-    .finally(() => {
-      this._lastSyncTime = Date.now()
-      this.scheduleNextSync()
+      .finally(() => {
+        this._lastSyncTime = Date.now()
+        this.scheduleNextSync()
+      })
+    }).catch((err) => {
+      this._logger.error({err}, `SyncWorker: Account could not be loaded. Sync worker will exit.`)
+      this._manager.removeWorkerForAccountId(this._account.id);
     })
   }
 

From fb9d770cc65a99a13976fb464f9da562c014359a Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 29 Nov 2016 16:20:18 -0800
Subject: [PATCH 419/800] refactor(sync-worker) async/await for Karim

---
 .../src/local-sync-worker/sync-worker.js      | 150 +++++++++---------
 1 file changed, 76 insertions(+), 74 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index 4ff198933..16f2a4896 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -46,28 +46,27 @@ class SyncWorker {
     this.syncNow({reason: 'IMAP IDLE Fired'});
   }
 
-  _getAccount() {
-    return LocalDatabaseConnector.forShared().then(({Account}) =>
-      Account.find({where: {id: this._account.id}})
-    );
+  async _getAccount() {
+    const {Account} = await LocalDatabaseConnector.forShared()
+    Account.find({where: {id: this._account.id}})
   }
 
   _getIdleFolder() {
     return this._db.Folder.find({where: {role: ['all', 'inbox']}})
   }
 
-  ensureConnection() {
+  async ensureConnection() {
     if (this._conn) {
-      return this._conn.connect();
+      return await this._conn.connect();
     }
     const settings = this._account.connectionSettings;
     const credentials = this._account.decryptedCredentials();
 
     if (!settings || !settings.imap_host) {
-      return Promise.reject(new Error("ensureConnection: There are no IMAP connection settings for this account."))
+      throw new Error("ensureConnection: There are no IMAP connection settings for this account.");
     }
     if (!credentials) {
-      return Promise.reject(new Error("ensureConnection: There are no IMAP connection credentials for this account."))
+      throw new Error("ensureConnection: There are no IMAP connection credentials for this account.");
     }
 
     const conn = new IMAPConnection({
@@ -86,51 +85,50 @@ class SyncWorker {
     });
 
     this._conn = conn;
-    return this._conn.connect();
+    return await this._conn.connect();
   }
 
-  syncbackMessageActions() {
+  async syncbackMessageActions() {
     const {SyncbackRequest} = this._db;
     const where = {where: {status: "NEW"}, limit: 100};
 
-    const tasks = SyncbackRequest.findAll(where).map((req) =>
+    const tasks = (await SyncbackRequest.findAll(where)).map((req) =>
       SyncbackTaskFactory.create(this._account, req)
     );
 
     return PromiseUtils.each(tasks, this.runSyncbackTask.bind(this));
   }
 
-  runSyncbackTask(task) {
-    const syncbackRequest = task.syncbackRequestObject()
-    return this._conn.runOperation(task)
-    .then(() => {
-      syncbackRequest.status = "SUCCEEDED"
-    })
-    .catch((error) => {
-      syncbackRequest.error = error
-      syncbackRequest.status = "FAILED"
-      this._logger.error(syncbackRequest.toJSON(), `${task.description()} failed`)
-    })
-    .finally(() => syncbackRequest.save())
+  async runSyncbackTask(task) {
+    const syncbackRequest = task.syncbackRequestObject();
+    try {
+      await this._conn.runOperation(task);
+      syncbackRequest.status = "SUCCEEDED";
+    } catch (error) {
+      syncbackRequest.error = error;
+      syncbackRequest.status = "FAILED";
+      this._logger.error(syncbackRequest.toJSON(), `${task.description()} failed`);
+    } finally {
+      await syncbackRequest.save();
+    }
   }
 
-  syncMessagesInAllFolders() {
+  async syncMessagesInAllFolders() {
     const {Folder} = this._db;
     const {folderSyncOptions} = this._account.syncPolicy;
 
-    return Folder.findAll().then((folders) => {
-      const priority = ['inbox', 'all', 'drafts', 'sent', 'spam', 'trash'].reverse();
-      const foldersSorted = folders.sort((a, b) =>
-        (priority.indexOf(a.role) - priority.indexOf(b.role)) * -1
-      )
+    const folders = await Folder.findAll();
+    const priority = ['inbox', 'all', 'drafts', 'sent', 'spam', 'trash'].reverse();
+    const foldersSorted = folders.sort((a, b) =>
+      (priority.indexOf(a.role) - priority.indexOf(b.role)) * -1
+    )
 
-      return Promise.all(foldersSorted.map((cat) =>
-        this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions, this._logger))
-      ))
-    });
+    return await Promise.all(foldersSorted.map((cat) =>
+      this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions, this._logger))
+    ))
   }
 
-  syncNow({reason} = {}) {
+  async syncNow({reason} = {}) {
     const syncInProgress = (this._syncTimer === null);
     if (syncInProgress) {
       return;
@@ -139,28 +137,33 @@ class SyncWorker {
     clearTimeout(this._syncTimer);
     this._syncTimer = null;
 
-    this._account.reload().then(() => {
-      console.log(this._account)
-      if (this._account.errored()) {
-        this._logger.error(`SyncWorker: Account is in error state - Retrying sync\n${this._account.syncError.message}`, this._account.syncError.stack.join('\n'))
-      }
-      this._logger.info({reason}, `SyncWorker: Account sync started`)
-
-      return this._account.update({syncError: null})
-      .then(() => this.ensureConnection())
-      .then(() => this.syncbackMessageActions())
-      .then(() => this._conn.runOperation(new FetchFolderList(this._account.provider, this._logger)))
-      .then(() => this.syncMessagesInAllFolders())
-      .then(() => this.onSyncDidComplete())
-      .catch((error) => this.onSyncError(error))
-      .finally(() => {
-        this._lastSyncTime = Date.now()
-        this.scheduleNextSync()
-      })
-    }).catch((err) => {
+    try {
+      await this._account.reload();
+    } catch (err) {
       this._logger.error({err}, `SyncWorker: Account could not be loaded. Sync worker will exit.`)
       this._manager.removeWorkerForAccountId(this._account.id);
-    })
+      return;
+    }
+
+    console.log(this._account)
+    if (this._account.errored()) {
+      this._logger.error(`SyncWorker: Account is in error state - Retrying sync\n${this._account.syncError.message}`, this._account.syncError.stack.join('\n'))
+    }
+    this._logger.info({reason}, `SyncWorker: Account sync started`)
+
+    try {
+      await this._account.update({syncError: null});
+      await this.ensureConnection();
+      await this.syncbackMessageActions();
+      await this._conn.runOperation(new FetchFolderList(this._account.provider, this._logger));
+      await this.syncMessagesInAllFolders();
+      await this.onSyncDidComplete();
+    } catch (error) {
+      this.onSyncError(error);
+    } finally {
+      this._lastSyncTime = Date.now()
+      this.scheduleNextSync()
+    }
   }
 
   onSyncError(error) {
@@ -177,7 +180,7 @@ class SyncWorker {
     return this._account.save()
   }
 
-  onSyncDidComplete() {
+  async onSyncDidComplete() {
     const now = Date.now();
 
     // Save metrics to the account object
@@ -186,39 +189,38 @@ class SyncWorker {
     }
 
     const syncGraphTimeLength = 60 * 30; // 30 minutes, should be the same as SyncGraph.config.timeLength
-    let lastSyncCompletions = [].concat(this._account.lastSyncCompletions)
-    lastSyncCompletions = [now, ...lastSyncCompletions]
+    let lastSyncCompletions = [].concat(this._account.lastSyncCompletions);
+    lastSyncCompletions = [now, ...lastSyncCompletions];
     while (now - lastSyncCompletions[lastSyncCompletions.length - 1] > 1000 * syncGraphTimeLength) {
       lastSyncCompletions.pop();
     }
 
-    this._logger.info('Syncworker: Completed sync cycle')
-    this._account.lastSyncCompletions = lastSyncCompletions
-    this._account.save()
+    this._logger.info('Syncworker: Completed sync cycle');
+    this._account.lastSyncCompletions = lastSyncCompletions;
+    this._account.save();
 
     // Start idling on the inbox
-    return this._getIdleFolder()
-    .then((idleFolder) => this._conn.openBox(idleFolder.name))
-    .then(() => this._logger.info('SyncWorker: Idling on inbox folder'))
+    const idleFolder = await this._getIdleFolder();
+    await this._conn.openBox(idleFolder.name);
+    this._logger.info('SyncWorker: Idling on inbox folder');
   }
 
-  scheduleNextSync() {
+  async scheduleNextSync() {
     const {intervals} = this._account.syncPolicy;
     const {Folder} = this._db;
 
-    return Folder.findAll().then((folders) => {
-      const moreToSync = folders.some((f) =>
-        f.syncState.fetchedmax < f.syncState.uidnext || f.syncState.fetchedmin > 1
-      )
+    const folders = await Folder.findAll();
+    const moreToSync = folders.some((f) =>
+      f.syncState.fetchedmax < f.syncState.uidnext || f.syncState.fetchedmin > 1
+    )
 
-      const target = this._lastSyncTime + (moreToSync ? 1 : intervals.active);
+    const target = this._lastSyncTime + (moreToSync ? 1 : intervals.active);
 
-      this._logger.info(`SyncWorker: Scheduling next sync iteration for ${target - Date.now()}ms}`)
+    this._logger.info(`SyncWorker: Scheduling next sync iteration for ${target - Date.now()}ms}`)
 
-      this._syncTimer = setTimeout(() => {
-        this.syncNow({reason: 'Scheduled'});
-      }, target - Date.now());
-    });
+    this._syncTimer = setTimeout(() => {
+      this.syncNow({reason: 'Scheduled'});
+    }, target - Date.now());
   }
 }
 

From 82f505b6595f7cda9d244859ad7a4b79b5f0356c Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 29 Nov 2016 17:04:01 -0800
Subject: [PATCH 420/800] refactor(sync-worker): Convert IMAP operations to
 async/await

---
 packages/local-sync/src/local-api/app.js      |   1 +
 .../imap/fetch-folder-list.js                 |  58 ++-
 .../imap/fetch-messages-in-folder.js          | 340 +++++++++---------
 3 files changed, 191 insertions(+), 208 deletions(-)

diff --git a/packages/local-sync/src/local-api/app.js b/packages/local-sync/src/local-api/app.js
index 2e7bb0929..f3d4c9729 100644
--- a/packages/local-sync/src/local-api/app.js
+++ b/packages/local-sync/src/local-api/app.js
@@ -1,3 +1,4 @@
+/* eslint import/no-dynamic-require: 0 */
 /* eslint global-require: 0 */
 const Hapi = require('hapi');
 const HapiSwagger = require('hapi-swagger');
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index f9e37c575..0b95dadf2 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -108,42 +108,40 @@ class FetchFolderList {
     return {next, created, deleted};
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     this._db = db;
 
-    return imap.getBoxes().then((boxes) => {
-      const {Folder, Label, sequelize} = this._db;
+    const boxes = await imap.getBoxes();
+    const {Folder, Label, sequelize} = this._db;
 
-      return sequelize.transaction((transaction) => {
-        return PromiseUtils.props({
-          folders: Folder.findAll({transaction}),
-          labels: Label.findAll({transaction}),
-        }).then(({folders, labels}) => {
-          const all = [].concat(folders, labels);
-          const {next, created, deleted} = this._updateCategoriesWithBoxes(all, boxes);
+    return sequelize.transaction(async (transaction) => {
+      const {folders, labels} = await PromiseUtils.props({
+        folders: Folder.findAll({transaction}),
+        labels: Label.findAll({transaction}),
+      })
+      const all = [].concat(folders, labels);
+      const {next, created, deleted} = this._updateCategoriesWithBoxes(all, boxes);
 
-          const categoriesByRoles = next.reduce((obj, cat) => {
-            const role = this._roleByName(cat.name);
-            if (role in obj) {
-              obj[role].push(cat);
-            } else {
-              obj[role] = [cat];
-            }
-            return obj;
-          }, {})
+      const categoriesByRoles = next.reduce((obj, cat) => {
+        const role = this._roleByName(cat.name);
+        if (role in obj) {
+          obj[role].push(cat);
+        } else {
+          obj[role] = [cat];
+        }
+        return obj;
+      }, {})
 
-          this._getMissingRoles(next).forEach((role) => {
-            if (categoriesByRoles[role] && categoriesByRoles[role].length === 1) {
-              categoriesByRoles[role][0].role = role;
-            }
-          })
+      this._getMissingRoles(next).forEach((role) => {
+        if (categoriesByRoles[role] && categoriesByRoles[role].length === 1) {
+          categoriesByRoles[role][0].role = role;
+        }
+      })
 
-          let promises = [Promise.resolve()]
-          promises = promises.concat(created.map(cat => cat.save({transaction})))
-          promises = promises.concat(deleted.map(cat => cat.destroy({transaction})))
-          return Promise.all(promises)
-        });
-      });
+      let promises = [Promise.resolve()]
+      promises = promises.concat(created.map(cat => cat.save({transaction})))
+      promises = promises.concat(deleted.map(cat => cat.destroy({transaction})))
+      return Promise.all(promises)
     });
   }
 }
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 88b3693b4..1580eadee 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -35,13 +35,13 @@ class FetchMessagesInFolder {
     return count ? Math.max(1, this._box.uidnext - count) : 1;
   }
 
-  _recoverFromUIDInvalidity() {
+  async _recoverFromUIDInvalidity() {
     // UID invalidity means the server has asked us to delete all the UIDs for
     // this folder and start from scratch. Instead of deleting all the messages,
     // we just remove the category ID and UID. We may re-assign the same message
     // the same UID. Otherwise they're eventually garbage collected.
     const {Message} = this._db;
-    return this._db.sequelize.transaction((transaction) =>
+    await this._db.sequelize.transaction((transaction) =>
       Message.update({
         folderImapUID: null,
         folderId: null,
@@ -54,7 +54,7 @@ class FetchMessagesInFolder {
     )
   }
 
-  _updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) {
+  async _updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) {
     const {sequelize, Label} = this._db;
 
     const messageAttributesMap = {};
@@ -65,57 +65,59 @@ class FetchMessagesInFolder {
     const createdUIDs = [];
     const flagChangeMessages = [];
 
-    return Label.findAll().then((preloadedLabels) => {
-      Object.keys(remoteUIDAttributes).forEach((uid) => {
-        const msg = messageAttributesMap[uid];
-        const attrs = remoteUIDAttributes[uid];
+    const preloadedLabels = await Label.findAll();
+    Object.keys(remoteUIDAttributes).forEach(async (uid) => {
+      const msg = messageAttributesMap[uid];
+      const attrs = remoteUIDAttributes[uid];
 
-        if (!msg) {
-          createdUIDs.push(uid);
-          return;
+      if (!msg) {
+        createdUIDs.push(uid);
+        return;
+      }
+
+      const unread = !attrs.flags.includes('\\Seen');
+      const starred = attrs.flags.includes('\\Flagged');
+      const xGmLabels = attrs['x-gm-labels'];
+      const xGmLabelsJSON = xGmLabels ? JSON.stringify(xGmLabels) : null;
+
+      if (msg.folderImapXGMLabels !== xGmLabelsJSON) {
+        await msg.setLabelsFromXGM(xGmLabels, {Label, preloadedLabels})
+        const thread = await msg.getThread();
+        if (thread) {
+          thread.updateLabels();
         }
+      }
 
-        const unread = !attrs.flags.includes('\\Seen');
-        const starred = attrs.flags.includes('\\Flagged');
-        const xGmLabels = attrs['x-gm-labels'];
-        const xGmLabelsJSON = xGmLabels ? JSON.stringify(xGmLabels) : null;
+      if (msg.unread !== unread || msg.starred !== starred) {
+        msg.unread = unread;
+        msg.starred = starred;
+        flagChangeMessages.push(msg);
+      }
+    })
 
-        if (msg.folderImapXGMLabels !== xGmLabelsJSON) {
-          msg.setLabelsFromXGM(xGmLabels, {Label, preloadedLabels})
-          .then(() => msg.getThread())
-          .then((thread) => (thread ? thread.updateLabels() : null))
-        }
-
-        if (msg.unread !== unread || msg.starred !== starred) {
-          msg.unread = unread;
-          msg.starred = starred;
-          flagChangeMessages.push(msg);
-        }
-      })
+    this._logger.info({
+      flag_changes: flagChangeMessages.length,
+    }, `FetchMessagesInFolder: found flag changes`);
 
+    if (createdUIDs.length > 0) {
       this._logger.info({
-        flag_changes: flagChangeMessages.length,
-      }, `FetchMessagesInFolder: found flag changes`)
-      if (createdUIDs.length > 0) {
-        this._logger.info({
-          new_messages: createdUIDs.length,
-        }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`)
-      }
+        new_messages: createdUIDs.length,
+      }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`);
+    }
 
-      if (flagChangeMessages.length === 0) {
-        return Promise.resolve();
-      }
+    if (flagChangeMessages.length === 0) {
+      return;
+    }
 
-      return sequelize.transaction((transaction) =>
-        Promise.all(flagChangeMessages.map(m => m.save({
-          fields: MessageFlagAttributes,
-          transaction,
-        })))
-      );
-    });
+    await sequelize.transaction((transaction) =>
+      Promise.all(flagChangeMessages.map(m => m.save({
+        fields: MessageFlagAttributes,
+        transaction,
+      })))
+    );
   }
 
-  _removeDeletedMessages(remoteUIDAttributes, localMessageAttributes) {
+  async _removeDeletedMessages(remoteUIDAttributes, localMessageAttributes) {
     const {Message} = this._db;
 
     const removedUIDs = localMessageAttributes
@@ -127,9 +129,10 @@ class FetchMessagesInFolder {
     }, `FetchMessagesInFolder: found messages no longer in the folder`)
 
     if (removedUIDs.length === 0) {
-      return Promise.resolve();
+      return;
     }
-    return this._db.sequelize.transaction((transaction) =>
+
+    await this._db.sequelize.transaction((transaction) =>
        Message.update({
          folderImapUID: null,
          folderId: null,
@@ -173,10 +176,10 @@ class FetchMessagesInFolder {
     return desired;
   }
 
-  _fetchMessagesAndQueueForProcessing(range) {
+  async _fetchMessagesAndQueueForProcessing(range) {
     const uidsByPart = {};
 
-    return this._box.fetchEach(range, {struct: true}, ({attributes}) => {
+    await this._box.fetchEach(range, {struct: true}, ({attributes}) => {
       const desiredParts = this._getDesiredMIMEParts(attributes.struct);
       if (desiredParts.length === 0) {
         return;
@@ -185,95 +188,88 @@ class FetchMessagesInFolder {
       uidsByPart[key] = uidsByPart[key] || [];
       uidsByPart[key].push(attributes.uid);
     })
-    .then(() => {
-      return PromiseUtils.each(Object.keys(uidsByPart), (key) => {
-        const uids = uidsByPart[key];
-        const desiredParts = JSON.parse(key);
-        const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
 
-        this._logger.info({
-          key,
-          num_messages: uids.length,
-        }, `FetchMessagesInFolder: Fetching parts for messages`)
+    await PromiseUtils.each(Object.keys(uidsByPart), (key) => {
+      const uids = uidsByPart[key];
+      const desiredParts = JSON.parse(key);
+      const bodies = ['HEADER'].concat(desiredParts.map(p => p.id));
 
-        // note: the order of UIDs in the array doesn't matter, Gmail always
-        // returns them in ascending (oldest => newest) order.
+      this._logger.info({
+        key,
+        num_messages: uids.length,
+      }, `FetchMessagesInFolder: Fetching parts for messages`)
 
-        return this._box.fetchEach(
-          uids,
-          {bodies, struct: true},
-          (imapMessage) => this._processMessage(imapMessage, desiredParts)
-        );
-      });
+      // note: the order of UIDs in the array doesn't matter, Gmail always
+      // returns them in ascending (oldest => newest) order.
+
+      return this._box.fetchEach(
+        uids,
+        {bodies, struct: true},
+        (imapMessage) => this._processMessage(imapMessage, desiredParts)
+      );
     });
   }
 
-  _processMessage(imapMessage, desiredParts) {
+  async _processMessage(imapMessage, desiredParts) {
     const {Message} = this._db
 
-    MessageFactory.parseFromImap(imapMessage, desiredParts, {
-      db: this._db,
-      accountId: this._db.accountId,
-      folderId: this._category.id,
-    })
-    .then((messageValues) => {
-      Message.find({where: {hash: messageValues.hash}})
-      .then((existingMessage) => {
-        if (existingMessage) {
-          existingMessage.update(messageValues)
-          .then(() => existingMessage.getThread())
-          .then((thread) => {
-            if (!thread) { return Promise.resolve() }
-            return (
-              thread.updateFolders()
-              .then(() => thread.updateLabels())
-            )
-          })
-          .then(() => {
-            this._logger.info({
-              message_id: existingMessage.id,
-              uid: existingMessage.folderImapUID,
-            }, `FetchMessagesInFolder: Updated message`)
-          })
-        } else {
-          processNewMessage(messageValues, imapMessage)
-          this._logger.info({
-            message: messageValues,
-          }, `FetchMessagesInFolder: Queued new message for processing`)
+    try {
+      const messageValues = await MessageFactory.parseFromImap(imapMessage, desiredParts, {
+        db: this._db,
+        accountId: this._db.accountId,
+        folderId: this._category.id,
+      });
+      const existingMessage = await Message.find({where: {hash: messageValues.hash}});
+
+      if (existingMessage) {
+        await existingMessage.update(messageValues)
+        const thread = await existingMessage.getThread();
+        if (thread) {
+          await thread.updateFolders();
+          await thread.updateLabels();
         }
-      })
-    })
-    .catch((err) => {
+        this._logger.info({
+          message_id: existingMessage.id,
+          uid: existingMessage.folderImapUID,
+        }, `FetchMessagesInFolder: Updated message`)
+      } else {
+        processNewMessage(messageValues, imapMessage)
+        this._logger.info({
+          message: messageValues,
+        }, `FetchMessagesInFolder: Queued new message for processing`)
+      }
+    } catch (err) {
       this._logger.error({
         imapMessage,
         desiredParts,
         underlying: err.toString(),
       }, `FetchMessagesInFolder: Could not build message`)
-    })
+    }
   }
 
-  _openMailboxAndEnsureValidity() {
-    return this._imap.openBox(this._category.name).then((box) => {
-      if (box.persistentUIDs === false) {
-        return Promise.reject(new Error("Mailbox does not support persistentUIDs."))
-      }
+  async _openMailboxAndEnsureValidity() {
+    const box = this._imap.openBox(this._category.name);
 
-      const lastUIDValidity = this._category.syncState.uidvalidity;
+    if (box.persistentUIDs === false) {
+      throw new Error("Mailbox does not support persistentUIDs.");
+    }
 
-      if (lastUIDValidity && (box.uidvalidity !== lastUIDValidity)) {
-        this._logger.info({
-          boxname: box.name,
-          categoryname: this._category.name,
-          remoteuidvalidity: box.uidvalidity,
-          localuidvalidity: lastUIDValidity,
-        }, `FetchMessagesInFolder: Recovering from UIDInvalidity`);
-        return this._recoverFromUIDInvalidity().thenReturn(box)
-      }
-      return Promise.resolve(box);
-    });
+    const lastUIDValidity = this._category.syncState.uidvalidity;
+
+    if (lastUIDValidity && (box.uidvalidity !== lastUIDValidity)) {
+      this._logger.info({
+        boxname: box.name,
+        categoryname: this._category.name,
+        remoteuidvalidity: box.uidvalidity,
+        localuidvalidity: lastUIDValidity,
+      }, `FetchMessagesInFolder: Recovering from UIDInvalidity`);
+      await this._recoverFromUIDInvalidity()
+    }
+
+    return box;
   }
 
-  _fetchUnsyncedMessages() {
+  async _fetchUnsyncedMessages() {
     const savedSyncState = this._category.syncState;
     const isFirstSync = savedSyncState.fetchedmax === undefined;
     const boxUidnext = this._box.uidnext;
@@ -305,25 +301,23 @@ class FetchMessagesInFolder {
       }
     }
 
-    return PromiseUtils.each(desiredRanges, ({min, max}) => {
+    await PromiseUtils.each(desiredRanges, async ({min, max}) => {
       this._logger.info({
         range: `${min}:${max}`,
       }, `FetchMessagesInFolder: Fetching range`);
 
-      return this._fetchMessagesAndQueueForProcessing(`${min}:${max}`).then(() => {
-        const {fetchedmin, fetchedmax} = this._category.syncState;
-        return this.updateFolderSyncState({
-          fetchedmin: fetchedmin ? Math.min(fetchedmin, min) : min,
-          fetchedmax: fetchedmax ? Math.max(fetchedmax, max) : max,
-          uidnext: boxUidnext,
-          uidvalidity: boxUidvalidity,
-          timeFetchedUnseen: Date.now(),
-        });
-      })
-    })
-    .then(() => {
-      this._logger.info(`FetchMessagesInFolder: Fetching messages finished`);
+      await this._fetchMessagesAndQueueForProcessing(`${min}:${max}`);
+      const {fetchedmin, fetchedmax} = this._category.syncState;
+      return this.updateFolderSyncState({
+        fetchedmin: fetchedmin ? Math.min(fetchedmin, min) : min,
+        fetchedmax: fetchedmax ? Math.max(fetchedmax, max) : max,
+        uidnext: boxUidnext,
+        uidvalidity: boxUidvalidity,
+        timeFetchedUnseen: Date.now(),
+      });
     });
+
+    this._logger.info(`FetchMessagesInFolder: Fetching messages finished`);
   }
 
   _runScan() {
@@ -339,7 +333,7 @@ class FetchMessagesInFolder {
     return Date.now() - (timeDeepScan || 0) > this._options.deepFolderScan;
   }
 
-  _runShallowScan() {
+  async _runShallowScan() {
     const {highestmodseq} = this._category.syncState;
     const nextHighestmodseq = this._box.highestmodseq;
 
@@ -358,56 +352,49 @@ class FetchMessagesInFolder {
       shallowFetch = this._box.fetchUIDAttributes(range);
     }
 
-    return shallowFetch
-    .then((remoteUIDAttributes) => (
-      this._db.Message.findAll({
-        where: {folderId: this._category.id},
-        attributes: MessageFlagAttributes,
-      })
-      .then((localMessageAttributes) => (
-        this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes)
-      ))
-      .then(() => {
-        this._logger.info(`FetchMessagesInFolder: finished fetching changes to messages`);
-        return this.updateFolderSyncState({
-          highestmodseq: nextHighestmodseq,
-          timeShallowScan: Date.now(),
-        })
-      })
-    ))
+    const remoteUIDAttributes = await shallowFetch;
+    const localMessageAttributes = await this._db.Message.findAll({
+      where: {folderId: this._category.id},
+      attributes: MessageFlagAttributes,
+    })
+
+    await this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes)
+
+    this._logger.info(`FetchMessagesInFolder: finished fetching changes to messages`);
+    return this.updateFolderSyncState({
+      highestmodseq: nextHighestmodseq,
+      timeShallowScan: Date.now(),
+    });
   }
 
-  _runDeepScan() {
+  async _runDeepScan() {
     const {Message} = this._db;
     const {fetchedmin, fetchedmax} = this._category.syncState;
     const range = `${fetchedmin}:${fetchedmax}`;
 
     this._logger.info({range}, `FetchMessagesInFolder: Deep attribute scan: fetching attributes in range`)
 
-    return this._box.fetchUIDAttributes(range)
-    .then((remoteUIDAttributes) => {
-      return Message.findAll({
-        where: {folderId: this._category.id},
-        attributes: MessageFlagAttributes,
-      })
-      .then((localMessageAttributes) => (
-        PromiseUtils.props({
-          updates: this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes),
-          deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes),
-        })
-      ))
-      .then(() => {
-        this._logger.info(`FetchMessagesInFolder: Deep scan finished.`);
-        return this.updateFolderSyncState({
-          highestmodseq: this._box.highestmodseq,
-          timeDeepScan: Date.now(),
-          timeShallowScan: Date.now(),
-        })
-      })
+    const remoteUIDAttributes = await this._box.fetchUIDAttributes(range)
+    const localMessageAttributes = await Message.findAll({
+      where: {folderId: this._category.id},
+      attributes: MessageFlagAttributes,
+    })
+
+    await PromiseUtils.props({
+      updates: this._updateMessageAttributes(remoteUIDAttributes, localMessageAttributes),
+      deletes: this._removeDeletedMessages(remoteUIDAttributes, localMessageAttributes),
+    })
+
+    this._logger.info(`FetchMessagesInFolder: Deep scan finished.`);
+
+    return this.updateFolderSyncState({
+      highestmodseq: this._box.highestmodseq,
+      timeDeepScan: Date.now(),
+      timeShallowScan: Date.now(),
     });
   }
 
-  updateFolderSyncState(newState) {
+  async updateFolderSyncState(newState) {
     if (_.isMatch(this._category.syncState, newState)) {
       return Promise.resolve();
     }
@@ -415,16 +402,13 @@ class FetchMessagesInFolder {
     return this._category.save();
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     this._db = db;
     this._imap = imap;
 
-    return this._openMailboxAndEnsureValidity().then((box) => {
-      this._box = box
-      return this._fetchUnsyncedMessages().then(() =>
-        this._runScan()
-      )
-    })
+    this._box = await this._openMailboxAndEnsureValidity();
+    await this._fetchUnsyncedMessages()
+    await this._runScan()
   }
 }
 

From ece519553fc94e5036aa345e241d0a316671768c Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 29 Nov 2016 18:02:25 -0800
Subject: [PATCH 421/800] fix(sync): Issue with CONDSTORE args not passed to
 fetch

---
 packages/isomorphic-core/src/imap-box.js             |  4 ++--
 .../imap/fetch-messages-in-folder.js                 | 12 ++++++------
 .../local-sync/src/local-sync-worker/sync-worker.js  |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/packages/isomorphic-core/src/imap-box.js b/packages/isomorphic-core/src/imap-box.js
index c0f08662f..ab91f5be4 100644
--- a/packages/isomorphic-core/src/imap-box.js
+++ b/packages/isomorphic-core/src/imap-box.js
@@ -124,10 +124,10 @@ class IMAPBox {
    * @return {Promise} that resolves to a map of uid -> attributes for every
    * message in the range
    */
-  fetchUIDAttributes(range) {
+  fetchUIDAttributes(range, fetchOptions = {}) {
     return this._conn.createConnectionPromise((resolve, reject) => {
       const attributesByUID = {};
-      const f = this._conn._imap.fetch(range, {});
+      const f = this._conn._imap.fetch(range, fetchOptions);
       f.on('message', (msg) => {
         msg.on('attributes', (attrs) => {
           attributesByUID[attrs.uid] = attrs;
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 1580eadee..fa7f59fb8 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -18,7 +18,7 @@ class FetchMessagesInFolder {
     this._db = null
     this._category = category;
     this._options = options;
-    this._logger = logger;
+    this._logger = logger.child({category_name: this._category.name});
     if (!this._logger) {
       throw new Error("FetchMessagesInFolder requires a logger")
     }
@@ -124,14 +124,14 @@ class FetchMessagesInFolder {
     .filter(msg => !remoteUIDAttributes[msg.folderImapUID])
     .map(msg => msg.folderImapUID)
 
-    this._logger.info({
-      removed_messages: removedUIDs.length,
-    }, `FetchMessagesInFolder: found messages no longer in the folder`)
-
     if (removedUIDs.length === 0) {
       return;
     }
 
+    this._logger.info({
+      removed_messages: removedUIDs.length,
+    }, `FetchMessagesInFolder: found messages no longer in the folder`)
+
     await this._db.sequelize.transaction((transaction) =>
        Message.update({
          folderImapUID: null,
@@ -248,7 +248,7 @@ class FetchMessagesInFolder {
   }
 
   async _openMailboxAndEnsureValidity() {
-    const box = this._imap.openBox(this._category.name);
+    const box = await this._imap.openBox(this._category.name);
 
     if (box.persistentUIDs === false) {
       throw new Error("Mailbox does not support persistentUIDs.");
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index 16f2a4896..43a455681 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -216,7 +216,7 @@ class SyncWorker {
 
     const target = this._lastSyncTime + (moreToSync ? 1 : intervals.active);
 
-    this._logger.info(`SyncWorker: Scheduling next sync iteration for ${target - Date.now()}ms}`)
+    this._logger.info(`SyncWorker: Scheduling next sync iteration for ${target - Date.now()}ms`)
 
     this._syncTimer = setTimeout(() => {
       this.syncNow({reason: 'Scheduled'});

From a37b90a209f6b78d8d44ab081abef7eb9909c105 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 30 Nov 2016 10:17:47 -0800
Subject: [PATCH 422/800] [local-sync] Remove unecessary db queries in
 detect-thread

---
 .../new-message-processor/detect-thread.js    | 20 +++++--------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/packages/local-sync/src/new-message-processor/detect-thread.js b/packages/local-sync/src/new-message-processor/detect-thread.js
index b98abeb16..e5ed3b42c 100644
--- a/packages/local-sync/src/new-message-processor/detect-thread.js
+++ b/packages/local-sync/src/new-message-processor/detect-thread.js
@@ -1,4 +1,3 @@
-const {PromiseUtils} = require('isomorphic-core')
 // const _ = require('underscore');
 
 
@@ -79,8 +78,6 @@ function detectThread({db, message}) {
     throw new Error("Threading processMessage expects folder value to be present.");
   }
 
-
-  const {Folder, Label} = db;
   let findOrBuildThread = null;
   if (message.headers['x-gm-thrid']) {
     findOrBuildThread = findOrBuildByRemoteThreadId(db, message.headers['x-gm-thrid'])
@@ -88,12 +85,7 @@ function detectThread({db, message}) {
     findOrBuildThread = findOrBuildByMatching(db, message)
   }
 
-  return PromiseUtils.props({
-    thread: findOrBuildThread,
-    sentFolder: Folder.find({where: {role: 'sent'}}),
-    sentLabel: Label.find({where: {role: 'sent'}}),
-  })
-  .then(({thread, sentFolder, sentLabel}) => {
+  return findOrBuildThread.then((thread) => {
     if (!(thread.labels instanceof Array)) {
       throw new Error("Threading processMessage expects thread.labels to be an inflated array.");
     }
@@ -132,12 +124,10 @@ function detectThread({db, message}) {
       thread.firstMessageDate = message.date;
     }
 
-    let isSent = false;
-    if (sentFolder) {
-      isSent = message.folderId === sentFolder.id
-    } else if (sentLabel) {
-      isSent = !!message.labels.find(l => l.id === sentLabel.id)
-    }
+    const isSent = (
+      message.folder.role === 'sent' ||
+      !!message.labels.find(l => l.role === 'sent')
+    )
 
     if (isSent && ((message.date > thread.lastMessageSentDate) || !thread.lastMessageSentDate)) {
       thread.lastMessageSentDate = message.date;

From a36b1e1f2857989a6d0cc0886d00d47e4567527a Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 29 Nov 2016 14:24:07 -0500
Subject: [PATCH 423/800] [*] add delta endpoints and DRY deltaStreamBuilder

[cloud-core] add objectType to Metadata

[*] binding fixes to delta stream
---
 packages/isomorphic-core/index.js             |  1 +
 .../src/delta-stream-builder.js               | 84 ++++++++++++++++++
 .../local-sync/src/local-api/routes/delta.js  | 85 +++----------------
 3 files changed, 96 insertions(+), 74 deletions(-)
 create mode 100644 packages/isomorphic-core/src/delta-stream-builder.js

diff --git a/packages/isomorphic-core/index.js b/packages/isomorphic-core/index.js
index 0f0622802..0ec271702 100644
--- a/packages/isomorphic-core/index.js
+++ b/packages/isomorphic-core/index.js
@@ -9,4 +9,5 @@ module.exports = {
   PromiseUtils: require('./src/promise-utils'),
   DatabaseTypes: require('./src/database-types'),
   loadModels: require('./src/load-models'),
+  deltaStreamBuilder: require('./src/delta-stream-builder'),
 }
diff --git a/packages/isomorphic-core/src/delta-stream-builder.js b/packages/isomorphic-core/src/delta-stream-builder.js
new file mode 100644
index 000000000..cbef15c6e
--- /dev/null
+++ b/packages/isomorphic-core/src/delta-stream-builder.js
@@ -0,0 +1,84 @@
+const _ = require('underscore');
+const Rx = require('rx')
+const stream = require('stream');
+
+function keepAlive(request) {
+  const until = Rx.Observable.fromCallback(request.on)("disconnect")
+  return Rx.Observable.interval(1000).map(() => "\n").takeUntil(until)
+}
+
+function inflateTransactions(db, transactionModels = []) {
+  let models = transactionModels;
+  if (!(_.isArray(models))) { models = [transactionModels] }
+  const transactions = models.map((mod) => mod.toJSON())
+  transactions.forEach((t) => { t.cursor = t.id });
+  const byModel = _.groupBy(transactions, "object");
+  const byObjectIds = _.groupBy(transactions, "objectId");
+
+  return Promise.all(Object.keys(byModel).map((object) => {
+    const ids = _.pluck(byModel[object], "objectId");
+    const modelConstructorName = object.charAt(0).toUpperCase() + object.slice(1);
+    const ModelKlass = db[modelConstructorName]
+    let includes = [];
+    if (ModelKlass.requiredAssociationsForJSON) {
+      includes = ModelKlass.requiredAssociationsForJSON(db)
+    }
+    return ModelKlass.findAll({where: {id: ids}, include: includes})
+    .then((objs = []) => {
+      for (const model of objs) {
+        const tsForId = byObjectIds[model.id];
+        if (!tsForId || tsForId.length === 0) { continue; }
+        for (const t of tsForId) { t.attributes = model.toJSON(); }
+      }
+    })
+  })).then(() => `${transactions.map(JSON.stringify).join("\n")}\n`)
+}
+
+function createOutputStream() {
+  const outputStream = stream.Readable();
+  outputStream._read = () => { return };
+  outputStream.pushJSON = (msg) => {
+    const jsonMsg = typeof msg === 'string' ? msg : JSON.stringify(msg);
+    outputStream.push(jsonMsg);
+  }
+  return outputStream
+}
+
+function initialTransactions(db, request) {
+  const cursor = (request.query || {}).cursor;
+  const where = cursor ? {id: {$gt: cursor}} : {createdAt: {$gte: new Date()}}
+  return db.Transaction
+           .streamAll({where})
+           .flatMap((objs) => inflateTransactions(db, objs))
+}
+
+function inflatedIncomingTransaction(db, request, transactionSource) {
+  transactionSource.flatMap((t) => inflateTransactions(db, [t]))
+}
+
+module.exports = {
+  buildStream(request, dbSource, transactionSource) {
+    const outputStream = createOutputStream();
+
+    dbSource().then((db) => {
+      const source = Rx.Observable.merge(
+        inflatedIncomingTransaction(db, request, transactionSource(db, request)),
+        initialTransactions(db, request),
+        keepAlive(request)
+      ).subscribe(outputStream.pushJSON)
+
+      request.on("disconnect", source.dispose.bind(source));
+    });
+
+    return outputStream
+  },
+
+  lastTransactionReply(dbSource, reply) {
+    dbSource().then((db) => {
+      db.Transaction.findOne({order: [['id', 'DESC']]})
+      .then((t) => {
+        reply({cursor: (t || {}).id})
+      })
+    })
+  },
+}
diff --git a/packages/local-sync/src/local-api/routes/delta.js b/packages/local-sync/src/local-api/routes/delta.js
index a92cc0184..9c291bbc5 100644
--- a/packages/local-sync/src/local-api/routes/delta.js
+++ b/packages/local-sync/src/local-api/routes/delta.js
@@ -1,66 +1,13 @@
-const Rx = require('rx')
-const _ = require('underscore');
 const TransactionConnector = require('../../shared/transaction-connector')
+const {deltaStreamBuilder} = require('isomorphic-core')
 
-function keepAlive(request) {
-  const until = Rx.Observable.fromCallback(request.on)("disconnect")
-  return Rx.Observable.interval(1000).map(() => "\n").takeUntil(until)
-}
-
-function inflateTransactions(db, transactionModels = []) {
-  if (!(_.isArray(transactionModels))) {
-    transactionModels = [transactionModels]
-  }
-  const transactions = transactionModels.map((mod) => mod.toJSON())
-  transactions.forEach((t) => t.cursor = t.id);
-  const byModel = _.groupBy(transactions, "object");
-  const byObjectIds = _.groupBy(transactions, "objectId");
-
-  return Promise.all(Object.keys(byModel).map((object) => {
-    const ids = _.pluck(byModel[object], "objectId");
-    const modelConstructorName = object.charAt(0).toUpperCase() + object.slice(1);
-    const ModelKlass = db[modelConstructorName]
-    let includes = [];
-    if (ModelKlass.requiredAssociationsForJSON) {
-      includes = ModelKlass.requiredAssociationsForJSON(db)
-    }
-    return ModelKlass.findAll({where: {id: ids}, include: includes})
-    .then((models = []) => {
-      for (const model of models) {
-        const tsForId = byObjectIds[model.id];
-        if (!tsForId || tsForId.length === 0) { continue; }
-        for (const t of tsForId) { t.attributes = model.toJSON(); }
-      }
-    })
-  })).then(() => `${transactions.map(JSON.stringify).join("\n")}\n`)
-}
-
-function createOutputStream() {
-  const outputStream = require('stream').Readable();
-  outputStream._read = () => { return };
-  outputStream.pushJSON = (msg) => {
-    const jsonMsg = typeof msg === 'string' ? msg : JSON.stringify(msg);
-    outputStream.push(jsonMsg);
-  }
-  return outputStream
-}
-
-function lastTransaction(db) {
-  return db.Transaction.findOne({order: [['id', 'DESC']]})
-}
-
-function initialTransactions(db, request) {
-  const cursor = (request.query || {}).cursor;
-  const where = cursor ? {id: {$gt: cursor}} : {createdAt: {$gte: new Date()}}
-  return db.Transaction
-           .streamAll({where})
-           .flatMap((objs) => inflateTransactions(db, objs))
-}
-
-function inflatedDeltas(db, request) {
+function transactionSource(db, request) {
   const accountId = request.auth.credentials.id;
   return TransactionConnector.getObservableForAccountId(accountId)
-    .flatMap((transaction) => inflateTransactions(db, [transaction]))
+}
+
+function dbSource(request) {
+  return request.getAccountDatabase.bind(request)
 }
 
 module.exports = (server) => {
@@ -68,18 +15,8 @@ module.exports = (server) => {
     method: 'GET',
     path: '/delta/streaming',
     handler: (request, reply) => {
-      const outputStream = createOutputStream();
-
-      request.getAccountDatabase().then((db) => {
-        const source = Rx.Observable.merge(
-          inflatedDeltas(db, request),
-          initialTransactions(db, request),
-          keepAlive(request)
-        ).subscribe(outputStream.pushJSON)
-
-        request.on("disconnect", source.dispose.bind(source));
-      });
-
+      const outputStream = deltaStreamBuilder.buildStream(request,
+          dbSource(request), transactionSource)
       reply(outputStream)
     },
   });
@@ -87,8 +24,8 @@ module.exports = (server) => {
   server.route({
     method: 'POST',
     path: '/delta/latest_cursor',
-    handler: (request, reply) => request.getAccountDatabase().then((db) =>
-      lastTransaction(db).then((t) => reply({cursor: (t || {}).id}))
-    ),
+    handler: (request, reply) =>
+      deltaStreamBuilder.lastTransactionReply(dbSource(request), reply)
+    ,
   });
 };

From 1e01878e5a2cc162e21a907a2800a638865b390f Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 13:55:46 -0800
Subject: [PATCH 424/800] fix(sync): Fix attribute updates applying to threads

---
 .../imap/fetch-messages-in-folder.js          | 110 +++++++++++-------
 packages/local-sync/src/models/message.js     |   8 +-
 2 files changed, 71 insertions(+), 47 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index fa7f59fb8..63afc693c 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -12,23 +12,23 @@ const FETCH_MESSAGES_FIRST_COUNT = 100;
 const FETCH_MESSAGES_COUNT = 200;
 
 class FetchMessagesInFolder {
-  constructor(category, options, logger) {
+  constructor(folder, options, logger) {
     this._imap = null
     this._box = null
     this._db = null
-    this._category = category;
+    this._folder = folder;
     this._options = options;
-    this._logger = logger.child({category_name: this._category.name});
+    this._logger = logger.child({category_name: this._folder.name});
     if (!this._logger) {
       throw new Error("FetchMessagesInFolder requires a logger")
     }
-    if (!this._category) {
+    if (!this._folder) {
       throw new Error("FetchMessagesInFolder requires a category")
     }
   }
 
   description() {
-    return `FetchMessagesInFolder (${this._category.name} - ${this._category.id})\n  Options: ${JSON.stringify(this._options)}`;
+    return `FetchMessagesInFolder (${this._folder.name} - ${this._folder.id})\n  Options: ${JSON.stringify(this._options)}`;
   }
 
   _getLowerBoundUID(count) {
@@ -48,14 +48,14 @@ class FetchMessagesInFolder {
       }, {
         transaction: transaction,
         where: {
-          folderId: this._category.id,
+          folderId: this._folder.id,
         },
       })
     )
   }
 
   async _updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) {
-    const {sequelize, Label} = this._db;
+    const {sequelize, Label, Thread} = this._db;
 
     const messageAttributesMap = {};
     for (const msg of localMessageAttributes) {
@@ -63,10 +63,11 @@ class FetchMessagesInFolder {
     }
 
     const createdUIDs = [];
-    const flagChangeMessages = [];
+    const messagesWithChangedFlags = [];
+    const messagesWithChangedLabels = [];
 
     const preloadedLabels = await Label.findAll();
-    Object.keys(remoteUIDAttributes).forEach(async (uid) => {
+    await PromiseUtils.each(Object.keys(remoteUIDAttributes), async (uid) => {
       const msg = messageAttributesMap[uid];
       const attrs = remoteUIDAttributes[uid];
 
@@ -82,39 +83,64 @@ class FetchMessagesInFolder {
 
       if (msg.folderImapXGMLabels !== xGmLabelsJSON) {
         await msg.setLabelsFromXGM(xGmLabels, {Label, preloadedLabels})
-        const thread = await msg.getThread();
-        if (thread) {
-          thread.updateLabels();
-        }
+        messagesWithChangedLabels.push(msg);
       }
 
       if (msg.unread !== unread || msg.starred !== starred) {
         msg.unread = unread;
         msg.starred = starred;
-        flagChangeMessages.push(msg);
+        messagesWithChangedFlags.push(msg);
       }
     })
 
-    this._logger.info({
-      flag_changes: flagChangeMessages.length,
-    }, `FetchMessagesInFolder: found flag changes`);
-
     if (createdUIDs.length > 0) {
       this._logger.info({
         new_messages: createdUIDs.length,
       }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`);
     }
 
-    if (flagChangeMessages.length === 0) {
-      return;
+    if (messagesWithChangedFlags.length > 0) {
+      this._logger.info({
+        impacted_messages: messagesWithChangedFlags.length,
+      }, `FetchMessagesInFolder: Saving flag changes`);
+
+      // Update counters on the associated threads
+      const threadIds = messagesWithChangedFlags.map(m => m.threadId);
+      const threads = await Thread.findAll({where: {id: threadIds}});
+      const threadsById = {};
+      for (const thread of threads) {
+        threadsById[thread.id] = thread;
+      }
+      for (const msg of messagesWithChangedFlags) {
+        // unread = true, previous = false? Add 1 to unreadCount.
+        // unread = false, previous = true? Add -1 to unreadCount.
+        threadsById[msg.threadId].unreadCount += msg.unread / 1 - msg.previous('unread') / 1;
+        threadsById[msg.threadId].starredCount += msg.starred / 1 - msg.previous('starred') / 1;
+      }
+
+      // Save modified messages
+      await sequelize.transaction(async (transaction) => {
+        await Promise.all(messagesWithChangedFlags.map(m =>
+          m.save({ fields: MessageFlagAttributes, transaction })
+        ))
+        await Promise.all(threads.map(t =>
+          t.save({ fields: ['starredCount', 'unreadCount'], transaction })
+        ))
+      });
     }
 
-    await sequelize.transaction((transaction) =>
-      Promise.all(flagChangeMessages.map(m => m.save({
-        fields: MessageFlagAttributes,
-        transaction,
-      })))
-    );
+    if (messagesWithChangedLabels.length > 0) {
+      this._logger.info({
+        impacted_messages: messagesWithChangedFlags.length,
+      }, `FetchMessagesInFolder: Saving label changes`);
+
+      // Propagate label changes to threads. Important that we do this after
+      // processing all the messages, since msgs in the same thread often change
+      // at the same time.
+      const threadIds = messagesWithChangedLabels.map(m => m.threadId);
+      const threads = await Thread.findAll({where: {id: threadIds}});
+      threads.forEach((thread) => thread.updateLabels());
+    }
   }
 
   async _removeDeletedMessages(remoteUIDAttributes, localMessageAttributes) {
@@ -139,7 +165,7 @@ class FetchMessagesInFolder {
        }, {
          transaction,
          where: {
-           folderId: this._category.id,
+           folderId: this._folder.id,
            folderImapUID: removedUIDs,
          },
        })
@@ -217,7 +243,7 @@ class FetchMessagesInFolder {
       const messageValues = await MessageFactory.parseFromImap(imapMessage, desiredParts, {
         db: this._db,
         accountId: this._db.accountId,
-        folderId: this._category.id,
+        folderId: this._folder.id,
       });
       const existingMessage = await Message.find({where: {hash: messageValues.hash}});
 
@@ -248,18 +274,18 @@ class FetchMessagesInFolder {
   }
 
   async _openMailboxAndEnsureValidity() {
-    const box = await this._imap.openBox(this._category.name);
+    const box = await this._imap.openBox(this._folder.name);
 
     if (box.persistentUIDs === false) {
       throw new Error("Mailbox does not support persistentUIDs.");
     }
 
-    const lastUIDValidity = this._category.syncState.uidvalidity;
+    const lastUIDValidity = this._folder.syncState.uidvalidity;
 
     if (lastUIDValidity && (box.uidvalidity !== lastUIDValidity)) {
       this._logger.info({
         boxname: box.name,
-        categoryname: this._category.name,
+        categoryname: this._folder.name,
         remoteuidvalidity: box.uidvalidity,
         localuidvalidity: lastUIDValidity,
       }, `FetchMessagesInFolder: Recovering from UIDInvalidity`);
@@ -270,7 +296,7 @@ class FetchMessagesInFolder {
   }
 
   async _fetchUnsyncedMessages() {
-    const savedSyncState = this._category.syncState;
+    const savedSyncState = this._folder.syncState;
     const isFirstSync = savedSyncState.fetchedmax === undefined;
     const boxUidnext = this._box.uidnext;
     const boxUidvalidity = this._box.uidvalidity;
@@ -307,7 +333,7 @@ class FetchMessagesInFolder {
       }, `FetchMessagesInFolder: Fetching range`);
 
       await this._fetchMessagesAndQueueForProcessing(`${min}:${max}`);
-      const {fetchedmin, fetchedmax} = this._category.syncState;
+      const {fetchedmin, fetchedmax} = this._folder.syncState;
       return this.updateFolderSyncState({
         fetchedmin: fetchedmin ? Math.min(fetchedmin, min) : min,
         fetchedmax: fetchedmax ? Math.max(fetchedmax, max) : max,
@@ -321,7 +347,7 @@ class FetchMessagesInFolder {
   }
 
   _runScan() {
-    const {fetchedmin, fetchedmax} = this._category.syncState;
+    const {fetchedmin, fetchedmax} = this._folder.syncState;
     if ((fetchedmin === undefined) || (fetchedmax === undefined)) {
       throw new Error("Unseen messages must be fetched at least once before the first update/delete scan.")
     }
@@ -329,12 +355,12 @@ class FetchMessagesInFolder {
   }
 
   _shouldRunDeepScan() {
-    const {timeDeepScan} = this._category.syncState;
+    const {timeDeepScan} = this._folder.syncState;
     return Date.now() - (timeDeepScan || 0) > this._options.deepFolderScan;
   }
 
   async _runShallowScan() {
-    const {highestmodseq} = this._category.syncState;
+    const {highestmodseq} = this._folder.syncState;
     const nextHighestmodseq = this._box.highestmodseq;
 
     let shallowFetch = null;
@@ -354,7 +380,7 @@ class FetchMessagesInFolder {
 
     const remoteUIDAttributes = await shallowFetch;
     const localMessageAttributes = await this._db.Message.findAll({
-      where: {folderId: this._category.id},
+      where: {folderId: this._folder.id},
       attributes: MessageFlagAttributes,
     })
 
@@ -369,14 +395,14 @@ class FetchMessagesInFolder {
 
   async _runDeepScan() {
     const {Message} = this._db;
-    const {fetchedmin, fetchedmax} = this._category.syncState;
+    const {fetchedmin, fetchedmax} = this._folder.syncState;
     const range = `${fetchedmin}:${fetchedmax}`;
 
     this._logger.info({range}, `FetchMessagesInFolder: Deep attribute scan: fetching attributes in range`)
 
     const remoteUIDAttributes = await this._box.fetchUIDAttributes(range)
     const localMessageAttributes = await Message.findAll({
-      where: {folderId: this._category.id},
+      where: {folderId: this._folder.id},
       attributes: MessageFlagAttributes,
     })
 
@@ -395,11 +421,11 @@ class FetchMessagesInFolder {
   }
 
   async updateFolderSyncState(newState) {
-    if (_.isMatch(this._category.syncState, newState)) {
+    if (_.isMatch(this._folder.syncState, newState)) {
       return Promise.resolve();
     }
-    this._category.syncState = Object.assign(this._category.syncState, newState);
-    return this._category.save();
+    this._folder.syncState = Object.assign(this._folder.syncState, newState);
+    return this._folder.save();
   }
 
   async run(db, imap) {
diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index a28f2fcb9..824b43f25 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -44,12 +44,10 @@ module.exports = (sequelize, Sequelize) => {
       },
     },
     instanceMethods: {
-      setLabelsFromXGM(xGmLabels, {Label, preloadedLabels} = {}) {
+      async setLabelsFromXGM(xGmLabels, {Label, preloadedLabels} = {}) {
         this.folderImapXGMLabels = JSON.stringify(xGmLabels);
-        return Label.findXGMLabels(xGmLabels, {preloadedLabels})
-        .then((labels) =>
-          this.save().then(() => this.setLabels(labels))
-        )
+        const labels = await Label.findXGMLabels(xGmLabels, {preloadedLabels})
+        return this.setLabels(labels);
       },
 
       fetchRaw({account, db, logger}) {

From 1470565bcf865345dde0eaed560a43b3c74df581 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 30 Nov 2016 16:12:35 -0800
Subject: [PATCH 425/800] [local-sync] Fix observable for deltas

We were missing a return :'(
---
 packages/isomorphic-core/src/delta-stream-builder.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/isomorphic-core/src/delta-stream-builder.js b/packages/isomorphic-core/src/delta-stream-builder.js
index cbef15c6e..068397e23 100644
--- a/packages/isomorphic-core/src/delta-stream-builder.js
+++ b/packages/isomorphic-core/src/delta-stream-builder.js
@@ -53,7 +53,7 @@ function initialTransactions(db, request) {
 }
 
 function inflatedIncomingTransaction(db, request, transactionSource) {
-  transactionSource.flatMap((t) => inflateTransactions(db, [t]))
+  return transactionSource.flatMap((t) => inflateTransactions(db, [t]))
 }
 
 module.exports = {

From 2476cae297df1169ced923236cb428b4d9c46a4c Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 16:22:08 -0800
Subject: [PATCH 426/800] [local-sync] Sync a bit faster for testing

---
 packages/local-sync/src/local-api/default-sync-policy.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/local-sync/src/local-api/default-sync-policy.js b/packages/local-sync/src/local-api/default-sync-policy.js
index 1e414bf72..a3bccc150 100644
--- a/packages/local-sync/src/local-api/default-sync-policy.js
+++ b/packages/local-sync/src/local-api/default-sync-policy.js
@@ -1,6 +1,6 @@
 const DefaultSyncPolicy = {
   intervals: {
-    active: 30 * 1000,
+    active: 10 * 1000,
     inactive: 5 * 60 * 1000,
   },
   folderSyncOptions: {

From 935d8cf14180a9feb479fc1e6f66b25f5fb0afb6 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 16:22:29 -0800
Subject: [PATCH 427/800] [local-sync] Fix transaction log to only ignore
 syncState

---
 .../src/shared/hook-increment-version-on-save.js         | 9 ++++++++-
 packages/local-sync/src/shared/hook-transaction-log.js   | 8 ++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/packages/local-sync/src/shared/hook-increment-version-on-save.js b/packages/local-sync/src/shared/hook-increment-version-on-save.js
index 3751d0fcf..5072db2af 100644
--- a/packages/local-sync/src/shared/hook-increment-version-on-save.js
+++ b/packages/local-sync/src/shared/hook-increment-version-on-save.js
@@ -1,13 +1,20 @@
+const _ = require('underscore');
 
 module.exports = (db) => {
   for (const modelName of Object.keys(db)) {
     const model = db[modelName];
 
+    const allIgnoredFields = (changedFields) => {
+      return _.isEqual(changedFields, ['syncState']);
+    }
+
     model.beforeCreate('increment-version-c', (instance) => {
       instance.version = 1;
     });
     model.beforeUpdate('increment-version-u', (instance) => {
-      instance.version = instance.version ? instance.version + 1 : 1;
+      if (!allIgnoredFields(Object.keys(instance._changed))) {
+        instance.version = instance.version ? instance.version + 1 : 1;
+      }
     });
   }
 }
diff --git a/packages/local-sync/src/shared/hook-transaction-log.js b/packages/local-sync/src/shared/hook-transaction-log.js
index 13b826d2e..903837a7c 100644
--- a/packages/local-sync/src/shared/hook-transaction-log.js
+++ b/packages/local-sync/src/shared/hook-transaction-log.js
@@ -10,20 +10,20 @@ module.exports = (db, sequelize) => {
   }
 
   const allIgnoredFields = (changedFields) => {
-    const IGNORED_FIELDS = ["syncState", "version"];
-    return _.difference(Object.keys(changedFields), IGNORED_FIELDS).length === 0
+    return _.isEqual(changedFields, ['syncState']);
   }
 
   const transactionLogger = (event) => {
     return ({dataValues, _changed, $modelOptions}) => {
-      if ((isTransaction($modelOptions) || allIgnoredFields(_changed))) {
+      const changedFields = Object.keys(_changed)
+      if ((isTransaction($modelOptions) || allIgnoredFields(changedFields))) {
         return;
       }
 
       const transactionData = Object.assign({event}, {
         object: $modelOptions.name.singular,
         objectId: dataValues.id,
-        changedFields: Object.keys(_changed),
+        changedFields: changedFields,
       });
 
       db.Transaction.create(transactionData).then((transaction) => {

From bf9db2738c9bf8f8d49e2683aa71dd60bf410f49 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 16:23:28 -0800
Subject: [PATCH 428/800] [local-sync] Fix ingestion of flags and labels

---
 .../imap/fetch-messages-in-folder.js          | 40 +++++++++----------
 packages/local-sync/src/models/thread.js      | 39 ++++++++----------
 2 files changed, 34 insertions(+), 45 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 63afc693c..2af0b13e6 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -66,6 +66,8 @@ class FetchMessagesInFolder {
     const messagesWithChangedFlags = [];
     const messagesWithChangedLabels = [];
 
+    // Step 1: Identify changed messages and update their attributes in place
+
     const preloadedLabels = await Label.findAll();
     await PromiseUtils.each(Object.keys(remoteUIDAttributes), async (uid) => {
       const msg = messageAttributesMap[uid];
@@ -99,12 +101,9 @@ class FetchMessagesInFolder {
       }, `FetchMessagesInFolder: found new messages. These will not be processed because we assume that they will be assigned uid = uidnext, and will be picked up in the next sync when we discover unseen messages.`);
     }
 
+    // Step 2: If flags were changed, apply the changes to the corresponding
+    // threads. We do this as a separate step so we can batch-load the threads.
     if (messagesWithChangedFlags.length > 0) {
-      this._logger.info({
-        impacted_messages: messagesWithChangedFlags.length,
-      }, `FetchMessagesInFolder: Saving flag changes`);
-
-      // Update counters on the associated threads
       const threadIds = messagesWithChangedFlags.map(m => m.threadId);
       const threads = await Thread.findAll({where: {id: threadIds}});
       const threadsById = {};
@@ -112,34 +111,32 @@ class FetchMessagesInFolder {
         threadsById[thread.id] = thread;
       }
       for (const msg of messagesWithChangedFlags) {
-        // unread = true, previous = false? Add 1 to unreadCount.
         // unread = false, previous = true? Add -1 to unreadCount.
+        // IMPORTANT: Relies on messages changed above not having been saved yet!
         threadsById[msg.threadId].unreadCount += msg.unread / 1 - msg.previous('unread') / 1;
         threadsById[msg.threadId].starredCount += msg.starred / 1 - msg.previous('starred') / 1;
       }
-
-      // Save modified messages
       await sequelize.transaction(async (transaction) => {
-        await Promise.all(messagesWithChangedFlags.map(m =>
-          m.save({ fields: MessageFlagAttributes, transaction })
-        ))
         await Promise.all(threads.map(t =>
           t.save({ fields: ['starredCount', 'unreadCount'], transaction })
-        ))
+        ));
       });
     }
 
-    if (messagesWithChangedLabels.length > 0) {
-      this._logger.info({
-        impacted_messages: messagesWithChangedFlags.length,
-      }, `FetchMessagesInFolder: Saving label changes`);
+    // Step 3: Persist the messages we've updated
+    const messagesChanged = [].concat(messagesWithChangedFlags, messagesWithChangedLabels);
+    await sequelize.transaction(async (transaction) => {
+      await Promise.all(messagesChanged.map(m =>
+        m.save({fields: MessageFlagAttributes, transaction})
+      ))
+    });
 
-      // Propagate label changes to threads. Important that we do this after
-      // processing all the messages, since msgs in the same thread often change
-      // at the same time.
+    // Step 4: If message labels were changed, retreive the impacted threads
+    // and re-compute their labels. This is fairly expensive at the moment.
+    if (messagesWithChangedLabels.length > 0) {
       const threadIds = messagesWithChangedLabels.map(m => m.threadId);
       const threads = await Thread.findAll({where: {id: threadIds}});
-      threads.forEach((thread) => thread.updateLabels());
+      threads.forEach((thread) => thread.updateLabelsAndFolders());
     }
   }
 
@@ -251,8 +248,7 @@ class FetchMessagesInFolder {
         await existingMessage.update(messageValues)
         const thread = await existingMessage.getThread();
         if (thread) {
-          await thread.updateFolders();
-          await thread.updateLabels();
+          await thread.updateLabelsAndFolders();
         }
         this._logger.info({
           message_id: existingMessage.id,
diff --git a/packages/local-sync/src/models/thread.js b/packages/local-sync/src/models/thread.js
index 1fb909ddb..7342c3142 100644
--- a/packages/local-sync/src/models/thread.js
+++ b/packages/local-sync/src/models/thread.js
@@ -37,30 +37,23 @@ module.exports = (sequelize, Sequelize) => {
       },
     },
     instanceMethods: {
-      updateFolders() {
-        return this.getMessages().then((messages) => {
-          const folderIds = new Set()
-          return Promise.all(messages.map((msg) =>
-            msg.getFolder({attributes: ['id']})
-            .then((folder) => folderIds.add(folder.id)))
-          )
-          .then(() =>
-            this.setFolders(Array.from(folderIds))
-          )
-        })
-      },
+      async updateLabelsAndFolders() {
+        const messages = await this.getMessages();
+        const labelIds = new Set()
+        const folderIds = new Set()
 
-      updateLabels() {
-        return this.getMessages().then((messages) => {
-          const labelIds = new Set()
-          return Promise.all(messages.map((msg) =>
-            msg.getLabels({attributes: ['id']})
-            .then((labels) => labels.forEach(({id}) => labelIds.add(id))))
-          )
-          .then(() =>
-            this.setLabels(Array.from(labelIds))
-          )
-        })
+        await Promise.all(messages.map(async (msg) => {
+          const labels = await msg.getLabels({attributes: ['id']})
+          labels.forEach(({id}) => labelIds.add(id));
+          folderIds.add(msg.folderId)
+        }));
+
+        await Promise.all([
+          this.setLabels(Array.from(labelIds)),
+          this.setFolders(Array.from(folderIds)),
+        ]);
+
+        return this.save();
       },
 
       toJSON() {

From d8703b90c414197c5b1912cd7440df76716b6e0f Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 16:25:08 -0800
Subject: [PATCH 429/800] [local-sync] Fix deletion of labels

---
 packages/isomorphic-core/src/models/transaction.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/isomorphic-core/src/models/transaction.js b/packages/isomorphic-core/src/models/transaction.js
index e957a6762..4a0ca1592 100644
--- a/packages/isomorphic-core/src/models/transaction.js
+++ b/packages/isomorphic-core/src/models/transaction.js
@@ -13,7 +13,7 @@ module.exports = (sequelize, Sequelize) => {
           id: `${this.id}`,
           event: this.event,
           object: this.object,
-          objectId: this.objectId,
+          objectId: `${this.objectId}`,
           changedFields: this.changedFields,
         }
       },

From 94722f27abec83f8990a836795ea7f83dbc684da Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 16:25:24 -0800
Subject: [PATCH 430/800] =?UTF-8?q?[local-sync]=20Fix=20=E2=80=9CAdd=20INB?=
 =?UTF-8?q?OX=20label=E2=80=9D=20issue?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../syncback_tasks/set-thread-labels.imap.js  | 37 +++++++------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
index 090d80caa..39b187fba 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
@@ -6,37 +6,28 @@ class SetThreadLabelsIMAP extends SyncbackTask {
     return `SetThreadLabels`;
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
     const labelIds = this.syncbackRequestObject().props.labelIds
 
-    if (!labelIds || labelIds.length === 0) {
-      return TaskHelpers.forEachMessageInThread({
-        db,
-        imap,
-        threadId,
-        callback: ({message, box}) => {
-          return message.getLabels().then((labels) => {
-            const labelNames = labels.map(({name}) => name)
-            return box.removeLabels(message.folderImapUID, labelNames)
-          })
-        },
-      })
-    }
+    const labels = await db.Label.findAll({where: {id: labelIds}});
+    const gmailLabelIdentifiers = labels.map((label) => {
+      if (label.role) {
+        return `\\${label.role[0].toUpperCase()}${label.role.slice(1)}`
+      }
+      return label.name;
+    });
+
+
+    // Ben TODO this is super inefficient because it makes IMAP requests
+    // one UID at a time, rather than gathering all the UIDs and making
+    // a single removeLabels call.
     return TaskHelpers.forEachMessageInThread({
       db,
       imap,
       threadId,
       callback: ({message, box}) => {
-        return db.Label.findAll({
-          where: {
-            id: {'in': labelIds},
-          },
-        })
-        .then((labels) => {
-          const labelNames = labels.map(({name}) => name)
-          return box.setLabels(message.folderImapUID, labelNames)
-        })
+        return box.setLabels(message.folderImapUID, gmailLabelIdentifiers)
       },
     })
   }

From 3d79f9b8be739dcf354c959f58e9819b93ba2633 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Wed, 30 Nov 2016 17:26:23 -0800
Subject: [PATCH 431/800] [local-sync] Run message processor for one message at
 a time

This avoids issues that arise when we process two messages on the same thread concurrently!
---
 .../src/new-message-processor/index.js        | 40 ++++++++++---------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/packages/local-sync/src/new-message-processor/index.js b/packages/local-sync/src/new-message-processor/index.js
index 184b5ff7b..5aed1b513 100644
--- a/packages/local-sync/src/new-message-processor/index.js
+++ b/packages/local-sync/src/new-message-processor/index.js
@@ -3,27 +3,29 @@ const extractFiles = require('./extract-files')
 const extractContacts = require('./extract-contacts')
 const LocalDatabaseConnector = require('../shared/local-database-connector')
 
+const Queue = require('promise-queue');
+const queue = new Queue(1, Infinity);
+
 function processNewMessage(message, imapMessage) {
-  process.nextTick(() => {
-    const {accountId} = message
+  queue.add(async () => {
+    const {accountId} = message;
     const logger = global.Logger.forAccount({id: accountId}).child({message})
-    LocalDatabaseConnector.forAccount(accountId).then((db) => {
-      detectThread({db, message})
-      .then((thread) => {
-        message.threadId = thread.id
-        return db.Message.create(message)
-      })
-      .then(() => extractFiles({db, message, imapMessage}))
-      .then(() => extractContacts({db, message}))
-      .then(() => {
-        logger.info({
-          message_id: message.id,
-          uid: message.folderImapUID,
-        }, `MessageProcessor: Created and processed message`)
-      })
-      .catch((err) => logger.error(err, `MessageProcessor: Failed`))
-    })
-  })
+    const db = await LocalDatabaseConnector.forAccount(accountId);
+
+    try {
+      const thread = await detectThread({db, message});
+      message.threadId = thread.id;
+      await db.Message.create(message);
+      await extractFiles({db, message, imapMessage});
+      await extractContacts({db, message});
+      logger.info({
+        message_id: message.id,
+        uid: message.folderImapUID,
+      }, `MessageProcessor: Created and processed message`);
+    } catch (err) {
+      logger.error(err, `MessageProcessor: Failed`);
+    }
+  });
 }
 
 module.exports = {processNewMessage}

From 47e0683cacbe058983a5ded7715345903cfa470a Mon Sep 17 00:00:00 2001
From: Karim Hamidou 
Date: Thu, 1 Dec 2016 11:15:13 -0800
Subject: [PATCH 432/800] First step to getting persistent id in K2

Summary:
This diff adds persistent unique ids for messages and contacts. For messages, we just take a hash of the headers. For contacts, we hash the contact's email address.

This diff bundles a couple of tiny fixes too, like always trying to restart an account's sync, even after an exception.

Note that since there's no reliable way to have persistent ids for threads, we'll have to change our code to use message ids instead. Alas, that's a story for another (massive) diff.

Test Plan: Tested manually.

Reviewers: bengotow

Reviewed By: bengotow

Differential Revision: https://phab.nylas.com/D3468
---
 .../local-sync-worker/imap/fetch-messages-in-folder.js |  2 +-
 .../local-sync/src/local-sync-worker/sync-worker.js    |  3 ---
 packages/local-sync/src/models/contact.js              |  5 +++--
 packages/local-sync/src/models/file.js                 |  9 ++++++++-
 packages/local-sync/src/models/message.js              | 10 +++++-----
 .../src/new-message-processor/extract-contacts.js      |  3 ++-
 .../local-sync/src/shared/local-database-connector.js  |  1 -
 packages/local-sync/src/shared/message-factory.js      |  2 +-
 test_accounts.txt                                      |  3 +++
 9 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 2af0b13e6..d6c15f817 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -242,7 +242,7 @@ class FetchMessagesInFolder {
         accountId: this._db.accountId,
         folderId: this._folder.id,
       });
-      const existingMessage = await Message.find({where: {hash: messageValues.hash}});
+      const existingMessage = await Message.find({where: {id: messageValues.id}});
 
       if (existingMessage) {
         await existingMessage.update(messageValues)
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index 43a455681..afc7f8bba 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -146,9 +146,6 @@ class SyncWorker {
     }
 
     console.log(this._account)
-    if (this._account.errored()) {
-      this._logger.error(`SyncWorker: Account is in error state - Retrying sync\n${this._account.syncError.message}`, this._account.syncError.stack.join('\n'))
-    }
     this._logger.info({reason}, `SyncWorker: Account sync started`)
 
     try {
diff --git a/packages/local-sync/src/models/contact.js b/packages/local-sync/src/models/contact.js
index a526732f6..8dc894eaf 100644
--- a/packages/local-sync/src/models/contact.js
+++ b/packages/local-sync/src/models/contact.js
@@ -1,5 +1,6 @@
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('contact', {
+    id: {type: Sequelize.STRING(65), primaryKey: true},
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     name: Sequelize.STRING,
@@ -8,13 +9,13 @@ module.exports = (sequelize, Sequelize) => {
     indexes: [
       {
         unique: true,
-        fields: ['email'],
+        fields: ['id'],
       },
     ],
     instanceMethods: {
       toJSON: function toJSON() {
         return {
-          id: `${this.id}`,
+          id: `${this.publicId}`,
           account_id: this.accountId,
           object: 'contact',
           email: this.email,
diff --git a/packages/local-sync/src/models/file.js b/packages/local-sync/src/models/file.js
index 2393fe124..952c74ba9 100644
--- a/packages/local-sync/src/models/file.js
+++ b/packages/local-sync/src/models/file.js
@@ -2,6 +2,7 @@ const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('file', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     filename: Sequelize.STRING(500),
@@ -14,6 +15,12 @@ module.exports = (sequelize, Sequelize) => {
         File.belongsTo(Message)
       },
     },
+    indexes: [
+      {
+        unique: true,
+        fields: ['id'],
+      },
+    ],
     instanceMethods: {
       fetch: function fetch({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())
@@ -39,7 +46,7 @@ module.exports = (sequelize, Sequelize) => {
       },
       toJSON: function toJSON() {
         return {
-          id: `${this.id}`,
+          id: this.id,
           object: 'file',
           account_id: this.accountId,
           message_id: this.messageId,
diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index 824b43f25..3e3478f03 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -1,10 +1,11 @@
-const crypto = require('crypto');
+const cryptography = require('crypto');
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 const {DatabaseTypes: {JSONType, JSONARRAYType}} = require('isomorphic-core');
 
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('message', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     headerMessageId: Sequelize.STRING,
@@ -12,7 +13,6 @@ module.exports = (sequelize, Sequelize) => {
     headers: JSONType('headers'),
     subject: Sequelize.STRING(500),
     snippet: Sequelize.STRING(255),
-    hash: Sequelize.STRING(65),
     date: Sequelize.DATE,
     unread: Sequelize.BOOLEAN,
     starred: Sequelize.BOOLEAN,
@@ -28,7 +28,7 @@ module.exports = (sequelize, Sequelize) => {
     indexes: [
       {
         unique: true,
-        fields: ['hash'],
+        fields: ['id'],
       },
     ],
     classMethods: {
@@ -40,7 +40,7 @@ module.exports = (sequelize, Sequelize) => {
       },
 
       hashForHeaders(headers) {
-        return crypto.createHash('sha256').update(headers, 'utf8').digest('hex');
+        return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
       },
     },
     instanceMethods: {
@@ -79,7 +79,7 @@ module.exports = (sequelize, Sequelize) => {
         // Message though and need to protect `this.date` from null
         // errors.
         return {
-          id: `${this.id}`,
+          id: this.id,
           account_id: this.accountId,
           object: 'message',
           body: this.body,
diff --git a/packages/local-sync/src/new-message-processor/extract-contacts.js b/packages/local-sync/src/new-message-processor/extract-contacts.js
index 580078f02..91fd3ccd5 100644
--- a/packages/local-sync/src/new-message-processor/extract-contacts.js
+++ b/packages/local-sync/src/new-message-processor/extract-contacts.js
@@ -1,3 +1,4 @@
+const cryptography = require('crypto');
 
 function isContactVerified(contact) {
   // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
@@ -21,13 +22,13 @@ function extractContacts({db, message}) {
   })
 
   const verifiedContacts = allContacts.filter(c => isContactVerified(c));
-
   return db.sequelize.transaction((transaction) => {
     return Promise.all(verifiedContacts.map((contact) =>
       Contact.upsert({
         name: contact.name,
         email: contact.email,
         accountId: message.accountId,
+        id: cryptography.createHash('sha256').update(contact.email, 'utf8').digest('hex'),
       }, {
         transaction,
       })
diff --git a/packages/local-sync/src/shared/local-database-connector.js b/packages/local-sync/src/shared/local-database-connector.js
index a08c6d8c4..b9988f9d9 100644
--- a/packages/local-sync/src/shared/local-database-connector.js
+++ b/packages/local-sync/src/shared/local-database-connector.js
@@ -32,7 +32,6 @@ class LocalDatabaseConnector {
     const newSequelize = this._sequelizePoolForDatabase(`a-${accountId}`);
     const db = loadModels(Sequelize, newSequelize, {
       modelDirs: [path.resolve(__dirname, '..', 'models')],
-      schema: `a${accountId}`,
     })
 
     HookTransactionLog(db, newSequelize);
diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index 8081650ad..534e9466d 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -38,12 +38,12 @@ function parseFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
   }
 
   const values = {
+    id: Message.hashForHeaders(headers),
     to: extractContacts(parsedHeaders.to),
     cc: extractContacts(parsedHeaders.cc),
     bcc: extractContacts(parsedHeaders.bcc),
     from: extractContacts(parsedHeaders.from),
     replyTo: extractContacts(parsedHeaders['reply-to']),
-    hash: Message.hashForHeaders(headers),
     accountId: accountId,
     body: body['text/html'] || body['text/plain'] || body['application/pgp-encrypted'] || '',
     snippet: body['text/plain'] ? body['text/plain'].substr(0, 255) : null,
diff --git a/test_accounts.txt b/test_accounts.txt
index 0049f9ad6..75a3c59e9 100644
--- a/test_accounts.txt
+++ b/test_accounts.txt
@@ -14,3 +14,6 @@ Others:
 curl -k -X POST -H "Content-Type: application/json" -d '{"email":"nylastest@runbox.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"nylastest","imap_host":"mail.runbox.com","imap_port":993,"smtp_host":"mail.runbox.com","smtp_port":0,"smtp_username":"nylastest", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
 curl -k -X POST -H "Content-Type: application/json" -d '{"email":"securemail@defendcyber.space", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"securemail@defendcyber.space","imap_host":"imap.secureserver.net","imap_port":143,"smtp_host":"smtpout.secureserver.net","smtp_port":25,"smtp_username":"securemail@defendcyber.space", "smtp_password":"IHate2Gmail!","imap_password":"IHate2Gmail!","ssl_required":false}}' "http://localhost:5100/auth?client_id=123"
 curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest4@gmail.com", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest4@gmail.com","imap_host":"imap.gmail.com","imap_port":993,"smtp_host":"smtp.gmail.com","smtp_port":465,"smtp_username":"inboxapptest4@gmail.com", "smtp_password":"ihategmail","imap_password":"ihategmail","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"
+
+Fastmail:
+curl -k -X POST -H "Content-Type: application/json" -d '{"email":"inboxapptest1@fastmail.fm", "name":"Ben Gotow", "provider":"imap", "settings":{"imap_username":"inboxapptest1@fastmail.fm","imap_host":"imap.fastmail.com","imap_port":993,"smtp_host":"smtp.fastmail.com","smtp_port":465,"smtp_username":"inboxapptest1@fastmail.fm", "smtp_password":"6e7bucyuxffmg2vj","imap_password":"6e7bucyuxffmg2vj","ssl_required":true}}' "http://localhost:5100/auth?client_id=123"

From 0aac73b6d23243f794618f863cbe85ef0f474920 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Thu, 1 Dec 2016 11:21:16 -0800
Subject: [PATCH 433/800] [cloud-api] New honeycomb ingestion endpoint

---
 pm2-dev.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pm2-dev.yml b/pm2-dev.yml
index 9d50d1fa7..10b5e9b65 100644
--- a/pm2-dev.yml
+++ b/pm2-dev.yml
@@ -10,6 +10,8 @@ apps:
       GMAIL_CLIENT_SECRET : "WhmxErj-ei6vJXLocNhBbfBF"
       GMAIL_REDIRECT_URL : "http://localhost:5100/auth/gmail/oauthcallback"
       NODE_ENV: 'development'
+      HONEY_DATASET: 'n1-cloud-staging'
+      HONEY_WRITE_KEY: 'XXXXXXXXXXXXX'
   - script   : packages/cloud-workers/app.js
     watch    : ["packages"]
     name     : workers

From 209122c3462d2d5b5352ac2265e766960f78d062 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Thu, 1 Dec 2016 11:39:25 -0800
Subject: [PATCH 434/800] [cloud-api] Fix more broken package.jsons

---
 packages/isomorphic-core/package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/isomorphic-core/package.json b/packages/isomorphic-core/package.json
index cfe25a4d3..5c2ad87f9 100644
--- a/packages/isomorphic-core/package.json
+++ b/packages/isomorphic-core/package.json
@@ -9,6 +9,7 @@
     "promise.prototype.finally": "1.0.1",
     "sequelize": "3.27.0",
     "underscore": "1.8.3",
+    "rx": "4.1.0",
     "xoauth2": "1.2.0"
   },
   "author": "Nylas",

From 8939364dc3f0e0d066b0cf98080c5ee21d15e8de Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 1 Dec 2016 11:57:23 -0800
Subject: [PATCH 435/800] [local-sync]: Add distinct ids to folders and labels

---
 packages/local-sync/src/local-api/routes/categories.js        | 2 +-
 packages/local-sync/src/local-api/serialization.js            | 4 ++--
 .../src/local-sync-worker/imap/fetch-folder-list.js           | 2 ++
 packages/local-sync/src/models/folder.js                      | 1 +
 packages/local-sync/src/models/label.js                       | 1 +
 5 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/local-sync/src/local-api/routes/categories.js b/packages/local-sync/src/local-api/routes/categories.js
index 7b62141f8..c1e97de3a 100644
--- a/packages/local-sync/src/local-api/routes/categories.js
+++ b/packages/local-sync/src/local-api/routes/categories.js
@@ -72,7 +72,7 @@ module.exports = (server) => {
         tags: [term],
         validate: {
           params: {
-            id: Joi.number().integer(),
+            id: Joi.string(),
           },
         },
         response: {
diff --git a/packages/local-sync/src/local-api/serialization.js b/packages/local-sync/src/local-api/serialization.js
index 07527b1f6..4d585d0ae 100644
--- a/packages/local-sync/src/local-api/serialization.js
+++ b/packages/local-sync/src/local-api/serialization.js
@@ -35,7 +35,7 @@ function jsonSchema(modelName) {
   }
   if (modelName === 'Folder') {
     return Joi.object().keys({
-      id: Joi.number(),
+      id: Joi.string(),
       object: Joi.string(),
       account_id: Joi.string(),
       name: Joi.string().allow(null),
@@ -44,7 +44,7 @@ function jsonSchema(modelName) {
   }
   if (modelName === 'Label') {
     return Joi.object().keys({
-      id: Joi.number(),
+      id: Joi.string(),
       object: Joi.string(),
       account_id: Joi.string(),
       name: Joi.string().allow(null),
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index 0b95dadf2..e3d848408 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -1,3 +1,4 @@
+const crypto = require('crypto')
 const {Provider, PromiseUtils} = require('isomorphic-core');
 const {localizedCategoryNames} = require('../sync-utils')
 
@@ -93,6 +94,7 @@ class FetchFolderList {
         const role = this._roleByAttr(box);
         const Klass = this._classForMailboxWithRole(role, this._db);
         category = Klass.build({
+          id: crypto.createHash('sha256').update(boxName, 'utf8').digest('hex'),
           name: boxName,
           accountId: this._db.accountId,
           role: role,
diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.js
index 5d8d2fe12..769f513b9 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.js
@@ -2,6 +2,7 @@ const {DatabaseTypes: {JSONType}} = require('isomorphic-core');
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('folder', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     name: Sequelize.STRING,
diff --git a/packages/local-sync/src/models/label.js b/packages/local-sync/src/models/label.js
index e6e64b735..8c9a67ba1 100644
--- a/packages/local-sync/src/models/label.js
+++ b/packages/local-sync/src/models/label.js
@@ -1,5 +1,6 @@
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('label', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     name: Sequelize.STRING,

From b43be69cd3df28896bdc84b38c84836245997cf1 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 1 Dec 2016 13:30:47 -0800
Subject: [PATCH 436/800] [local-sync] Give better name to thread folders task

---
 packages/local-sync/src/local-api/routes/threads.js         | 2 +-
 .../src/local-sync-worker/syncback-task-factory.js          | 4 ++--
 ...move-to-folder.imap.js => move-thread-to-folder.imap.js} | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)
 rename packages/local-sync/src/local-sync-worker/syncback_tasks/{move-to-folder.imap.js => move-thread-to-folder.imap.js} (81%)

diff --git a/packages/local-sync/src/local-api/routes/threads.js b/packages/local-sync/src/local-api/routes/threads.js
index 62aba7151..3814535d5 100644
--- a/packages/local-sync/src/local-api/routes/threads.js
+++ b/packages/local-sync/src/local-api/routes/threads.js
@@ -78,7 +78,7 @@ module.exports = (server) => {
       const payload = request.payload
       if (payload.folder_id || payload.folder) {
         createSyncbackRequest(request, reply, {
-          type: "MoveToFolder",
+          type: "MoveThreadToFolder",
           props: {
             folderId: payload.folder_id || payload.folder,
             threadId: request.params.id,
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
index 489b83cd2..f7433a62e 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
@@ -6,8 +6,8 @@ class SyncbackTaskFactory {
   static create(account, syncbackRequest) {
     let Task = null;
     switch (syncbackRequest.type) {
-      case "MoveToFolder":
-        Task = require('./syncback_tasks/move-to-folder.imap'); break;
+      case "MoveThreadToFolder":
+        Task = require('./syncback_tasks/move-thread-to-folder.imap'); break;
       case "SetThreadLabels":
         Task = require('./syncback_tasks/set-thread-labels.imap'); break;
       case "MarkThreadAsRead":
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-to-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
similarity index 81%
rename from packages/local-sync/src/local-sync-worker/syncback_tasks/move-to-folder.imap.js
rename to packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
index 989d534c7..58f92d679 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-to-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
@@ -1,9 +1,9 @@
 const SyncbackTask = require('./syncback-task')
 const TaskHelpers = require('./task-helpers')
 
-class MoveToFolderIMAP extends SyncbackTask {
+class MoveThreadToFolderIMAP extends SyncbackTask {
   description() {
-    return `MoveToFolder`;
+    return `MoveThreadToFolder`;
   }
 
   run(db, imap) {
@@ -19,4 +19,4 @@ class MoveToFolderIMAP extends SyncbackTask {
     return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})
   }
 }
-module.exports = MoveToFolderIMAP
+module.exports = MoveThreadToFolderIMAP

From aba77ca6372634ed13c9c81e9f02c2836473e0b1 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Thu, 1 Dec 2016 16:32:35 -0500
Subject: [PATCH 437/800] [cloud-api] add npm start for pm2 launch

---
 package.json                          | 1 +
 packages/isomorphic-core/package.json | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index bbdb38cf2..803b53356 100644
--- a/package.json
+++ b/package.json
@@ -17,6 +17,7 @@
     "eslint-plugin-react": "6.7.1"
   },
   "scripts": {
+    "start": "pm2 start ./pm2-dev.yml --no-daemon",
     "postinstall": "lerna bootstrap"
   },
   "repository": {
diff --git a/packages/isomorphic-core/package.json b/packages/isomorphic-core/package.json
index 5c2ad87f9..6a5c04e21 100644
--- a/packages/isomorphic-core/package.json
+++ b/packages/isomorphic-core/package.json
@@ -7,9 +7,9 @@
     "imap": "0.8.18",
     "promise-props": "1.0.0",
     "promise.prototype.finally": "1.0.1",
+    "rx": "4.1.0",
     "sequelize": "3.27.0",
     "underscore": "1.8.3",
-    "rx": "4.1.0",
     "xoauth2": "1.2.0"
   },
   "author": "Nylas",

From ce27b0db2a3a642abca21ab9564a97ec33a2eda0 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 1 Dec 2016 13:39:46 -0800
Subject: [PATCH 438/800] [local-sync] Fix setlabels task

Make sure we /remove/ the labels when we get an empty set of label ids
---
 .../syncback_tasks/set-thread-labels.imap.js  | 24 ++++++++++++-------
 packages/local-sync/src/models/label.js       |  9 ++++++-
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
index 39b187fba..9aaa2695a 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
@@ -10,14 +10,22 @@ class SetThreadLabelsIMAP extends SyncbackTask {
     const threadId = this.syncbackRequestObject().props.threadId
     const labelIds = this.syncbackRequestObject().props.labelIds
 
-    const labels = await db.Label.findAll({where: {id: labelIds}});
-    const gmailLabelIdentifiers = labels.map((label) => {
-      if (label.role) {
-        return `\\${label.role[0].toUpperCase()}${label.role.slice(1)}`
-      }
-      return label.name;
-    });
+    if (!labelIds || labelIds.length === 0) {
+      return TaskHelpers.forEachMessageInThread({
+        db,
+        imap,
+        threadId,
+        callback: ({message, box}) => {
+          return message.getLabels().then((labels) => {
+            const labelIdentifiers = labels.map(label => label.imapLabelIdentifier())
+            return box.removeLabels(message.folderImapUID, labelIdentifiers)
+          })
+        },
+      })
+    }
 
+    const labels = await db.Label.findAll({where: {id: labelIds}});
+    const labelIdentifiers = labels.map(label => label.imapLabelIdentifier());
 
     // Ben TODO this is super inefficient because it makes IMAP requests
     // one UID at a time, rather than gathering all the UIDs and making
@@ -27,7 +35,7 @@ class SetThreadLabelsIMAP extends SyncbackTask {
       imap,
       threadId,
       callback: ({message, box}) => {
-        return box.setLabels(message.folderImapUID, gmailLabelIdentifiers)
+        return box.setLabels(message.folderImapUID, labelIdentifiers)
       },
     })
   }
diff --git a/packages/local-sync/src/models/label.js b/packages/local-sync/src/models/label.js
index 8c9a67ba1..4217d2299 100644
--- a/packages/local-sync/src/models/label.js
+++ b/packages/local-sync/src/models/label.js
@@ -36,7 +36,14 @@ module.exports = (sequelize, Sequelize) => {
       },
     },
     instanceMethods: {
-      toJSON: function toJSON() {
+      imapLabelIdentifier() {
+        if (this.role) {
+          return `\\${this.role[0].toUpperCase()}${this.role.slice(1)}`
+        }
+        return this.name;
+      },
+
+      toJSON() {
         return {
           id: `${this.id}`,
           account_id: this.accountId,

From 47d8614ed72c9670ebb599677c50819a79c14a30 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Thu, 1 Dec 2016 14:23:49 -0800
Subject: [PATCH 439/800] [local-sync] Ship first sync metrics to honeycomb

---
 .../src/local-sync-dashboard/root.jsx         |  2 +-
 .../sync-metrics-reporter.js                  | 58 +++++++++++++++++++
 .../local-sync-worker/sync-process-manager.js |  1 -
 .../src/local-sync-worker/sync-worker.js      | 31 ++++++++++
 .../src/new-message-processor/index.js        |  8 +--
 5 files changed, 94 insertions(+), 6 deletions(-)
 create mode 100644 packages/local-sync/src/local-sync-worker/sync-metrics-reporter.js

diff --git a/packages/local-sync/src/local-sync-dashboard/root.jsx b/packages/local-sync/src/local-sync-dashboard/root.jsx
index 0106d1657..cad562aaf 100644
--- a/packages/local-sync/src/local-sync-dashboard/root.jsx
+++ b/packages/local-sync/src/local-sync-dashboard/root.jsx
@@ -84,7 +84,7 @@ class AccountCard extends React.Component {
 
     let firstSyncDuration = "Incomplete";
     if (account.firstSyncCompletion) {
-      firstSyncDuration = (new Date(account.firstSyncCompletion) - new Date(account.createdAt)) / 1000;
+      firstSyncDuration = (new Date(account.firstSyncCompletion / 1) - new Date(account.createdAt)) / 1000;
     }
 
     const position = calcAcctPosition(this.props.count);
diff --git a/packages/local-sync/src/local-sync-worker/sync-metrics-reporter.js b/packages/local-sync/src/local-sync-worker/sync-metrics-reporter.js
new file mode 100644
index 000000000..f129fcc77
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/sync-metrics-reporter.js
@@ -0,0 +1,58 @@
+const {N1CloudAPI, NylasAPIRequest, AccountStore} = require('nylas-exports');
+const os = require('os');
+
+class SyncMetricsReporter {
+  constructor() {
+    this._logger = global.Logger.child();
+  }
+
+  async collectCPUUsage() {
+    return new Promise((resolve) => {
+      const startUsage = process.cpuUsage();
+      const sampleDuration = 400;
+      setTimeout(() => {
+        const {user, system} = process.cpuUsage(startUsage);
+        const fractionToPrecent = 100.0;
+        resolve(Math.round((user + system) / (sampleDuration * 1000.0) * fractionToPrecent));
+      }, sampleDuration);
+    });
+  }
+
+  async reportEvent(info) {
+    if (!info.emailAddress) {
+      throw new Error("You must include email_address");
+    }
+
+    const {workingSetSize, privateBytes, sharedBytes} = process.getProcessMemoryInfo();
+    const percentCPU = await this.collectCPUUsage();
+
+    info.hostname = os.hostname();
+    info.cpus = os.cpus().length;
+    info.arch = os.arch();
+    info.platform = process.platform;
+    info.version = NylasEnv.getVersion();
+    info.processWorkingSetSize = workingSetSize;
+    info.processPrivateBytes = privateBytes;
+    info.processSharedBytes = sharedBytes;
+    info.processPercentCPU = percentCPU;
+
+    const req = new NylasAPIRequest({
+      api: N1CloudAPI,
+      options: {
+        path: `/ingest-metrics`,
+        method: 'POST',
+        body: info,
+        error: () => {
+          this._logger.warn(info, "Metrics Collector: Submission Failed.");
+        },
+        accountId: AccountStore.accountForEmail(info.emailAddress).id,
+        success: () => {
+          this._logger.info(info, "Metrics Collector: Submitted.");
+        },
+      },
+    });
+    req.run();
+  }
+}
+
+module.exports = new SyncMetricsReporter();
diff --git a/packages/local-sync/src/local-sync-worker/sync-process-manager.js b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
index c9004ba9e..3fcd2ac20 100644
--- a/packages/local-sync/src/local-sync-worker/sync-process-manager.js
+++ b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
@@ -66,7 +66,6 @@ class SyncProcessManager {
       this._workers[accountId] = null;
     }
   }
-
 }
 
 module.exports = new SyncProcessManager();
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index afc7f8bba..04500ca6d 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -11,6 +11,7 @@ const {
 const FetchFolderList = require('./imap/fetch-folder-list')
 const FetchMessagesInFolder = require('./imap/fetch-messages-in-folder')
 const SyncbackTaskFactory = require('./syncback-task-factory')
+const SyncMetricsReporter = require('./sync-metrics-reporter');
 
 
 class SyncWorker {
@@ -27,6 +28,36 @@ class SyncWorker {
     this._syncTimer = setTimeout(() => {
       this.syncNow({reason: 'Initial'});
     }, 0);
+
+    // setup metrics collection. We do this in an isolated way by hooking onto
+    // the database, because otherwise things get /crazy/ messy and I don't like
+    // having counters and garbage everywhere.
+    if (!account.firstSyncCompletion) {
+      this._logger.info("This is initial sync. Setting up metrics collection!");
+
+      let seen = 0;
+      db.Thread.addHook('afterCreate', 'metricsCollection', () => {
+        if (seen === 0) {
+          SyncMetricsReporter.reportEvent({
+            type: 'imap',
+            emailAddress: account.emailAddress,
+            msecToFirstThread: (Date.now() - new Date(account.createdAt).getTime()),
+          })
+        }
+        if (seen === 500) {
+          SyncMetricsReporter.reportEvent({
+            type: 'imap',
+            emailAddress: account.emailAddress,
+            msecToFirst500Threads: (Date.now() - new Date(account.createdAt).getTime()),
+          })
+        }
+
+        if (seen > 500) {
+          db.Thread.removeHook('afterCreate', 'metricsCollection')
+        }
+        seen += 1;
+      });
+    }
   }
 
   cleanup() {
diff --git a/packages/local-sync/src/new-message-processor/index.js b/packages/local-sync/src/new-message-processor/index.js
index 5aed1b513..bc5d072b7 100644
--- a/packages/local-sync/src/new-message-processor/index.js
+++ b/packages/local-sync/src/new-message-processor/index.js
@@ -1,7 +1,7 @@
-const detectThread = require('./detect-thread')
-const extractFiles = require('./extract-files')
-const extractContacts = require('./extract-contacts')
-const LocalDatabaseConnector = require('../shared/local-database-connector')
+const detectThread = require('./detect-thread');
+const extractFiles = require('./extract-files');
+const extractContacts = require('./extract-contacts');
+const LocalDatabaseConnector = require('../shared/local-database-connector');
 
 const Queue = require('promise-queue');
 const queue = new Queue(1, Infinity);

From 2d932dd090f230dc428bdafbcb66999f52d6638b Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 1 Dec 2016 14:44:41 -0800
Subject: [PATCH 440/800] [*] Generate persistent id for Accounts based on
 email+settings

This also ensures that we can use the accountId as part of the ids for
other models, like labels and folders

Before this commit, labels and folders with the same name for different
accounts would have the same id, screwing things up in n1
---
 packages/isomorphic-core/src/models/account.js       |  7 +++++++
 packages/local-sync/src/local-api/routes/auth.js     | 12 +++++-------
 .../src/local-sync-worker/imap/fetch-folder-list.js  |  5 +++--
 packages/local-sync/src/models/folder.js             |  4 ++++
 packages/local-sync/src/models/label.js              |  4 ++++
 5 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/packages/isomorphic-core/src/models/account.js b/packages/isomorphic-core/src/models/account.js
index 1af57971d..cc43902af 100644
--- a/packages/isomorphic-core/src/models/account.js
+++ b/packages/isomorphic-core/src/models/account.js
@@ -5,6 +5,7 @@ const {DB_ENCRYPTION_ALGORITHM, DB_ENCRYPTION_PASSWORD} = process.env;
 
 module.exports = (sequelize, Sequelize) => {
   const Account = sequelize.define('account', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     name: Sequelize.STRING,
     provider: Sequelize.STRING,
     emailAddress: Sequelize.STRING,
@@ -19,6 +20,12 @@ module.exports = (sequelize, Sequelize) => {
     },
     lastSyncCompletions: JSONARRAYType('lastSyncCompletions'),
   }, {
+    indexes: [
+      {
+        unique: true,
+        fields: ['id'],
+      },
+    ],
     classMethods: {
       associate: ({AccountToken}) => {
         Account.hasMany(AccountToken, {as: 'tokens'})
diff --git a/packages/local-sync/src/local-api/routes/auth.js b/packages/local-sync/src/local-api/routes/auth.js
index 53d8d256d..ca9633231 100644
--- a/packages/local-sync/src/local-api/routes/auth.js
+++ b/packages/local-sync/src/local-api/routes/auth.js
@@ -1,6 +1,6 @@
 const Joi = require('joi');
 const _ = require('underscore');
-
+const crypto = require('crypto');
 const Serialization = require('../serialization');
 const {
   IMAPConnection,
@@ -36,13 +36,11 @@ const buildAccountWith = ({name, email, provider, settings, credentials}) => {
   return LocalDatabaseConnector.forShared().then((db) => {
     const {AccountToken, Account} = db;
 
-    return Account.find({
-      where: {
-        emailAddress: email,
-        connectionSettings: JSON.stringify(settings),
-      },
-    }).then((existing) => {
+    const idString = `${email}${JSON.stringify(settings)}`
+    const id = crypto.createHash('sha256').update(idString, 'utf8').digest('hex')
+    return Account.findById(id).then((existing) => {
       const account = existing || Account.build({
+        id,
         name: name,
         provider: provider,
         emailAddress: email,
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index e3d848408..ba09ecc56 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -93,10 +93,11 @@ class FetchFolderList {
       if (!category) {
         const role = this._roleByAttr(box);
         const Klass = this._classForMailboxWithRole(role, this._db);
+        const {accountId} = this._db
         category = Klass.build({
-          id: crypto.createHash('sha256').update(boxName, 'utf8').digest('hex'),
+          accountId,
+          id: crypto.createHash('sha256').update(`${accountId}${boxName}`, 'utf8').digest('hex'),
           name: boxName,
-          accountId: this._db.accountId,
           role: role,
         });
         created.push(category);
diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.js
index 769f513b9..9272dc610 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.js
@@ -14,6 +14,10 @@ module.exports = (sequelize, Sequelize) => {
         unique: true,
         fields: ['role'],
       },
+      {
+        unique: true,
+        fields: ['id'],
+      },
     ],
     classMethods: {
       associate: ({Folder, Message, Thread}) => {
diff --git a/packages/local-sync/src/models/label.js b/packages/local-sync/src/models/label.js
index 4217d2299..5960362a7 100644
--- a/packages/local-sync/src/models/label.js
+++ b/packages/local-sync/src/models/label.js
@@ -11,6 +11,10 @@ module.exports = (sequelize, Sequelize) => {
         unique: true,
         fields: ['role'],
       },
+      {
+        unique: true,
+        fields: ['id'],
+      },
     ],
     classMethods: {
       associate({Label, Message, MessageLabel, Thread, ThreadLabel}) {

From aa49f85980f9356182c0da8987ec30ce39b36689 Mon Sep 17 00:00:00 2001
From: Karim Hamidou 
Date: Thu, 1 Dec 2016 15:29:47 -0800
Subject: [PATCH 441/800] Fix bug where we wouldn't create an id for file
 parts.

---
 packages/local-sync/src/models/file.js                         | 2 +-
 packages/local-sync/src/new-message-processor/extract-files.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/local-sync/src/models/file.js b/packages/local-sync/src/models/file.js
index 952c74ba9..28904ba1a 100644
--- a/packages/local-sync/src/models/file.js
+++ b/packages/local-sync/src/models/file.js
@@ -2,7 +2,7 @@ const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('file', {
-    id: { type: Sequelize.STRING(65), primaryKey: true },
+    id: { type: Sequelize.STRING(500), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     filename: Sequelize.STRING(500),
diff --git a/packages/local-sync/src/new-message-processor/extract-files.js b/packages/local-sync/src/new-message-processor/extract-files.js
index 841ab9c1e..d208052ed 100644
--- a/packages/local-sync/src/new-message-processor/extract-files.js
+++ b/packages/local-sync/src/new-message-processor/extract-files.js
@@ -10,7 +10,6 @@ function collectFilesFromStruct({db, message, struct}) {
       // Only exposes partId for inline attachments
       const partId = part.disposition.type === 'inline' ? part.partID : null;
       const filename = part.disposition.params ? part.disposition.params.filename : null;
-
       collected.push(File.build({
         filename: filename,
         partId: partId,
@@ -18,6 +17,7 @@ function collectFilesFromStruct({db, message, struct}) {
         contentType: `${part.type}/${part.subtype}`,
         accountId: message.accountId,
         size: part.size,
+        id: `${message.id}-${partId}-${part.size}`,
       }));
     }
   }

From 4b4ab726e25ef1f45492787b8769d81fc484d4af Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Thu, 1 Dec 2016 15:37:44 -0800
Subject: [PATCH 442/800] =?UTF-8?q?[=F0=9F=92=84]=20fix=20eslint=20issues?=
 =?UTF-8?q?=20before=20they=20get=20overwhelming?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 packages/isomorphic-core/index.js             |  1 +
 packages/isomorphic-core/src/imap-errors.js   | 58 +++++++++----------
 .../migrations/20160617002207-create-user.js  | 25 ++++----
 packages/isomorphic-core/src/promise-utils.js |  4 +-
 .../src/local-sync-worker/sync-utils.js       | 45 +++++++-------
 .../syncback-task-factory.js                  |  1 +
 .../local-sync/src/models/messageLabel.js     |  2 +-
 .../local-sync/src/models/threadFolder.js     |  2 +-
 packages/local-sync/src/models/threadLabel.js |  2 +-
 .../spec/fixtures/thread.js                   | 15 +----
 .../spec/threading-spec.js                    | 26 ++++-----
 packages/local-sync/stylesheets/index.less    |  2 +-
 12 files changed, 88 insertions(+), 95 deletions(-)

diff --git a/packages/isomorphic-core/index.js b/packages/isomorphic-core/index.js
index 0ec271702..2a7e6407b 100644
--- a/packages/isomorphic-core/index.js
+++ b/packages/isomorphic-core/index.js
@@ -1,3 +1,4 @@
+/* eslint global-require: 0 */
 module.exports = {
   Provider: {
     Gmail: 'gmail',
diff --git a/packages/isomorphic-core/src/imap-errors.js b/packages/isomorphic-core/src/imap-errors.js
index 543ab611c..8f8bcab35 100644
--- a/packages/isomorphic-core/src/imap-errors.js
+++ b/packages/isomorphic-core/src/imap-errors.js
@@ -1,3 +1,31 @@
+/**
+ * An abstract base class that can be used to indicate IMAPErrors that may
+ * fix themselves when retried
+ */
+class RetryableError extends Error { }
+
+/**
+ * IMAPErrors that originate from NodeIMAP. See `convertImapError` for
+ * documentation on underlying causes
+ */
+class IMAPSocketError extends RetryableError { }
+class IMAPConnectionTimeoutError extends RetryableError { }
+class IMAPAuthenticationTimeoutError extends RetryableError { }
+class IMAPProtocolError extends Error { }
+class IMAPAuthenticationError extends Error { }
+
+class IMAPConnectionNotReadyError extends RetryableError {
+  constructor(funcName) {
+    super(`${funcName} - You must call connect() first.`);
+  }
+}
+
+class IMAPConnectionEndedError extends Error {
+  constructor(msg = "The IMAP Connection was ended.") {
+    super(msg);
+  }
+}
+
 /**
  * IMAPErrors may come from:
  *
@@ -39,7 +67,7 @@
  */
 function convertImapError(imapError) {
   let error;
-  switch(imapError.source) {
+  switch (imapError.source) {
     case "socket-timeout":
       error = new IMAPConnectionTimeoutError(imapError); break;
     case "timeout":
@@ -59,34 +87,6 @@ function convertImapError(imapError) {
   return error
 }
 
-/**
- * An abstract base class that can be used to indicate IMAPErrors that may
- * fix themselves when retried
- */
-class RetryableError extends Error {  }
-
-/**
- * IMAPErrors that originate from NodeIMAP. See `convertImapError` for
- * documentation on underlying causes
- */
-class IMAPSocketError extends RetryableError { }
-class IMAPConnectionTimeoutError extends RetryableError { }
-class IMAPAuthenticationTimeoutError extends RetryableError { }
-class IMAPProtocolError extends Error { }
-class IMAPAuthenticationError extends Error { }
-
-class IMAPConnectionNotReadyError extends RetryableError {
-  constructor(funcName) {
-    super(`${funcName} - You must call connect() first.`);
-  }
-}
-
-class IMAPConnectionEndedError extends Error {
-  constructor(msg = "The IMAP Connection was ended.") {
-    super(msg);
-  }
-}
-
 module.exports = {
   convertImapError,
   RetryableError,
diff --git a/packages/isomorphic-core/src/migrations/20160617002207-create-user.js b/packages/isomorphic-core/src/migrations/20160617002207-create-user.js
index 1b8f61170..bc89a59fd 100644
--- a/packages/isomorphic-core/src/migrations/20160617002207-create-user.js
+++ b/packages/isomorphic-core/src/migrations/20160617002207-create-user.js
@@ -1,33 +1,34 @@
-'use strict';
+/* eslint no-unused-vars: 0 */
+
 module.exports = {
-  up: function(queryInterface, Sequelize) {
+  up: function up(queryInterface, Sequelize) {
     return queryInterface.createTable('Users', {
       id: {
         allowNull: false,
         autoIncrement: true,
         primaryKey: true,
-        type: Sequelize.INTEGER
+        type: Sequelize.INTEGER,
       },
       first_name: {
-        type: Sequelize.STRING
+        type: Sequelize.STRING,
       },
       last_name: {
-        type: Sequelize.STRING
+        type: Sequelize.STRING,
       },
       bio: {
-        type: Sequelize.TEXT
+        type: Sequelize.TEXT,
       },
       createdAt: {
         allowNull: false,
-        type: Sequelize.DATE
+        type: Sequelize.DATE,
       },
       updatedAt: {
         allowNull: false,
-        type: Sequelize.DATE
-      }
+        type: Sequelize.DATE,
+      },
     });
   },
-  down: function(queryInterface, Sequelize) {
+  down: function down(queryInterface, Sequelize) {
     return queryInterface.dropTable('Users');
-  }
-};
\ No newline at end of file
+  },
+};
diff --git a/packages/isomorphic-core/src/promise-utils.js b/packages/isomorphic-core/src/promise-utils.js
index 6490c5019..1aa698ce2 100644
--- a/packages/isomorphic-core/src/promise-utils.js
+++ b/packages/isomorphic-core/src/promise-utils.js
@@ -1,7 +1,7 @@
 /* eslint no-restricted-syntax: 0 */
 
 require('promise.prototype.finally')
-
+const props = require('promise-props');
 const _ = require('underscore')
 
 global.Promise.prototype.thenReturn = function thenReturn(value) {
@@ -52,5 +52,5 @@ module.exports = {
   sleep,
   promisify,
   promisifyAll,
-  props: require('promise-props'),
+  props: props,
 }
diff --git a/packages/local-sync/src/local-sync-worker/sync-utils.js b/packages/local-sync/src/local-sync-worker/sync-utils.js
index 8c55ab3d4..d84efb6ca 100644
--- a/packages/local-sync/src/local-sync-worker/sync-utils.js
+++ b/packages/local-sync/src/local-sync-worker/sync-utils.js
@@ -17,25 +17,30 @@ module.exports = {
   //
   // Make sure these are lower case! (for comparison purposes)
   localizedCategoryNames: {
-    trash: ['gel\xc3\xb6scht', 'papierkorb',
-              '\xd0\x9a\xd0\xbe\xd1\x80\xd0\xb7\xd0\xb8\xd0\xbd\xd0\xb0',
-              '[imap]/trash', 'papelera', 'borradores',
-              '[imap]/\xd0\x9a\xd0\xbe\xd1\x80',
-              '\xd0\xb7\xd0\xb8\xd0\xbd\xd0\xb0', 'deleted items',
-              '\xd0\xa1\xd0\xbc\xd1\x96\xd1\x82\xd1\x82\xd1\x8f',
-              'papierkorb/trash', 'gel\xc3\xb6schte elemente',
-              'deleted messages', '[gmail]/trash', 'inbox/trash', 'trash',
-              'mail/trash', 'inbox.trash'],
-    spam: ['roskaposti', 'inbox.spam', 'inbox.spam', 'skr\xc3\xa4ppost',
-             'spamverdacht', 'spam', 'spam', '[gmail]/spam', '[imap]/spam',
-             '\xe5\x9e\x83\xe5\x9c\xbe\xe9\x82\xae\xe4\xbb\xb6', 'junk',
-             'junk mail', 'junk e-mail'],
-    inbox: ['inbox'],
-    sent: ['postausgang', 'inbox.gesendet', '[gmail]/sent mail',
-             '\xeb\xb3\xb4\xeb\x82\xbc\xed\x8e\xb8\xec\xa7\x80\xed\x95\xa8',
-             'elementos enviados', 'sent', 'sent items', 'sent messages',
-             'inbox.papierkorb', 'odeslan\xc3\xa9', 'mail/sent-mail',
-             'ko\xc5\xa1', 'outbox', 'outbox', 'inbox.sentmail', 'gesendet',
-             'ko\xc5\xa1/sent items', 'gesendete elemente'],
+    trash: [
+      'gel\xc3\xb6scht', 'papierkorb',
+      '\xd0\x9a\xd0\xbe\xd1\x80\xd0\xb7\xd0\xb8\xd0\xbd\xd0\xb0',
+      '[imap]/trash', 'papelera', 'borradores',
+      '[imap]/\xd0\x9a\xd0\xbe\xd1\x80',
+      '\xd0\xb7\xd0\xb8\xd0\xbd\xd0\xb0', 'deleted items',
+      '\xd0\xa1\xd0\xbc\xd1\x96\xd1\x82\xd1\x82\xd1\x8f',
+      'papierkorb/trash', 'gel\xc3\xb6schte elemente',
+      'deleted messages', '[gmail]/trash', 'inbox/trash', 'trash',
+      'mail/trash', 'inbox.trash'],
+    spam: [
+      'roskaposti', 'inbox.spam', 'inbox.spam', 'skr\xc3\xa4ppost',
+      'spamverdacht', 'spam', 'spam', '[gmail]/spam', '[imap]/spam',
+      '\xe5\x9e\x83\xe5\x9c\xbe\xe9\x82\xae\xe4\xbb\xb6', 'junk',
+      'junk mail', 'junk e-mail'],
+    inbox: [
+      'inbox',
+    ],
+    sent: [
+      'postausgang', 'inbox.gesendet', '[gmail]/sent mail',
+      '\xeb\xb3\xb4\xeb\x82\xbc\xed\x8e\xb8\xec\xa7\x80\xed\x95\xa8',
+      'elementos enviados', 'sent', 'sent items', 'sent messages',
+      'inbox.papierkorb', 'odeslan\xc3\xa9', 'mail/sent-mail',
+      'ko\xc5\xa1', 'outbox', 'outbox', 'inbox.sentmail', 'gesendet',
+      'ko\xc5\xa1/sent items', 'gesendete elemente'],
   },
 }
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
index f7433a62e..a2453aa4b 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
@@ -1,3 +1,4 @@
+/* eslint global-require: 0 */
 /**
  * Given a `SyncbackRequestObject` it creates the appropriate syncback task.
  *
diff --git a/packages/local-sync/src/models/messageLabel.js b/packages/local-sync/src/models/messageLabel.js
index adce4300d..6684b3ed2 100644
--- a/packages/local-sync/src/models/messageLabel.js
+++ b/packages/local-sync/src/models/messageLabel.js
@@ -1,4 +1,4 @@
-module.exports = (sequelize, Sequelize) => {
+module.exports = (sequelize) => {
   return sequelize.define('messageLabel', {
   }, {
   });
diff --git a/packages/local-sync/src/models/threadFolder.js b/packages/local-sync/src/models/threadFolder.js
index 2bda8ccca..298c84fae 100644
--- a/packages/local-sync/src/models/threadFolder.js
+++ b/packages/local-sync/src/models/threadFolder.js
@@ -1,4 +1,4 @@
-module.exports = (sequelize, Sequelize) => {
+module.exports = (sequelize) => {
   return sequelize.define('threadFolder', {
   }, {
   });
diff --git a/packages/local-sync/src/models/threadLabel.js b/packages/local-sync/src/models/threadLabel.js
index 8f3328a1d..c76f2f557 100644
--- a/packages/local-sync/src/models/threadLabel.js
+++ b/packages/local-sync/src/models/threadLabel.js
@@ -1,4 +1,4 @@
-module.exports = (sequelize, Sequelize) => {
+module.exports = (sequelize) => {
   return sequelize.define('threadLabel', {
   }, {
   });
diff --git a/packages/local-sync/src/new-message-processor/spec/fixtures/thread.js b/packages/local-sync/src/new-message-processor/spec/fixtures/thread.js
index ce407afb7..82be1660b 100644
--- a/packages/local-sync/src/new-message-processor/spec/fixtures/thread.js
+++ b/packages/local-sync/src/new-message-processor/spec/fixtures/thread.js
@@ -1,27 +1,14 @@
 export const message = {
   id: 1,
-	subject: "Loved your work and interests",
+  subject: "Loved your work and interests",
   body: "Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
   headers: {
     "Delivered-To": "jackiehluo@gmail.com",
-    "Received": `by 10.107.182.215 with SMTP id g206csp311103iof;
-      Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
-    "X-Received": `by 10.66.16.133 with SMTP id g5mr1799805pad.145.1466181525915;
-      Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
-    "Return-Path": "",
-    "Received": `from mail-pf0-f174.google.com (mail-pf0-f174.google.com.
-      [209.85.192.174])
-      by mx.google.com with ESMTPS id n6si15649421pav.242.2016.06.17.09.38.45
-      for 
-      (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
-      Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
     "Received-SPF": `pass (google.com: domain of sagy26.1991@gmail.com
       designates 209.85.192.174 as permitted sender) client-ip=209.85.192.174;`,
     "Authentication-Results": `mx.google.com;
        spf=pass (google.com: domain of sagy26.1991@gmail.com designates
        209.85.192.174 as permitted sender) smtp.mailfrom=sagy26.1991@gmail.com`,
-    "Received": `by mail-pf0-f174.google.com with SMTP id i123so25772868pfg.0
-        for ; Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
     "X-Google-DKIM-Signature": `v=1; a=rsa-sha256; c=relaxed/relaxed;
         d=1e100.net; s=20130820;
         h=x-gm-message-state:date:user-agent:message-id:to:from:subject
diff --git a/packages/local-sync/src/new-message-processor/spec/threading-spec.js b/packages/local-sync/src/new-message-processor/spec/threading-spec.js
index c8f1a44c6..8fbe812cf 100644
--- a/packages/local-sync/src/new-message-processor/spec/threading-spec.js
+++ b/packages/local-sync/src/new-message-processor/spec/threading-spec.js
@@ -1,19 +1,17 @@
+/* eslint global-require: 0 */
+/* eslint import/no-dynamic-require: 0 */
 const path = require('path')
-const fs = require('fs')
-const LocalDatabaseConnector = require('../../shared/local-database-connector')
 const {processMessage} = require('../processors/threading')
 
 const BASE_PATH = path.join(__dirname, 'fixtures')
 
-
 it('adds the message to the thread', (done) => {
   const {message, reply} = require(`${BASE_PATH}/thread`)
   const accountId = 'a-1'
   const mockDb = {
     Thread: {
       findAll: () => {
-        return Promise.resolve([
-        {
+        return Promise.resolve([{
           id: 1,
           subject: "Loved your work and interests",
           messages: [message],
@@ -24,15 +22,15 @@ it('adds the message to the thread', (done) => {
       },
       create: (thread) => {
         thread.id = 1
-        thread.addMessage = (message) => {
+        thread.addMessage = (newMessage) => {
           if (thread.messages) {
-            thread.messages.push(message.id)
+            thread.messages.push(newMessage.id)
           } else {
-            thread.messages = [message.id]
+            thread.messages = [newMessage.id]
           }
         }
         return Promise.resolve(thread)
-      }
+      },
     },
     Message: {
       findAll: () => {
@@ -41,13 +39,13 @@ it('adds the message to the thread', (done) => {
       find: () => {
         return Promise.resolve(reply)
       },
-      create: (message) => {
+      create: () => {
         message.setThread = (thread) => {
           message.thread = thread.id
-        }
-        return Promise.resolve(message)
-      }
-    }
+        };
+        return Promise.resolve(message);
+      },
+    },
   }
 
   processMessage({db: mockDb, message: reply, accountId}).then((processed) => {
diff --git a/packages/local-sync/stylesheets/index.less b/packages/local-sync/stylesheets/index.less
index e159cabca..a9f277209 100644
--- a/packages/local-sync/stylesheets/index.less
+++ b/packages/local-sync/stylesheets/index.less
@@ -170,7 +170,7 @@
 
   .dropdown-option {
     position: relative;
-    padding: 0px 2px;
+    padding: 0 2px;
   }
 
   .dropdown-option:hover {

From d650be5429b822182e4cc34f226f9d345106cfde Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 1 Dec 2016 16:06:47 -0800
Subject: [PATCH 443/800] [local-sync] Refactor tasks and /actually/ fix moving
 between Gmail folders

Moving to between gmail folders (all, spam, trash) or moving to inbox,
involves changing labels /and/ folders, simultaneously. For this I added
a task to perform both operations, and apply labels first before
attempting to move the folder
---
 .../src/local-api/routes/threads.js           | 11 ++++++-
 .../syncback-task-factory.js                  |  2 ++
 .../move-message-to-folder.imap.js            | 12 +++-----
 .../move-thread-to-folder.imap.js             | 19 ++++++------
 .../syncback_tasks/set-message-labels.imap.js | 22 ++------------
 .../set-thread-folder-and-labels.imap.js      | 29 +++++++++++++++++++
 .../syncback_tasks/set-thread-labels.imap.js  | 21 ++------------
 .../syncback_tasks/task-helpers.js            | 29 +++++++++++++++++++
 8 files changed, 89 insertions(+), 56 deletions(-)
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js

diff --git a/packages/local-sync/src/local-api/routes/threads.js b/packages/local-sync/src/local-api/routes/threads.js
index 3814535d5..1aedb0f27 100644
--- a/packages/local-sync/src/local-api/routes/threads.js
+++ b/packages/local-sync/src/local-api/routes/threads.js
@@ -76,7 +76,16 @@ module.exports = (server) => {
     },
     handler: (request, reply) => {
       const payload = request.payload
-      if (payload.folder_id || payload.folder) {
+      if ((payload.folder_id || payload.folder) && (payload.label_ids || payload.labels)) {
+        createSyncbackRequest(request, reply, {
+          type: "SetThreadFolderAndLabels",
+          props: {
+            folderId: payload.folder_id || payload.folder,
+            labelIds: payload.label_ids || payload.labels,
+            threadId: request.params.id,
+          },
+        })
+      } else if (payload.folder_id || payload.folder) {
         createSyncbackRequest(request, reply, {
           type: "MoveThreadToFolder",
           props: {
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
index a2453aa4b..bcd8fc585 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
@@ -11,6 +11,8 @@ class SyncbackTaskFactory {
         Task = require('./syncback_tasks/move-thread-to-folder.imap'); break;
       case "SetThreadLabels":
         Task = require('./syncback_tasks/set-thread-labels.imap'); break;
+      case "SetThreadFolderAndLabels":
+        Task = require('./syncback_tasks/set-thread-folder-and-labels.imap'); break;
       case "MarkThreadAsRead":
         Task = require('./syncback_tasks/mark-thread-as-read.imap'); break;
       case "MarkThreadAsUnread":
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js
index 4f342d047..5beecb32f 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js
@@ -6,16 +6,12 @@ class MoveMessageToFolderIMAP extends SyncbackTask {
     return `MoveMessageToFolder`;
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
-    const toFolderId = this.syncbackRequestObject().props.folderId
+    const targetFolderId = this.syncbackRequestObject().props.folderId
 
-    return TaskHelpers.openMessageBox({messageId, db, imap})
-      .then(({box, message}) => {
-        return db.Folder.findById(toFolderId).then((newFolder) => {
-          return box.moveFromBox(message.folderImapUID, newFolder.name)
-        })
-      })
+    const {box, message} = await TaskHelpers.openMessageBox({messageId, db, imap})
+    return TaskHelpers.moveMessageToFolder({db, box, message, targetFolderId})
   }
 }
 module.exports = MoveMessageToFolderIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
index 58f92d679..1ab698215 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
@@ -6,17 +6,18 @@ class MoveThreadToFolderIMAP extends SyncbackTask {
     return `MoveThreadToFolder`;
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
-    const toFolderId = this.syncbackRequestObject().props.folderId
+    const targetFolderId = this.syncbackRequestObject().props.folderId
 
-    const eachMsg = ({message, box}) => {
-      return db.Folder.findById(toFolderId).then((category) => {
-        return box.moveFromBox(message.folderImapUID, category.name)
-      })
-    }
-
-    return TaskHelpers.forEachMessageInThread({threadId, db, imap, callback: eachMsg})
+    return TaskHelpers.forEachMessageInThread({
+      db,
+      imap,
+      threadId,
+      async callback({message, box}) {
+        return TaskHelpers.moveMessageToFolder({db, box, message, targetFolderId})
+      },
+    })
   }
 }
 module.exports = MoveThreadToFolderIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js
index cc0cabc63..19aa1bf09 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js
@@ -6,28 +6,12 @@ class SetMessageLabelsIMAP extends SyncbackTask {
     return `SetMessageLabels`;
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
     const labelIds = this.syncbackRequestObject().props.labelIds
 
-    return TaskHelpers.openMessageBox({messageId, db, imap})
-    .then(({box, message}) => {
-      if (!labelIds || labelIds.length === 0) {
-        return message.getLabels().then((labels) => {
-          const labelNames = labels.map(({name}) => name)
-          return box.removeLabels(message.folderImapUID, labelNames)
-        })
-      }
-      return db.Label.findAll({
-        where: {
-          id: {'in': labelIds},
-        },
-      })
-      .then((labels) => {
-        const labelNames = labels.map(({name}) => name)
-        return box.setLabels(message.folderImapUID, labelNames)
-      })
-    })
+    const {box, message} = await TaskHelpers.openMessageBox({messageId, db, imap})
+    return TaskHelpers.setMessageLabels({message, db, box, labelIds})
   }
 }
 module.exports = SetMessageLabelsIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js
new file mode 100644
index 000000000..15f0adc0d
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js
@@ -0,0 +1,29 @@
+const SyncbackTask = require('./syncback-task')
+const TaskHelpers = require('./task-helpers')
+
+class SetThreadFolderAndLabelsIMAP extends SyncbackTask {
+  description() {
+    return `SetThreadFolderAndLabels`;
+  }
+
+  async run(db, imap) {
+    const threadId = this.syncbackRequestObject().props.threadId
+    const labelIds = this.syncbackRequestObject().props.labelIds
+    const targetFolderId = this.syncbackRequestObject().props.folderId
+
+    // Ben TODO this is super inefficient because it makes IMAP requests
+    // one UID at a time, rather than gathering all the UIDs and making
+    // a single removeLabels call.
+    return TaskHelpers.forEachMessageInThread({
+      db,
+      imap,
+      threadId,
+      async callback({message, box}) {
+        await TaskHelpers.setMessageLabels({message, db, box, labelIds})
+        return TaskHelpers.moveMessageToFolder({db, box, message, targetFolderId})
+      },
+    })
+  }
+}
+module.exports = SetThreadFolderAndLabelsIMAP
+
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
index 9aaa2695a..4bc6c1351 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
@@ -10,23 +10,6 @@ class SetThreadLabelsIMAP extends SyncbackTask {
     const threadId = this.syncbackRequestObject().props.threadId
     const labelIds = this.syncbackRequestObject().props.labelIds
 
-    if (!labelIds || labelIds.length === 0) {
-      return TaskHelpers.forEachMessageInThread({
-        db,
-        imap,
-        threadId,
-        callback: ({message, box}) => {
-          return message.getLabels().then((labels) => {
-            const labelIdentifiers = labels.map(label => label.imapLabelIdentifier())
-            return box.removeLabels(message.folderImapUID, labelIdentifiers)
-          })
-        },
-      })
-    }
-
-    const labels = await db.Label.findAll({where: {id: labelIds}});
-    const labelIdentifiers = labels.map(label => label.imapLabelIdentifier());
-
     // Ben TODO this is super inefficient because it makes IMAP requests
     // one UID at a time, rather than gathering all the UIDs and making
     // a single removeLabels call.
@@ -34,8 +17,8 @@ class SetThreadLabelsIMAP extends SyncbackTask {
       db,
       imap,
       threadId,
-      callback: ({message, box}) => {
-        return box.setLabels(message.folderImapUID, labelIdentifiers)
+      async callback({message, box}) {
+        return TaskHelpers.setMessageLabels({message, db, box, labelIds})
       },
     })
   }
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/task-helpers.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/task-helpers.js
index 2326eb48f..0577b09dc 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/task-helpers.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/task-helpers.js
@@ -33,5 +33,34 @@ const TaskHelpers = {
       })
     }))
   },
+
+  async moveMessageToFolder({db, box, message, targetFolderId}) {
+    if (!targetFolderId) {
+      throw new Error('TaskHelpers.moveMessageToFolder: targetFolderId is required')
+    }
+    if (targetFolderId === message.folderId) {
+      return Promise.resolve()
+    }
+    const targetFolder = await db.Folder.findById(targetFolderId)
+    if (!targetFolder) {
+      return Promise.resolve()
+    }
+    return box.moveFromBox(message.folderImapUID, targetFolder.name)
+  },
+
+  async setMessageLabels({db, box, message, labelIds}) {
+    if (!labelIds || labelIds.length === 0) {
+      const labels = await message.getLabels()
+      if (labels.length === 0) {
+        return Promise.resolve()
+      }
+      const labelIdentifiers = labels.map(label => label.imapLabelIdentifier())
+      return box.removeLabels(message.folderImapUID, labelIdentifiers)
+    }
+
+    const labels = await db.Label.findAll({where: {id: labelIds}});
+    const labelIdentifiers = labels.map(label => label.imapLabelIdentifier());
+    return box.setLabels(message.folderImapUID, labelIdentifiers)
+  },
 }
 module.exports = TaskHelpers

From 15cfe2cec0ba95f28fca5c5b6b58a4dfcf868507 Mon Sep 17 00:00:00 2001
From: Karim Hamidou 
Date: Thu, 1 Dec 2016 16:49:09 -0800
Subject: [PATCH 444/800] Strip the '[Gmail]/' prefix from folder names.

Summary:
T7253 has two related parts:
1. Stripping the '[Gmail]/' prefix from any canonical Gmail folders
2. Handling nested IMAP folders.

This diff fixes 1. Changes for 2. are forthcoming.

Test Plan: Tested manually. Checked that the part was stripped from the N1 folder list.

Reviewers: juan

Reviewed By: juan

Differential Revision: https://phab.nylas.com/D3475
---
 packages/local-sync/src/models/folder.js           |  3 ++-
 packages/local-sync/src/models/label.js            |  4 +++-
 packages/local-sync/src/shared/imap-paths-utils.js | 10 ++++++++++
 3 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 packages/local-sync/src/shared/imap-paths-utils.js

diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.js
index 9272dc610..8d892a312 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.js
@@ -1,4 +1,5 @@
 const {DatabaseTypes: {JSONType}} = require('isomorphic-core');
+const {formatImapPath} = require('../shared/imap-paths-utils');
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('folder', {
@@ -32,7 +33,7 @@ module.exports = (sequelize, Sequelize) => {
           account_id: this.accountId,
           object: 'folder',
           name: this.role,
-          display_name: this.name,
+          display_name: formatImapPath(this.name),
         };
       },
     },
diff --git a/packages/local-sync/src/models/label.js b/packages/local-sync/src/models/label.js
index 5960362a7..5f1315bd9 100644
--- a/packages/local-sync/src/models/label.js
+++ b/packages/local-sync/src/models/label.js
@@ -1,3 +1,5 @@
+const {formatImapPath} = require('../shared/imap-paths-utils');
+
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('label', {
     id: { type: Sequelize.STRING(65), primaryKey: true },
@@ -53,7 +55,7 @@ module.exports = (sequelize, Sequelize) => {
           account_id: this.accountId,
           object: 'label',
           name: this.role,
-          display_name: this.name,
+          display_name: formatImapPath(this.name),
         };
       },
     },
diff --git a/packages/local-sync/src/shared/imap-paths-utils.js b/packages/local-sync/src/shared/imap-paths-utils.js
new file mode 100644
index 000000000..2b4c84f6d
--- /dev/null
+++ b/packages/local-sync/src/shared/imap-paths-utils.js
@@ -0,0 +1,10 @@
+function formatImapPath(pathStr) {
+  if (!pathStr) {
+    throw new Error("Can not format an empty path!");
+  }
+
+  const s = pathStr.replace(/^\[Gmail\]\//, '');
+  return s;
+}
+
+module.exports = {formatImapPath}

From 77122694021c38230fc9c8a62727079422a9f28d Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Thu, 1 Dec 2016 18:41:40 -0800
Subject: [PATCH 445/800] [*] fix(deltas): Cloud-API not filtering deltas at
 all, refactor a few things
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Don’t need functions in delta.js which must be called to return promsies. Fun of promsies is that you don’t need to care when they’re built to attach a .then.

- Make boundary between route handler and delta stream builder more explicit, don’t do query parsing in helpers, always reply from handler.

- Remove pushJSON extension to outputStream which never actually received JSON.

- Remove `takeUntil` - disposing of the downstream observable should dispose of all the merged/upstream observables

- Rename inflate => stringify since the returned value is a string not an object.

- Remove support for delta streams with no cursors. Don’t think this was supposed to be a feature.

- Add accountId to Transaction models

- Make database hooks shared in isomorphic core
---
 packages/isomorphic-core/index.js             |   4 +-
 .../src/delta-stream-builder.js               | 102 +++++++-----------
 .../src}/hook-increment-version-on-save.js    |   0
 .../src}/hook-transaction-log.js              |  24 +++--
 .../isomorphic-core/src/models/transaction.js |   2 +-
 .../local-sync/src/local-api/routes/delta.js  |  42 +++++---
 .../src/shared/local-database-connector.js    |  12 ++-
 7 files changed, 97 insertions(+), 89 deletions(-)
 rename packages/{local-sync/src/shared => isomorphic-core/src}/hook-increment-version-on-save.js (100%)
 rename packages/{local-sync/src/shared => isomorphic-core/src}/hook-transaction-log.js (62%)

diff --git a/packages/isomorphic-core/index.js b/packages/isomorphic-core/index.js
index 2a7e6407b..67721a8b6 100644
--- a/packages/isomorphic-core/index.js
+++ b/packages/isomorphic-core/index.js
@@ -10,5 +10,7 @@ module.exports = {
   PromiseUtils: require('./src/promise-utils'),
   DatabaseTypes: require('./src/database-types'),
   loadModels: require('./src/load-models'),
-  deltaStreamBuilder: require('./src/delta-stream-builder'),
+  DeltaStreamBuilder: require('./src/delta-stream-builder'),
+  HookTransactionLog: require('./src/hook-transaction-log'),
+  HookIncrementVersionOnSave: require('./src/hook-increment-version-on-save'),
 }
diff --git a/packages/isomorphic-core/src/delta-stream-builder.js b/packages/isomorphic-core/src/delta-stream-builder.js
index 068397e23..edbd5dce0 100644
--- a/packages/isomorphic-core/src/delta-stream-builder.js
+++ b/packages/isomorphic-core/src/delta-stream-builder.js
@@ -2,83 +2,63 @@ const _ = require('underscore');
 const Rx = require('rx')
 const stream = require('stream');
 
-function keepAlive(request) {
-  const until = Rx.Observable.fromCallback(request.on)("disconnect")
-  return Rx.Observable.interval(1000).map(() => "\n").takeUntil(until)
-}
+function stringifyTransactions(db, transactions = []) {
+  const transactionJSONs = transactions.map((t) => t.toJSON())
+  transactionJSONs.forEach((t) => { t.cursor = t.id });
 
-function inflateTransactions(db, transactionModels = []) {
-  let models = transactionModels;
-  if (!(_.isArray(models))) { models = [transactionModels] }
-  const transactions = models.map((mod) => mod.toJSON())
-  transactions.forEach((t) => { t.cursor = t.id });
-  const byModel = _.groupBy(transactions, "object");
-  const byObjectIds = _.groupBy(transactions, "objectId");
+  const byModel = _.groupBy(transactionJSONs, "object");
+  const byObjectIds = _.groupBy(transactionJSONs, "objectId");
 
-  return Promise.all(Object.keys(byModel).map((object) => {
-    const ids = _.pluck(byModel[object], "objectId");
-    const modelConstructorName = object.charAt(0).toUpperCase() + object.slice(1);
+  return Promise.all(Object.keys(byModel).map((modelName) => {
+    const modelIds = byModel[modelName].map(t => t.objectId);
+    const modelConstructorName = modelName.charAt(0).toUpperCase() + modelName.slice(1);
     const ModelKlass = db[modelConstructorName]
+
     let includes = [];
     if (ModelKlass.requiredAssociationsForJSON) {
       includes = ModelKlass.requiredAssociationsForJSON(db)
     }
-    return ModelKlass.findAll({where: {id: ids}, include: includes})
-    .then((objs = []) => {
-      for (const model of objs) {
-        const tsForId = byObjectIds[model.id];
-        if (!tsForId || tsForId.length === 0) { continue; }
-        for (const t of tsForId) { t.attributes = model.toJSON(); }
+    return ModelKlass.findAll({
+      where: {id: modelIds},
+      include: includes,
+    }).then((models) => {
+      for (const model of models) {
+        const transactionsForModel = byObjectIds[model.id];
+        for (const t of transactionsForModel) {
+          t.attributes = model.toJSON();
+        }
       }
-    })
-  })).then(() => `${transactions.map(JSON.stringify).join("\n")}\n`)
-}
-
-function createOutputStream() {
-  const outputStream = stream.Readable();
-  outputStream._read = () => { return };
-  outputStream.pushJSON = (msg) => {
-    const jsonMsg = typeof msg === 'string' ? msg : JSON.stringify(msg);
-    outputStream.push(jsonMsg);
-  }
-  return outputStream
-}
-
-function initialTransactions(db, request) {
-  const cursor = (request.query || {}).cursor;
-  const where = cursor ? {id: {$gt: cursor}} : {createdAt: {$gte: new Date()}}
-  return db.Transaction
-           .streamAll({where})
-           .flatMap((objs) => inflateTransactions(db, objs))
-}
-
-function inflatedIncomingTransaction(db, request, transactionSource) {
-  return transactionSource.flatMap((t) => inflateTransactions(db, [t]))
+    });
+  })).then(() => {
+    return `${transactionJSONs.map(JSON.stringify).join("\n")}\n`;
+  });
 }
 
 module.exports = {
-  buildStream(request, dbSource, transactionSource) {
-    const outputStream = createOutputStream();
+  buildStream(request, {databasePromise, cursor, accountId, deltasSource}) {
+    return databasePromise.then((db) => {
+      const initialSource = db.Transaction.streamAll({where: { id: {$gt: cursor}, accountId }});
 
-    dbSource().then((db) => {
       const source = Rx.Observable.merge(
-        inflatedIncomingTransaction(db, request, transactionSource(db, request)),
-        initialTransactions(db, request),
-        keepAlive(request)
-      ).subscribe(outputStream.pushJSON)
+        initialSource.flatMap((t) => stringifyTransactions(db, t)),
+        deltasSource.flatMap((t) => stringifyTransactions(db, [t])),
+        Rx.Observable.interval(1000).map(() => "\n")
+      )
 
-      request.on("disconnect", source.dispose.bind(source));
+      const outputStream = stream.Readable();
+      outputStream._read = () => { return };
+      source.subscribe((str) => outputStream.push(str))
+      request.on("disconnect", () => source.dispose());
+
+      return outputStream;
     });
-
-    return outputStream
   },
 
-  lastTransactionReply(dbSource, reply) {
-    dbSource().then((db) => {
-      db.Transaction.findOne({order: [['id', 'DESC']]})
-      .then((t) => {
-        reply({cursor: (t || {}).id})
-      })
-    })
+  buildCursor({databasePromise}) {
+    return databasePromise.then(({Transaction}) => {
+      return Transaction.findOne({order: [['id', 'DESC']]}).then((t) => {
+        return (t || {}).id;
+      });
+    });
   },
 }
diff --git a/packages/local-sync/src/shared/hook-increment-version-on-save.js b/packages/isomorphic-core/src/hook-increment-version-on-save.js
similarity index 100%
rename from packages/local-sync/src/shared/hook-increment-version-on-save.js
rename to packages/isomorphic-core/src/hook-increment-version-on-save.js
diff --git a/packages/local-sync/src/shared/hook-transaction-log.js b/packages/isomorphic-core/src/hook-transaction-log.js
similarity index 62%
rename from packages/local-sync/src/shared/hook-transaction-log.js
rename to packages/isomorphic-core/src/hook-transaction-log.js
index 903837a7c..d3b1feac6 100644
--- a/packages/local-sync/src/shared/hook-transaction-log.js
+++ b/packages/isomorphic-core/src/hook-transaction-log.js
@@ -1,7 +1,6 @@
 const _ = require('underscore')
-const TransactionConnector = require('./transaction-connector')
 
-module.exports = (db, sequelize) => {
+module.exports = (db, sequelize, {only, onCreatedTransaction} = {}) => {
   if (!db.Transaction) {
     throw new Error("Cannot enable transaction logging, there is no Transaction model class in this database.")
   }
@@ -15,20 +14,33 @@ module.exports = (db, sequelize) => {
 
   const transactionLogger = (event) => {
     return ({dataValues, _changed, $modelOptions}) => {
+      let name = $modelOptions.name.singular;
+      if (name === 'metadatum') {
+        name = 'metadata';
+      }
+
+      if (only && !only.includes(name)) {
+        return;
+      }
+
       const changedFields = Object.keys(_changed)
       if ((isTransaction($modelOptions) || allIgnoredFields(changedFields))) {
         return;
       }
 
+      const accountId = db.accountId ? db.accountId : dataValues.accountId;
+      if (!accountId) {
+        throw new Error("Assertion failure: Cannot create a transaction - could not resolve accountId.")
+      }
+
       const transactionData = Object.assign({event}, {
-        object: $modelOptions.name.singular,
+        object: name,
         objectId: dataValues.id,
+        accountId: accountId,
         changedFields: changedFields,
       });
 
-      db.Transaction.create(transactionData).then((transaction) => {
-        TransactionConnector.notifyDelta(db.accountId, transaction);
-      })
+      db.Transaction.create(transactionData).then(onCreatedTransaction)
     }
   }
 
diff --git a/packages/isomorphic-core/src/models/transaction.js b/packages/isomorphic-core/src/models/transaction.js
index 4a0ca1592..25b4a830f 100644
--- a/packages/isomorphic-core/src/models/transaction.js
+++ b/packages/isomorphic-core/src/models/transaction.js
@@ -5,6 +5,7 @@ module.exports = (sequelize, Sequelize) => {
     event: Sequelize.STRING,
     object: Sequelize.STRING,
     objectId: Sequelize.STRING,
+    accountId: Sequelize.STRING,
     changedFields: JSONARRAYType('changedFields'),
   }, {
     instanceMethods: {
@@ -14,7 +15,6 @@ module.exports = (sequelize, Sequelize) => {
           event: this.event,
           object: this.object,
           objectId: `${this.objectId}`,
-          changedFields: this.changedFields,
         }
       },
     },
diff --git a/packages/local-sync/src/local-api/routes/delta.js b/packages/local-sync/src/local-api/routes/delta.js
index 9c291bbc5..2fdbfd1a4 100644
--- a/packages/local-sync/src/local-api/routes/delta.js
+++ b/packages/local-sync/src/local-api/routes/delta.js
@@ -1,31 +1,41 @@
+const Joi = require('joi');
 const TransactionConnector = require('../../shared/transaction-connector')
-const {deltaStreamBuilder} = require('isomorphic-core')
-
-function transactionSource(db, request) {
-  const accountId = request.auth.credentials.id;
-  return TransactionConnector.getObservableForAccountId(accountId)
-}
-
-function dbSource(request) {
-  return request.getAccountDatabase.bind(request)
-}
+const {DeltaStreamBuilder} = require('isomorphic-core')
 
 module.exports = (server) => {
   server.route({
     method: 'GET',
     path: '/delta/streaming',
+    config: {
+      validate: {
+        query: {
+          cursor: Joi.string().required(),
+        },
+      },
+    },
     handler: (request, reply) => {
-      const outputStream = deltaStreamBuilder.buildStream(request,
-          dbSource(request), transactionSource)
-      reply(outputStream)
+      const account = request.auth.credentials;
+
+      DeltaStreamBuilder.buildStream(request, {
+        cursor: request.query.cursor,
+        accountId: account.id,
+        databasePromise: request.getAccountDatabase(),
+        deltasSource: TransactionConnector.getObservableForAccountId(account.id),
+      }).then((stream) => {
+        reply(stream)
+      });
     },
   });
 
   server.route({
     method: 'POST',
     path: '/delta/latest_cursor',
-    handler: (request, reply) =>
-      deltaStreamBuilder.lastTransactionReply(dbSource(request), reply)
-    ,
+    handler: (request, reply) => {
+      DeltaStreamBuilder.buildCursor({
+        databasePromise: request.getAccountDatabase(),
+      }).then((cursor) => {
+        reply({cursor})
+      });
+    },
   });
 };
diff --git a/packages/local-sync/src/shared/local-database-connector.js b/packages/local-sync/src/shared/local-database-connector.js
index b9988f9d9..e6a8494f1 100644
--- a/packages/local-sync/src/shared/local-database-connector.js
+++ b/packages/local-sync/src/shared/local-database-connector.js
@@ -1,9 +1,8 @@
 const Sequelize = require('sequelize');
 const fs = require('fs');
 const path = require('path');
-const {loadModels, PromiseUtils} = require('isomorphic-core');
-const HookTransactionLog = require('./hook-transaction-log');
-const HookIncrementVersionOnSave = require('./hook-increment-version-on-save');
+const {loadModels, PromiseUtils, HookIncrementVersionOnSave, HookTransactionLog} = require('isomorphic-core');
+const TransactionConnector = require('./transaction-connector')
 
 require('./database-extensions'); // Extends Sequelize on require
 
@@ -34,7 +33,12 @@ class LocalDatabaseConnector {
       modelDirs: [path.resolve(__dirname, '..', 'models')],
     })
 
-    HookTransactionLog(db, newSequelize);
+    HookTransactionLog(db, newSequelize, {
+      onCreatedTransaction: (transaction) => {
+        TransactionConnector.notifyDelta(db.accountId, transaction);
+      },
+    });
+
     HookIncrementVersionOnSave(db, newSequelize);
 
     db.sequelize = newSequelize;

From 5ff7a921977a1732a153d5817851a334ffcafba8 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Thu, 1 Dec 2016 18:54:26 -0800
Subject: [PATCH 446/800] [*] fix(deltas): 0 as first cursor, deltas from redis
 are already JSON

---
 packages/isomorphic-core/src/delta-stream-builder.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/isomorphic-core/src/delta-stream-builder.js b/packages/isomorphic-core/src/delta-stream-builder.js
index edbd5dce0..5b9b91491 100644
--- a/packages/isomorphic-core/src/delta-stream-builder.js
+++ b/packages/isomorphic-core/src/delta-stream-builder.js
@@ -3,7 +3,7 @@ const Rx = require('rx')
 const stream = require('stream');
 
 function stringifyTransactions(db, transactions = []) {
-  const transactionJSONs = transactions.map((t) => t.toJSON())
+  const transactionJSONs = transactions.map((t) => (t.toJSON ? t.toJSON() : t))
   transactionJSONs.forEach((t) => { t.cursor = t.id });
 
   const byModel = _.groupBy(transactionJSONs, "object");
@@ -57,7 +57,7 @@ module.exports = {
   buildCursor({databasePromise}) {
     return databasePromise.then(({Transaction}) => {
       return Transaction.findOne({order: [['id', 'DESC']]}).then((t) => {
-        return (t || {}).id;
+        return t ? t.id : 0;
       });
     });
   },

From 40791028d5c51e8ba3cccba1ba6cf309207fbff6 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Thu, 1 Dec 2016 18:28:14 -0500
Subject: [PATCH 447/800] [cloud-api] Update cloud-api readme & remove unused
 files

[cloud-api] move Dockerfile and pm2 files into cloud-api

[*] additions to .gitignore

[cloud-workers]: move pm2 file into cloud-workers

updates to readme

Revert "[cloud-workers]: move pm2 file into cloud-workers"

This reverts commit 952292124b6e8dfa304c5fbd945c0b00f427e154.

Revert "[cloud-api] move Dockerfile and pm2 files into cloud-api"

This reverts commit aa92f691bf991848a51c00a2d01a5848f20a35a2.

Update deploy readme
---
 .arcconfig | 4 ----
 .env       | 5 -----
 .gitignore | 5 +++++
 3 files changed, 5 insertions(+), 9 deletions(-)
 delete mode 100644 .arcconfig
 delete mode 100644 .env

diff --git a/.arcconfig b/.arcconfig
deleted file mode 100644
index 00ab68a97..000000000
--- a/.arcconfig
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "project_id" : "K2",
-  "conduit_uri" : "https://phab.nylas.com/"
-}
diff --git a/.env b/.env
deleted file mode 100644
index 431b7e615..000000000
--- a/.env
+++ /dev/null
@@ -1,5 +0,0 @@
-DB_ENCRYPTION_ALGORITHM='aes-256-ctr'
-DB_ENCRYPTION_PASSWORD='d6F3Efeq'
-GMAIL_CLIENT_ID='271342407743-nibas08fua1itr1utq9qjladbkv3esdm.apps.googleusercontent.com'
-GMAIL_CLIENT_SECRET='WhmxErj-ei6vJXLocNhBbfBF'
-GMAIL_REDIRECT_URL='http://localhost:5100/auth/gmail/oauthcallback'
diff --git a/.gitignore b/.gitignore
index cd78b26fa..0f53da4f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,8 @@
+.arcconfig
+.arclint
+arclib
+*.swp
+*~
 .DS_Store
 node_modules
 dump.rdb

From 5ac3fe9ea857f28bdbedcfb7d1d6a65488794ce4 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Fri, 2 Dec 2016 11:27:11 -0800
Subject: [PATCH 448/800] [local-sync] Fix starred role on folders

---
 .../local-sync/src/local-sync-worker/imap/fetch-folder-list.js  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index ba09ecc56..4a9bced73 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -50,7 +50,7 @@ class FetchFolderList {
         '\\Trash': 'trash',
         '\\All': 'all',
         '\\Important': 'important',
-        '\\Flagged': 'flagged',
+        '\\Flagged': 'starred',
         '\\Inbox': 'inbox',
       }[attrib];
       if (role) {

From f1491fa4cca83a8c3828f3cc91492764ae986e48 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Fri, 2 Dec 2016 12:09:35 -0800
Subject: [PATCH 449/800] fix observable disposable

---
 packages/isomorphic-core/src/delta-stream-builder.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/isomorphic-core/src/delta-stream-builder.js b/packages/isomorphic-core/src/delta-stream-builder.js
index 5b9b91491..12f4f8a75 100644
--- a/packages/isomorphic-core/src/delta-stream-builder.js
+++ b/packages/isomorphic-core/src/delta-stream-builder.js
@@ -47,8 +47,8 @@ module.exports = {
 
       const outputStream = stream.Readable();
       outputStream._read = () => { return };
-      source.subscribe((str) => outputStream.push(str))
-      request.on("disconnect", () => source.dispose());
+      const disposable = source.subscribe((str) => outputStream.push(str))
+      request.on("disconnect", () => disposable.dispose());
 
       return outputStream;
     });

From b36f61812d3a5140c5ccf5e2da20665b62537ac1 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Fri, 2 Dec 2016 12:14:09 -0800
Subject: [PATCH 450/800] [local-sync]: Don't create or sync \NonExistent or
 \NoSelect folders

---
 .../src/local-sync-worker/imap/fetch-folder-list.js          | 5 +++++
 .../src/local-sync-worker/imap/fetch-messages-in-folder.js   | 3 +--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index 4a9bced73..f3eb51eb2 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -89,6 +89,11 @@ class FetchFolderList {
         });
       }
 
+      const lowerCaseAttrs = box.attribs.map(attr => attr.toLowerCase())
+      if (lowerCaseAttrs.includes('\\noselect') || lowerCaseAttrs.includes('\\nonexistent')) {
+        continue;
+      }
+
       let category = categories.find((cat) => cat.name === boxName);
       if (!category) {
         const role = this._roleByAttr(box);
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index d6c15f817..b5d5c30bd 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -261,10 +261,9 @@ class FetchMessagesInFolder {
         }, `FetchMessagesInFolder: Queued new message for processing`)
       }
     } catch (err) {
-      this._logger.error({
+      this._logger.error(err, {
         imapMessage,
         desiredParts,
-        underlying: err.toString(),
       }, `FetchMessagesInFolder: Could not build message`)
     }
   }

From b06566d8bdf0d0929edc11b08af5dd444f8ff891 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Fri, 2 Dec 2016 16:14:30 -0500
Subject: [PATCH 451/800] [local-sync] make syncbackRequest objects N1-ready

---
 packages/local-sync/src/local-api/route-helpers.js | 2 ++
 packages/local-sync/src/models/syncbackRequest.js  | 7 +++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/local-sync/src/local-api/route-helpers.js b/packages/local-sync/src/local-api/route-helpers.js
index 8ed888dc2..6dc9cc620 100644
--- a/packages/local-sync/src/local-api/route-helpers.js
+++ b/packages/local-sync/src/local-api/route-helpers.js
@@ -3,6 +3,8 @@ const Serialization = require('./serialization');
 module.exports = {
   createSyncbackRequest: function createSyncbackRequest(request, reply, syncRequestArgs) {
     request.getAccountDatabase().then((db) => {
+      const accountId = request.auth.credentials.id;
+      syncRequestArgs.accountId = accountId
       db.SyncbackRequest.create(syncRequestArgs).then((syncbackRequest) => {
         reply(Serialization.jsonStringify(syncbackRequest))
       })
diff --git a/packages/local-sync/src/models/syncbackRequest.js b/packages/local-sync/src/models/syncbackRequest.js
index bb163ba44..6ffe3e0d0 100644
--- a/packages/local-sync/src/models/syncbackRequest.js
+++ b/packages/local-sync/src/models/syncbackRequest.js
@@ -10,15 +10,18 @@ module.exports = (sequelize, Sequelize) => {
     },
     error: JSONType('error'),
     props: JSONType('props'),
+    accountId: { type: Sequelize.STRING, allowNull: false },
   }, {
     instanceMethods: {
       toJSON: function toJSON() {
         return {
           id: `${this.id}`,
           type: this.type,
-          status: this.status,
-          error: this.error,
+          error: JSON.stringify(this.error || {}),
           props: this.props,
+          status: this.status,
+          object: 'providerSyncbackRequest',
+          account_id: this.accountId,
         }
       },
     },

From d9f3edad0f7dcff82dab9587f71771a859b7f6d5 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Fri, 2 Dec 2016 16:25:42 -0500
Subject: [PATCH 452/800] [isomorphic-core] ignore folder deltas due to just
 syncState version issues

---
 packages/isomorphic-core/src/hook-transaction-log.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/isomorphic-core/src/hook-transaction-log.js b/packages/isomorphic-core/src/hook-transaction-log.js
index d3b1feac6..df6e97a8c 100644
--- a/packages/isomorphic-core/src/hook-transaction-log.js
+++ b/packages/isomorphic-core/src/hook-transaction-log.js
@@ -9,7 +9,7 @@ module.exports = (db, sequelize, {only, onCreatedTransaction} = {}) => {
   }
 
   const allIgnoredFields = (changedFields) => {
-    return _.isEqual(changedFields, ['syncState']);
+    return _.isEqual(changedFields, ['syncState', 'updatedAt', 'version']) || _.isEqual(changedFields, ['syncState']);
   }
 
   const transactionLogger = (event) => {

From 9752eea9f70689e8f353b21230cc4fc24484987c Mon Sep 17 00:00:00 2001
From: Christine Spang 
Date: Fri, 2 Dec 2016 14:10:54 -0800
Subject: [PATCH 453/800] [local-sync] Set thread IDs to the ID of a message in
 the thread

Since message IDs are now static but there's no good way to generate
static thread IDs while syncing an account from newest message first,
we give threads the ID of any message on that thread and, when setting
metadata, look up the local thread ID by first going through the
message table.
---
 packages/local-sync/src/models/thread.js                     | 1 +
 .../local-sync/src/new-message-processor/detect-thread.js    | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/packages/local-sync/src/models/thread.js b/packages/local-sync/src/models/thread.js
index 7342c3142..6b588ee6a 100644
--- a/packages/local-sync/src/models/thread.js
+++ b/packages/local-sync/src/models/thread.js
@@ -2,6 +2,7 @@ const {DatabaseTypes: {JSONARRAYType}} = require('isomorphic-core');
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('thread', {
+    id: { type: Sequelize.STRING(65), primaryKey: true },
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     remoteThreadId: Sequelize.STRING,
diff --git a/packages/local-sync/src/new-message-processor/detect-thread.js b/packages/local-sync/src/new-message-processor/detect-thread.js
index e5ed3b42c..8cf15d8fb 100644
--- a/packages/local-sync/src/new-message-processor/detect-thread.js
+++ b/packages/local-sync/src/new-message-processor/detect-thread.js
@@ -95,6 +95,11 @@ function detectThread({db, message}) {
 
     // update the basic properties of the thread
     thread.accountId = message.accountId;
+    // Threads may, locally, have the ID of any message within the thread
+    // (message IDs are globally unique, even across accounts!)
+    if (!thread.id) {
+      thread.id = `t:${message.id}`
+    }
 
     // update the participants on the thread
     const threadParticipants = [].concat(thread.participants);

From 2ba326dfc6490d4d8e96ee44f6851c33f6c2c94a Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Fri, 2 Dec 2016 17:33:51 -0500
Subject: [PATCH 454/800] [local-sync] only enable logging in dev mode of N1

---
 packages/local-sync/src/local-sync-worker/sync-worker.js | 1 -
 packages/local-sync/src/shared/logger.es6                | 9 +++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index 04500ca6d..d80532fd8 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -176,7 +176,6 @@ class SyncWorker {
       return;
     }
 
-    console.log(this._account)
     this._logger.info({reason}, `SyncWorker: Account sync started`)
 
     try {
diff --git a/packages/local-sync/src/shared/logger.es6 b/packages/local-sync/src/shared/logger.es6
index f394a8c48..eae42aeb2 100644
--- a/packages/local-sync/src/shared/logger.es6
+++ b/packages/local-sync/src/shared/logger.es6
@@ -1,6 +1,12 @@
 const _ = require('underscore')
 
+let ENABLE_LOGGING = true;
+
 function Logger(boundArgs = {}) {
+  if (NylasEnv && !NylasEnv.inDevMode()) {
+    ENABLE_LOGGING = false
+  }
+
   if (!_.isObject(boundArgs)) {
     throw new Error('Logger: Bound arguments must be an object')
   }
@@ -8,6 +14,9 @@ function Logger(boundArgs = {}) {
   const loggerFns = ['log', 'info', 'warn', 'error']
   loggerFns.forEach((logFn) => {
     logger[logFn] = (first, ...args) => {
+      if (!ENABLE_LOGGING && logFn !== "error") {
+        return () => {}
+      }
       if (first instanceof Error || !_.isObject(first)) {
         if (_.isEmpty(boundArgs)) {
           return console[logFn](first, ...args)

From 6a51036e4837303c0c438bd18d81f86343751a04 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Tue, 29 Nov 2016 16:38:21 -0800
Subject: [PATCH 455/800] [local-sync, iso-core, cloud-core] feat(send): add
 multi-send support

Also renames JSONType() -> buildJSONColumnOptions() and
JSONARRAYType() -> buildJSONARRAYColumnOptions() to prevent passing
those return values in as just the type value instead of the entire
options object.
---
 .../isomorphic-core/src/database-types.js     |   4 +-
 .../isomorphic-core/src/models/account.js     |  10 +-
 .../isomorphic-core/src/models/transaction.js |   4 +-
 packages/local-sync/package.json              |   1 +
 packages/local-sync/src/local-api/app.js      |   2 -
 .../local-sync/src/local-api/routes/send.js   | 207 ++++++++++++++++--
 .../local-sync/src/local-api/sending-utils.js | 132 +++++++++++
 .../src/local-api/sendmail-client.js          | 110 ++++++++++
 .../syncback-task-factory.js                  |   6 +-
 .../syncback_tasks/delete-message.imap.js     |  18 ++
 .../perm-delete-message.imap.js               |  24 ++
 .../syncback_tasks/save-sent-message.imap.js  |  15 ++
 packages/local-sync/src/models/folder.js      |   4 +-
 packages/local-sync/src/models/message.js     |  78 ++++++-
 .../local-sync/src/models/syncbackRequest.js  |   6 +-
 packages/local-sync/src/models/thread.js      |  79 ++++++-
 .../new-message-processor/detect-thread.js    |  58 +----
 17 files changed, 654 insertions(+), 104 deletions(-)
 create mode 100644 packages/local-sync/src/local-api/sending-utils.js
 create mode 100644 packages/local-sync/src/local-api/sendmail-client.js
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js

diff --git a/packages/isomorphic-core/src/database-types.js b/packages/isomorphic-core/src/database-types.js
index 7a59c5577..a8f1f3be2 100644
--- a/packages/isomorphic-core/src/database-types.js
+++ b/packages/isomorphic-core/src/database-types.js
@@ -1,7 +1,7 @@
 const Sequelize = require('sequelize');
 
 module.exports = {
-  JSONType: (fieldName, {defaultValue = {}} = {}) => ({
+  buildJSONColumnOptions: (fieldName, {defaultValue = {}} = {}) => ({
     type: Sequelize.TEXT,
     get: function get() {
       const val = this.getDataValue(fieldName);
@@ -14,7 +14,7 @@ module.exports = {
       this.setDataValue(fieldName, JSON.stringify(val));
     },
   }),
-  JSONARRAYType: (fieldName) => ({
+  buildJSONARRAYColumnOptions: (fieldName) => ({
     type: Sequelize.TEXT,
     get: function get() {
       const val = this.getDataValue(fieldName);
diff --git a/packages/isomorphic-core/src/models/account.js b/packages/isomorphic-core/src/models/account.js
index cc43902af..557fd86de 100644
--- a/packages/isomorphic-core/src/models/account.js
+++ b/packages/isomorphic-core/src/models/account.js
@@ -1,5 +1,5 @@
 const crypto = require('crypto');
-const {JSONType, JSONARRAYType} = require('../database-types');
+const {buildJSONColumnOptions, buildJSONARRAYColumnOptions} = require('../database-types');
 
 const {DB_ENCRYPTION_ALGORITHM, DB_ENCRYPTION_PASSWORD} = process.env;
 
@@ -9,16 +9,16 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     provider: Sequelize.STRING,
     emailAddress: Sequelize.STRING,
-    connectionSettings: JSONType('connectionSettings'),
+    connectionSettings: buildJSONColumnOptions('connectionSettings'),
     connectionCredentials: Sequelize.TEXT,
-    syncPolicy: JSONType('syncPolicy'),
-    syncError: JSONType('syncError', {defaultValue: null}),
+    syncPolicy: buildJSONColumnOptions('syncPolicy'),
+    syncError: buildJSONColumnOptions('syncError', {defaultValue: null}),
     firstSyncCompletion: {
       type: Sequelize.STRING(14),
       allowNull: true,
       defaultValue: null,
     },
-    lastSyncCompletions: JSONARRAYType('lastSyncCompletions'),
+    lastSyncCompletions: buildJSONARRAYColumnOptions('lastSyncCompletions'),
   }, {
     indexes: [
       {
diff --git a/packages/isomorphic-core/src/models/transaction.js b/packages/isomorphic-core/src/models/transaction.js
index 25b4a830f..52a99c185 100644
--- a/packages/isomorphic-core/src/models/transaction.js
+++ b/packages/isomorphic-core/src/models/transaction.js
@@ -1,4 +1,4 @@
-const {JSONARRAYType} = require('../database-types');
+const {buildJSONARRAYColumnOptions} = require('../database-types');
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('transaction', {
@@ -6,7 +6,7 @@ module.exports = (sequelize, Sequelize) => {
     object: Sequelize.STRING,
     objectId: Sequelize.STRING,
     accountId: Sequelize.STRING,
-    changedFields: JSONARRAYType('changedFields'),
+    changedFields: buildJSONARRAYColumnOptions('changedFields'),
   }, {
     instanceMethods: {
       toJSON: function toJSON() {
diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index fa7417563..5e866d3f2 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -21,6 +21,7 @@
     "rx": "4.1.0",
     "sequelize": "3.27.0",
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz",
+    "striptags": "2.1.1",
     "underscore": "1.8.3",
     "utf7": "^1.0.2",
     "vision": "4.1.0"
diff --git a/packages/local-sync/src/local-api/app.js b/packages/local-sync/src/local-api/app.js
index f3d4c9729..e0c9d4ea7 100644
--- a/packages/local-sync/src/local-api/app.js
+++ b/packages/local-sync/src/local-api/app.js
@@ -11,8 +11,6 @@ const fs = require('fs');
 const path = require('path');
 const LocalDatabaseConnector = require('../shared/local-database-connector')
 
-if (!global.Logger) { global.Logger = console }
-
 const server = new Hapi.Server({
   connections: {
     router: {
diff --git a/packages/local-sync/src/local-api/routes/send.js b/packages/local-sync/src/local-api/routes/send.js
index fe308f74c..ec861ea5c 100644
--- a/packages/local-sync/src/local-api/routes/send.js
+++ b/packages/local-sync/src/local-api/routes/send.js
@@ -1,36 +1,199 @@
 const Joi = require('joi');
-const nodemailer = require('nodemailer');
 const LocalDatabaseConnector = require('../../shared/local-database-connector');
+const SendingUtils = require('../sending-utils');
+const SendmailClient = require('../sendmail-client');
 
-function toParticipant(payload) {
-  return payload.map((p) => `${p.name} <${p.email}>`).join(',')
+const SEND_TIMEOUT = 1000 * 60; // millliseconds
+
+const recipient = Joi.object().keys({
+  name: Joi.string().required(),
+  email: Joi.string().email().required(),
+  account_id: Joi.string(),
+  client_id: Joi.string(),
+  id: Joi.string(),
+  thirdPartyData: Joi.object(),
+});
+const recipientList = Joi.array().items(recipient);
+
+const respondWithError = (request, reply, error) => {
+  if (!error.httpCode) {
+    error.type = 'apiError';
+    error.httpCode = 500;
+  }
+  request.logger.error('responding with error', error, error.logContext);
+  reply(JSON.stringify(error)).code(error.httpCode);
 }
 
 module.exports = (server) => {
   server.route({
     method: 'POST',
     path: '/send',
-    handler: (request, reply) => { LocalDatabaseConnector.forShared().then((db) => {
-      const accountId = request.auth.credentials.id;
-      db.Account.findById(accountId).then((account) => {
-        const sender = nodemailer.createTransport(account.smtpConfig());
-        const data = request.payload;
+    handler: async (request, reply) => {
+      try {
+        const account = request.auth.credentials;
+        const db = await LocalDatabaseConnector.forAccount(account.id)
+        const draft = await SendingUtils.findOrCreateMessageFromJSON(request.payload, db);
+        // Calculate the response now to prevent errors after the draft has
+        // already been sent.
+        const responseOnSuccess = draft.toJSON();
+        const sender = new SendmailClient(account, request.logger);
+        await sender.send(draft);
+        reply(responseOnSuccess);
+      } catch (err) {
+        respondWithError(request, reply, err);
+      }
+    },
+  });
 
-        const msg = {}
-        for (key of ['from', 'to', 'cc', 'bcc']) {
-          if (data[key]) msg[key] = toParticipant(data[key])
-        }
-        if (!msg.from || msg.from.length === 0) {
-          msg.from = `${account.name} <${account.emailAddress}>`
-        }
-        msg.subject = data.subject,
-        msg.html = data.body,
+  // Initiates a multi-send session by creating a new multi-send draft.
+  server.route({
+    method: 'POST',
+    path: '/send-multiple',
+    config: {
+      validate: {
+        payload: {
+          to: recipientList,
+          cc: recipientList,
+          bcc: recipientList,
+          from: recipientList.length(1).required(),
+          reply_to: recipientList.min(0).max(1),
+          subject: Joi.string().required(),
+          body: Joi.string().required(),
+          thread_id: Joi.string(),
+          reply_to_message_id: Joi.string(),
+          client_id: Joi.string(),
+          account_id: Joi.string(),
+          id: Joi.string(),
+          object: Joi.string(),
+          metadata: Joi.array().items(Joi.string()),
+          date: Joi.number(),
+          files: Joi.array().items(Joi.string()),
+          file_ids: Joi.array().items(Joi.string()),
+          uploads: Joi.array().items(Joi.string()),
+          events: Joi.array().items(Joi.string()),
+          pristine: Joi.boolean(),
+          categories: Joi.array().items(Joi.string()),
+          draft: Joi.boolean(),
+        },
+      },
+    },
+    handler: async (request, reply) => {
+      try {
+        const accountId = request.auth.credentials.id;
+        const db = await LocalDatabaseConnector.forAccount(accountId)
+        const draft = await SendingUtils.findOrCreateMessageFromJSON(request.payload, db, false)
+        await (draft.isSending = true);
+        const savedDraft = await draft.save();
+        reply(savedDraft.toJSON());
+      } catch (err) {
+        respondWithError(request, reply, err);
+      }
+    },
+  });
 
-        sender.sendMail(msg, (error, info) => {
-          if (error) { reply(error).code(400) }
-          else { reply(info.response) }
+  // Performs a single send operation in an individualized multi-send
+  // session. Sends a copy of the draft at draft_id to the specified address
+  // with the specified body, and ensures that a corresponding sent message is
+  // either not created in the user's Sent folder or is immediately
+  // deleted from it.
+  server.route({
+    method: 'POST',
+    path: '/send-multiple/{draftId}',
+    config: {
+      validate: {
+        params: {
+          draftId: Joi.string(),
+        },
+        payload: {
+          send_to: recipient,
+          body: Joi.string(),
+        },
+      },
+    },
+    handler: async (request, reply) => {
+      try {
+        const requestStarted = new Date();
+        const account = request.auth.credentials;
+        const {draftId} = request.params;
+        SendingUtils.validateBase36(draftId, 'draftId')
+        const sendTo = request.payload.send_to;
+        const db = await LocalDatabaseConnector.forAccount(account.id)
+        const draft = await SendingUtils.findMultiSendDraft(draftId, db)
+        const {to, cc, bcc} = draft;
+        const recipients = [].concat(to, cc, bcc);
+        if (!recipients.find(contact => contact.email === sendTo.email)) {
+          throw new SendingUtils.HTTPError(
+            "Invalid sendTo, not present in message recipients",
+            400
+          );
+        }
+
+        const sender = new SendmailClient(account, request.logger);
+
+        if (new Date() - requestStarted > SEND_TIMEOUT) {
+          // Preemptively time out the request if we got stuck doing database work
+          // -- we don't want clients to disconnect and then still send the
+          // message.
+          reply('Request timeout out.').code(504);
+        }
+        const response = await sender.sendCustomBody(draft, request.payload.body, {to: [sendTo]})
+        reply(response);
+      } catch (err) {
+        respondWithError(request, reply, err);
+      }
+    },
+  });
+
+  // Closes out a multi-send session by marking the sending draft as sent
+  // and moving it to the user's Sent folder.
+  server.route({
+    method: 'DELETE',
+    path: '/send-multiple/{draftId}',
+    config: {
+      validate: {
+        params: {
+          draftId: Joi.string(),
+        },
+      },
+    },
+    handler: async (request, reply) => {
+      try {
+        const account = request.auth.credentials;
+        const {draftId} = request.params;
+        SendingUtils.validateBase36(draftId);
+
+        const db = await LocalDatabaseConnector.forAccount(account.id);
+        const draft = await SendingUtils.findMultiSendDraft(draftId, db);
+
+        // gmail creates sent messages for each one, go through and delete them
+        if (account.provider === 'gmail') {
+          try {
+            // TODO: use type: "PermananentDeleteMessage" once it's fully implemented
+            await db.SyncbackRequest.create({
+              type: "DeleteMessage",
+              props: { messageId: draft.id },
+            });
+          } catch (err) {
+            // Even if this fails, we need to finish the multi-send session,
+            request.logger.error(err, err.logContext);
+          }
+        }
+
+        const sender = new SendmailClient(account, request.logger);
+        const rawMime = await sender.buildMime(draft);
+
+        await db.SyncbackRequest.create({
+          accountId: account.id,
+          type: "SaveSentMessage",
+          props: {rawMime},
         });
-      })
-    })},
+
+        await (draft.isSent = true);
+        const savedDraft = await draft.save();
+        reply(savedDraft.toJSON());
+      } catch (err) {
+        respondWithError(request, reply, err);
+      }
+    },
   });
 };
diff --git a/packages/local-sync/src/local-api/sending-utils.js b/packages/local-sync/src/local-api/sending-utils.js
new file mode 100644
index 000000000..869f8045f
--- /dev/null
+++ b/packages/local-sync/src/local-api/sending-utils.js
@@ -0,0 +1,132 @@
+const _ = require('underscore');
+
+const setReplyHeaders = (newMessage, prevMessage) => {
+  if (prevMessage.messageIdHeader) {
+    newMessage.inReplyTo = prevMessage.headerMessageId;
+    if (prevMessage.references) {
+      newMessage.references = prevMessage.references.concat(prevMessage.headerMessageId);
+    } else {
+      newMessage.references = [prevMessage.messageIdHeader];
+    }
+  }
+}
+
+class HTTPError extends Error {
+  constructor(message, httpCode, logContext) {
+    super(message);
+    this.httpCode = httpCode;
+    this.logContext = logContext;
+  }
+}
+
+module.exports = {
+  HTTPError,
+  findOrCreateMessageFromJSON: async (data, db, isDraft) => {
+    const {Thread, Message} = db;
+
+    const existingMessage = await Message.findById(data.id);
+    if (existingMessage) {
+      return existingMessage;
+    }
+
+    const {to, cc, bcc, from, replyTo, subject, body, account_id, date, id} = data;
+
+    const message = Message.build({
+      accountId: account_id,
+      from: from,
+      to: to,
+      cc: cc,
+      bcc: bcc,
+      replyTo: replyTo,
+      subject: subject,
+      body: body,
+      unread: true,
+      isDraft: isDraft,
+      isSent: false,
+      version: 0,
+      date: date,
+      id: id,
+    });
+
+    // TODO
+    // Attach files
+    // Update our contact list
+    // Add events
+    // Add metadata??
+
+    let replyToThread;
+    let replyToMessage;
+    if (data.thread_id != null) {
+      replyToThread = await Thread.find({
+        where: {id: data.thread_id},
+        include: [{
+          model: Message,
+          as: 'messages',
+          attributes: _.without(Object.keys(Message.attributes), 'body'),
+        }],
+      });
+    }
+    if (data.reply_to_message_id != null) {
+      replyToMessage = await Message.findById(data.reply_to_message_id);
+    }
+
+    if (replyToThread && replyToMessage) {
+      if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
+        throw new HTTPError(
+          `Message ${replyToMessage.id} is not in thread ${replyToThread.id}`,
+          400
+        )
+      }
+    }
+
+    let thread;
+    if (replyToMessage) {
+      setReplyHeaders(message, replyToMessage);
+      thread = await message.getThread();
+    } else if (replyToThread) {
+      thread = replyToThread;
+      const previousMessages = thread.messages.filter(msg => !msg.isDraft);
+      if (previousMessages.length > 0) {
+        const lastMessage = previousMessages[previousMessages.length - 1]
+        setReplyHeaders(message, lastMessage);
+      }
+    } else {
+      thread = Thread.build({
+        accountId: account_id,
+        subject: message.subject,
+        firstMessageDate: message.date,
+        lastMessageDate: message.date,
+        lastMessageSentDate: message.date,
+      })
+    }
+
+    const savedMessage = await message.save();
+    const savedThread = await thread.save();
+    await savedThread.addMessage(savedMessage);
+
+    return savedMessage;
+  },
+  findMultiSendDraft: async (draftId, db) => {
+    const draft = await db.Message.findById(draftId)
+    if (!draft) {
+      throw new HTTPError(`Couldn't find multi-send draft ${draftId}`, 400);
+    }
+    if (draft.isSent || !draft.isSending) {
+      throw new HTTPError(`Message ${draftId} is not a multi-send draft`, 400);
+    }
+    return draft;
+  },
+  validateRecipientsPresent: (draft) => {
+    const {to, cc, bcc} = draft;
+    const recipients = [].concat(to, cc, bcc);
+    if (recipients.length === 0) {
+      throw new HTTPError("No recipients specified", 400);
+    }
+  },
+  validateBase36: (value, name) => {
+    if (value == null) { return; }
+    if (isNaN(parseInt(value, 36))) {
+      throw new HTTPError(`${name} is not a base-36 integer`, 400)
+    }
+  },
+}
diff --git a/packages/local-sync/src/local-api/sendmail-client.js b/packages/local-sync/src/local-api/sendmail-client.js
new file mode 100644
index 000000000..e8da66dd6
--- /dev/null
+++ b/packages/local-sync/src/local-api/sendmail-client.js
@@ -0,0 +1,110 @@
+const nodemailer = require('nodemailer');
+const mailcomposer = require('mailcomposer');
+const {HTTPError} = require('./sending-utils');
+
+const MAX_RETRIES = 1;
+
+const formatParticipants = (participants) => {
+  return participants.map(p => `${p.name} <${p.email}>`).join(',');
+}
+
+class SendmailClient {
+  constructor(account, logger) {
+    this._transporter = nodemailer.createTransport(account.smtpConfig());
+    this._logger = logger;
+  }
+
+  async _send(msgData) {
+    let partialFailure;
+    let error;
+    for (let i = 0; i <= MAX_RETRIES; i++) {
+      try {
+        const results = await this._transporter.sendMail(msgData);
+        const {rejected, pending} = results;
+        if ((rejected && rejected.length > 0) || (pending && pending.length > 0)) {
+          // At least one recipient was rejected by the server,
+          // but at least one recipient got it. Don't retry; throw an
+          // error so that we fail to client.
+          partialFailure = new HTTPError(
+            'Sending to at least one recipient failed', 200, results);
+          throw partialFailure;
+        } else {
+          // Sending was successful!
+          return
+        }
+      } catch (err) {
+        error = err;
+        if (err === partialFailure) {
+          // We don't want to retry in this case, so re-throw the error
+          throw err;
+        }
+        this._logger.error(err);
+      }
+    }
+    this._logger.error('Max sending retries reached');
+
+    // TODO: figure out how to parse different errors, like in cloud-core
+    // https://github.com/nylas/cloud-core/blob/production/sync-engine/inbox/sendmail/smtp/postel.py#L354
+    throw new HTTPError('Sending failed', 500, error)
+  }
+
+  _draftToMsgData(draft) {
+    const msgData = {};
+    for (const field of ['from', 'to', 'cc', 'bcc']) {
+      if (draft[field]) {
+        msgData[field] = formatParticipants(draft[field])
+      }
+    }
+    msgData.subject = draft.subject;
+    msgData.html = draft.body;
+
+    // TODO: attachments
+
+    if (draft.replyTo) {
+      msgData.replyTo = formatParticipants(draft.replyTo);
+    }
+
+    msgData.inReplyTo = draft.inReplyTo;
+    msgData.references = draft.references;
+    msgData.headers = draft.headers;
+    msgData.headers['User-Agent'] = `NylasMailer-K2`
+
+    // TODO: do we want to set messageId or date?
+
+    return msgData;
+  }
+
+  async buildMime(draft) {
+    const builder = mailcomposer(this._draftToMsgData(draft))
+    return new Promise((resolve, reject) => {
+      builder.build((error, result) => {
+        error ? reject(error) : resolve(result)
+      })
+    })
+  }
+
+  async send(draft) {
+    if (draft.isSent) {
+      throw new Error(`Cannot send message ${draft.id}, it has already been sent`);
+    }
+    await this._send(this._draftToMsgData(draft));
+    await (draft.isSent = true);
+    await draft.save();
+  }
+
+  async sendCustomBody(draft, body, recipients) {
+    const origBody = draft.body;
+    draft.body = body;
+    const envelope = {};
+    for (const field of Object.keys(recipients)) {
+      envelope[field] = recipients[field].map(r => r.email);
+    }
+    const raw = await this.buildMime(draft);
+    const responseOnSuccess = draft.toJSON();
+    draft.body = origBody;
+    await this._send({raw, envelope})
+    return responseOnSuccess
+  }
+}
+
+module.exports = SendmailClient;
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
index bcd8fc585..43d939074 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
@@ -39,8 +39,12 @@ class SyncbackTaskFactory {
         Task = require('./syncback_tasks/rename-folder.imap'); break;
       case "DeleteFolder":
         Task = require('./syncback_tasks/delete-folder.imap'); break;
+      case "DeleteMessage":
+        Task = require('./syncback_tasks/delete-message.imap'); break;
+      case "SaveSentMessage":
+        Task = require('./syncback_tasks/save-sent-message.imap'); break;
       default:
-        throw new Error(`Invalid Task Type: ${syncbackRequest.type}`)
+        throw new Error(`Task type not defined in syncback-task-factory: ${syncbackRequest.type}`)
     }
     return new Task(account, syncbackRequest)
   }
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
new file mode 100644
index 000000000..5ec2d6d4c
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
@@ -0,0 +1,18 @@
+const SyncbackTask = require('./syncback-task')
+const TaskHelpers = require('./task-helpers')
+
+class DeleteMessageIMAP extends SyncbackTask {
+  description() {
+    return `DeleteMessage`;
+  }
+
+  run(db, imap) {
+    const messageId = this.syncbackRequestObject().props.messageId
+
+    return TaskHelpers.openMessageBox({messageId, db, imap})
+      .then(({box, message}) => {
+        return box.addFlags(message.folderImapUID, 'DELETED')
+      })
+  }
+}
+module.exports = DeleteMessageIMAP;
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js
new file mode 100644
index 000000000..96488595c
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js
@@ -0,0 +1,24 @@
+const SyncbackTask = require('./syncback-task')
+
+class PermanentlyDeleteMessageIMAP extends SyncbackTask {
+  description() {
+    return `PermanentlyDeleteMessage`;
+  }
+
+  async run(db, imap) {
+    const messageId = this.syncbackRequestObject().props.messageId
+    const message = await db.Message.findById(messageId);
+    const folder = await db.Folder.findById(message.folderId);
+    const box = await imap.openBox(folder.name);
+    const result = await box.addFlags(message.folderImapUID, 'DELETED');
+    return result;
+
+    // TODO: We need to also delete the message from the trash
+    // if (folder.role === 'trash') { return result; }
+    //
+    // const trash = await db.Folder.find({where: {role: 'trash'}});
+    // const trashBox = await imap.openBox(trash.name);
+    // return await trashBox.addFlags(message.folderImapUID, 'DELETED');
+  }
+}
+module.exports = PermanentlyDeleteMessageIMAP;
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
new file mode 100644
index 000000000..2eb0cb6c1
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
@@ -0,0 +1,15 @@
+const SyncbackTask = require('./syncback-task')
+
+class SaveSentMessageIMAP extends SyncbackTask {
+  description() {
+    return `SaveSentMessage`;
+  }
+
+  async run(db, imap) {
+    // TODO: gmail doesn't have a sent folder
+    const folder = await db.Folder.find({where: {role: 'sent'}});
+    const box = await imap.openBox(folder.name);
+    return box.append(this.syncbackRequestObject().props.rawMime);
+  }
+}
+module.exports = SaveSentMessageIMAP;
diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.js
index 8d892a312..6a810c420 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.js
@@ -1,4 +1,4 @@
-const {DatabaseTypes: {JSONType}} = require('isomorphic-core');
+const {DatabaseTypes: {buildJSONColumnOptions}} = require('isomorphic-core');
 const {formatImapPath} = require('../shared/imap-paths-utils');
 
 module.exports = (sequelize, Sequelize) => {
@@ -8,7 +8,7 @@ module.exports = (sequelize, Sequelize) => {
     version: Sequelize.INTEGER,
     name: Sequelize.STRING,
     role: Sequelize.STRING,
-    syncState: JSONType('syncState'),
+    syncState: buildJSONColumnOptions('syncState'),
   }, {
     indexes: [
       {
diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index 3e3478f03..41f150a99 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -1,7 +1,19 @@
 const cryptography = require('crypto');
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
-const {DatabaseTypes: {JSONType, JSONARRAYType}} = require('isomorphic-core');
+const {DatabaseTypes: {buildJSONColumnOptions, buildJSONARRAYColumnOptions}} = require('isomorphic-core');
+const striptags = require('striptags');
+const SendingUtils = require('../local-api/sending-utils');
 
+const SNIPPET_LENGTH = 191;
+
+const getValidateArrayLength1 = (fieldName) => {
+  return (stringifiedArr) => {
+    const arr = JSON.parse(stringifiedArr);
+    if (arr.length !== 1) {
+      throw new Error(`Value for ${fieldName} must have a length of 1. Value: ${stringifiedArr}`);
+    }
+  };
+}
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('message', {
@@ -10,20 +22,55 @@ module.exports = (sequelize, Sequelize) => {
     version: Sequelize.INTEGER,
     headerMessageId: Sequelize.STRING,
     body: Sequelize.TEXT('long'),
-    headers: JSONType('headers'),
+    headers: buildJSONColumnOptions('headers'),
     subject: Sequelize.STRING(500),
     snippet: Sequelize.STRING(255),
     date: Sequelize.DATE,
+    isDraft: Sequelize.BOOLEAN,
+    isSent: {
+      type: Sequelize.BOOLEAN,
+      set: async function set(val) {
+        if (val) {
+          this.isDraft = false;
+          this.date = (new Date()).getTime();
+          const thread = await this.getThread();
+          await thread.updateFromMessage(this)
+        }
+        this.setDataValue('isSent', val);
+      },
+    },
     unread: Sequelize.BOOLEAN,
     starred: Sequelize.BOOLEAN,
     processed: Sequelize.INTEGER,
-    to: JSONARRAYType('to'),
-    from: JSONARRAYType('from'),
-    cc: JSONARRAYType('cc'),
-    bcc: JSONARRAYType('bcc'),
-    replyTo: JSONARRAYType('replyTo'),
+    to: buildJSONARRAYColumnOptions('to'),
+    from: Object.assign(buildJSONARRAYColumnOptions('from'), {
+      allowNull: true,
+      validate: {validateArrayLength1: getValidateArrayLength1('Message.from')},
+    }),
+    cc: buildJSONARRAYColumnOptions('cc'),
+    bcc: buildJSONARRAYColumnOptions('bcc'),
+    replyTo: Object.assign(buildJSONARRAYColumnOptions('replyTo'), {
+      allowNull: true,
+      validate: {validateArrayLength1: getValidateArrayLength1('Message.replyTo')},
+    }),
+    inReplyTo: { type: Sequelize.STRING, allowNull: true},
+    references: buildJSONARRAYColumnOptions('references'),
     folderImapUID: { type: Sequelize.STRING, allowNull: true},
     folderImapXGMLabels: { type: Sequelize.TEXT, allowNull: true},
+    isSending: {
+      type: Sequelize.BOOLEAN,
+      set: function set(val) {
+        if (val) {
+          if (this.isSent) {
+            throw new Error("Cannot mark a sent message as sending");
+          }
+          SendingUtils.validateRecipientsPresent(this);
+          this.isDraft = false;
+          this.regenerateHeaderMessageId();
+        }
+        this.setDataValue('isSending', val);
+      },
+    },
   }, {
     indexes: [
       {
@@ -69,6 +116,13 @@ module.exports = (sequelize, Sequelize) => {
         })
       },
 
+      // The uid in this header is simply the draft id and version concatenated.
+      // Because this uid identifies the draft on the remote provider, we
+      // regenerate it on each draft revision so that we can delete the old draft
+      // and add the new one on the remote.
+      regenerateHeaderMessageId() {
+        this.headerMessageId = `<${this.id}-${this.version}@mailer.nylas.com>`
+      },
       toJSON() {
         if (this.folder_id && !this.folder) {
           throw new Error("Message.toJSON called on a message where folder were not eagerly loaded.")
@@ -99,5 +153,15 @@ module.exports = (sequelize, Sequelize) => {
         };
       },
     },
+    hooks: {
+      beforeUpdate: (message) => {
+        // Update the snippet if the body has changed
+        if (!message.changed('body')) { return; }
+
+        const plainText = striptags(message.body);
+        // consolidate whitespace groups into single spaces and then truncate
+        message.snippet = plainText.split(/\s+/).join(" ").substring(0, SNIPPET_LENGTH)
+      },
+    },
   });
 };
diff --git a/packages/local-sync/src/models/syncbackRequest.js b/packages/local-sync/src/models/syncbackRequest.js
index 6ffe3e0d0..4cfdaac21 100644
--- a/packages/local-sync/src/models/syncbackRequest.js
+++ b/packages/local-sync/src/models/syncbackRequest.js
@@ -1,4 +1,4 @@
-const {DatabaseTypes: {JSONType}} = require('isomorphic-core');
+const {DatabaseTypes: {buildJSONColumnOptions}} = require('isomorphic-core');
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('syncbackRequest', {
@@ -8,8 +8,8 @@ module.exports = (sequelize, Sequelize) => {
       defaultValue: "NEW",
       allowNull: false,
     },
-    error: JSONType('error'),
-    props: JSONType('props'),
+    error: buildJSONColumnOptions('error'),
+    props: buildJSONColumnOptions('props'),
     accountId: { type: Sequelize.STRING, allowNull: false },
   }, {
     instanceMethods: {
diff --git a/packages/local-sync/src/models/thread.js b/packages/local-sync/src/models/thread.js
index 6b588ee6a..a6c27156c 100644
--- a/packages/local-sync/src/models/thread.js
+++ b/packages/local-sync/src/models/thread.js
@@ -1,4 +1,4 @@
-const {DatabaseTypes: {JSONARRAYType}} = require('isomorphic-core');
+const {DatabaseTypes: {buildJSONARRAYColumnOptions}} = require('isomorphic-core');
 
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('thread', {
@@ -8,13 +8,19 @@ module.exports = (sequelize, Sequelize) => {
     remoteThreadId: Sequelize.STRING,
     subject: Sequelize.STRING(500),
     snippet: Sequelize.STRING(255),
-    unreadCount: Sequelize.INTEGER,
-    starredCount: Sequelize.INTEGER,
+    unreadCount: {
+      type: Sequelize.INTEGER,
+      get: function get() { return this.getDataValue('unreadCount') || 0 },
+    },
+    starredCount: {
+      type: Sequelize.INTEGER,
+      get: function get() { return this.getDataValue('starredCount') || 0 },
+    },
     firstMessageDate: Sequelize.DATE,
     lastMessageDate: Sequelize.DATE,
     lastMessageReceivedDate: Sequelize.DATE,
     lastMessageSentDate: Sequelize.DATE,
-    participants: JSONARRAYType('participants'),
+    participants: buildJSONARRAYColumnOptions('participants'),
   }, {
     indexes: [
       { fields: ['subject'] },
@@ -56,7 +62,72 @@ module.exports = (sequelize, Sequelize) => {
 
         return this.save();
       },
+      async updateFromMessage(message) {
+        if (message.isDraft) {
+          return this;
+        }
 
+        if (!(message.labels instanceof Array)) {
+          throw new Error("Expected message.labels to be an inflated array.");
+        }
+        if (!message.folder) {
+          throw new Error("Expected message.folder value to be present.");
+        }
+
+        // Update thread participants
+        const {to, cc, bcc} = message;
+        const participantEmails = this.participants.map(contact => contact.email);
+        const newParticipants = to.concat(cc, bcc).filter(contact => {
+          if (participantEmails.includes(contact.email)) {
+            return false;
+          }
+          participantEmails.push(contact.email);
+          return true;
+        })
+        this.participants = this.participants.concat(newParticipants);
+
+        // Update starred/unread counts
+        this.starredCount += message.starred ? 1 : 0;
+        this.unreadCount += message.unread ? 1 : 0;
+
+        // Update dates/snippet
+        if (!this.lastMessageDate || (message.date > this.lastMessageDate)) {
+          this.lastMessageDate = message.date;
+          this.snippet = message.snippet;
+        }
+        if (!this.firstMessageDate || (message.date < this.firstMessageDate)) {
+          this.firstMessageDate = message.date;
+        }
+
+        // Figure out if the message is sent or received and update more dates
+        const isSent = (
+          message.folder.role === 'sent' ||
+          !!message.labels.find(l => l.role === 'sent')
+        );
+
+        if (isSent && ((message.date > this.lastMessageSentDate) || !this.lastMessageSentDate)) {
+          this.lastMessageSentDate = message.date;
+        }
+        if (!isSent && ((message.date > this.lastMessageReceivedDate) || !this.lastMessageReceivedDate)) {
+          this.lastMessageReceivedDate = message.date;
+        }
+
+        const savedThread = await this.save();
+
+        // Update folders/labels
+        // This has to be done after the thread has been saved, because the
+        // thread may not have had an assigned id yet. addFolder()/addLabel()
+        // need an existing thread id to work properly.
+        if (!savedThread.folders.find(f => f.id === message.folderId)) {
+          await savedThread.addFolder(message.folder)
+        }
+        for (const label of message.labels) {
+          if (!savedThread.labels.find(l => l.id === label)) {
+            await savedThread.addLabel(label)
+          }
+        }
+        return savedThread;
+      },
       toJSON() {
         if (!(this.labels instanceof Array)) {
           throw new Error("Thread.toJSON called on a thread where labels were not eagerly loaded.")
diff --git a/packages/local-sync/src/new-message-processor/detect-thread.js b/packages/local-sync/src/new-message-processor/detect-thread.js
index 8cf15d8fb..b948b0472 100644
--- a/packages/local-sync/src/new-message-processor/detect-thread.js
+++ b/packages/local-sync/src/new-message-processor/detect-thread.js
@@ -35,6 +35,7 @@ function emptyThread({Thread, accountId}, options = {}) {
   const t = Thread.build(Object.assign({accountId}, options))
   t.folders = [];
   t.labels = [];
+  t.participants = [];
   return Promise.resolve(t)
 }
 
@@ -95,66 +96,15 @@ function detectThread({db, message}) {
 
     // update the basic properties of the thread
     thread.accountId = message.accountId;
+
     // Threads may, locally, have the ID of any message within the thread
     // (message IDs are globally unique, even across accounts!)
     if (!thread.id) {
       thread.id = `t:${message.id}`
     }
 
-    // update the participants on the thread
-    const threadParticipants = [].concat(thread.participants);
-    const threadEmails = thread.participants.map(p => p.email);
-
-    for (const p of [].concat(message.to, message.cc, message.from)) {
-      if (!threadEmails.includes(p.email)) {
-        threadParticipants.push(p);
-        threadEmails.push(p.email);
-      }
-    }
-    thread.participants = threadParticipants;
-
-    // update starred and unread
-    if (thread.starredCount == null) { thread.starredCount = 0; }
-    thread.starredCount += message.starred ? 1 : 0;
-    if (thread.unreadCount == null) { thread.unreadCount = 0; }
-    thread.unreadCount += message.unread ? 1 : 0;
-
-    // update dates
-    if (!thread.lastMessageDate || (message.date > thread.lastMessageDate)) {
-      thread.lastMessageDate = message.date;
-      thread.snippet = message.snippet;
-      thread.subject = cleanSubject(message.subject);
-    }
-    if (!thread.firstMessageDate || (message.date < thread.firstMessageDate)) {
-      thread.firstMessageDate = message.date;
-    }
-
-    const isSent = (
-      message.folder.role === 'sent' ||
-      !!message.labels.find(l => l.role === 'sent')
-    )
-
-    if (isSent && ((message.date > thread.lastMessageSentDate) || !thread.lastMessageSentDate)) {
-      thread.lastMessageSentDate = message.date;
-    }
-    if (!isSent && ((message.date > thread.lastMessageReceivedDate) || !thread.lastMessageReceivedDate)) {
-      thread.lastMessageReceivedDate = message.date;
-    }
-
-    return thread.save()
-    .then((saved) => {
-      const promises = []
-      // update folders and labels
-      if (!saved.folders.find(f => f.id === message.folderId)) {
-        promises.push(saved.addFolder(message.folder))
-      }
-      for (const label of message.labels) {
-        if (!saved.labels.find(l => l.id === label)) {
-          promises.push(saved.addLabel(label))
-        }
-      }
-      return Promise.all(promises).thenReturn(saved)
-    })
+    thread.subject = cleanSubject(message.subject);
+    return thread.updateFromMessage(message);
   });
 }
 

From 017e22c88d73f2973d53fe29d8062f61592015b2 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Mon, 5 Dec 2016 09:57:47 -0800
Subject: [PATCH 456/800] [*] Allow zero elements in the replyTo field

---
 packages/local-sync/src/models/message.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index 41f150a99..c482f3597 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -6,11 +6,11 @@ const SendingUtils = require('../local-api/sending-utils');
 
 const SNIPPET_LENGTH = 191;
 
-const getValidateArrayLength1 = (fieldName) => {
+const getValidateArrayLength = (fieldName, min, max) => {
   return (stringifiedArr) => {
     const arr = JSON.parse(stringifiedArr);
-    if (arr.length !== 1) {
-      throw new Error(`Value for ${fieldName} must have a length of 1. Value: ${stringifiedArr}`);
+    if ((arr.length < min) || (arr.length > max)) {
+      throw new Error(`Value for ${fieldName} must have a length in range [${min}-${max}]. Value: ${stringifiedArr}`);
     }
   };
 }
@@ -44,14 +44,14 @@ module.exports = (sequelize, Sequelize) => {
     processed: Sequelize.INTEGER,
     to: buildJSONARRAYColumnOptions('to'),
     from: Object.assign(buildJSONARRAYColumnOptions('from'), {
+      validate: {validateArrayLength1: getValidateArrayLength('Message.from', 1, 1)},
       allowNull: true,
-      validate: {validateArrayLength1: getValidateArrayLength1('Message.from')},
     }),
     cc: buildJSONARRAYColumnOptions('cc'),
     bcc: buildJSONARRAYColumnOptions('bcc'),
     replyTo: Object.assign(buildJSONARRAYColumnOptions('replyTo'), {
+      validate: {validateArrayLength1: getValidateArrayLength('Message.replyTo', 0, 1)},
       allowNull: true,
-      validate: {validateArrayLength1: getValidateArrayLength1('Message.replyTo')},
     }),
     inReplyTo: { type: Sequelize.STRING, allowNull: true},
     references: buildJSONARRAYColumnOptions('references'),

From 30c8bedd7a71f30e428f22133845a3f2bca80cdb Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Mon, 5 Dec 2016 12:16:53 -0800
Subject: [PATCH 457/800] [local-sync] fix(specs): run `npm test` in local-sync
 dir

---
 .gitignore                                    |   1 +
 packages/local-sync/package.json              |   3 +
 .../spec/fixtures/1-99174-body.txt            |   0
 .../spec/fixtures/1-99174-headers.txt         |   0
 .../parseFromImap/base-case.json              |   5 +
 .../spec/fixtures/thread.js                   |   0
 .../fetch-folder-list-spec.js                 | 115 ------------------
 .../spec/local-sync-worker/mock-database.js   |  55 ---------
 .../local-sync/spec/message-factory-spec.js   |  41 +++++++
 packages/local-sync/spec/threading-spec.js    |  55 +++++++++
 .../imap/fetch-messages-in-folder.js          |   2 +-
 .../spec/parsing-spec.js                      |  23 ----
 .../src/new-message-processor/spec/run.js     |   5 -
 .../spec/support/jasmine.json                 |   7 --
 .../spec/threading-spec.js                    |  55 ---------
 .../src/shared/local-database-connector.js    |   3 +-
 .../local-sync/src/shared/message-factory.js  |  43 ++-----
 17 files changed, 117 insertions(+), 296 deletions(-)
 rename packages/local-sync/{src/new-message-processor => }/spec/fixtures/1-99174-body.txt (100%)
 rename packages/local-sync/{src/new-message-processor => }/spec/fixtures/1-99174-headers.txt (100%)
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json
 rename packages/local-sync/{src/new-message-processor => }/spec/fixtures/thread.js (100%)
 delete mode 100644 packages/local-sync/spec/local-sync-worker/fetch-folder-list-spec.js
 delete mode 100644 packages/local-sync/spec/local-sync-worker/mock-database.js
 create mode 100644 packages/local-sync/spec/message-factory-spec.js
 create mode 100644 packages/local-sync/spec/threading-spec.js
 delete mode 100644 packages/local-sync/src/new-message-processor/spec/parsing-spec.js
 delete mode 100644 packages/local-sync/src/new-message-processor/spec/run.js
 delete mode 100644 packages/local-sync/src/new-message-processor/spec/support/jasmine.json
 delete mode 100644 packages/local-sync/src/new-message-processor/spec/threading-spec.js

diff --git a/.gitignore b/.gitignore
index 0f53da4f7..d502bdbd8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ newrelic_agent.log
 .elasticbeanstalk/*
 !.elasticbeanstalk/*.cfg.yml
 !.elasticbeanstalk/*.global.yml
+/packages/local-sync/spec-saved-state.json
diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index 5e866d3f2..d77e66392 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -26,6 +26,9 @@
     "utf7": "^1.0.2",
     "vision": "4.1.0"
   },
+  "scripts": {
+    "test": "../../../../node_modules/.bin/electron ../../../../ --test --spec-directory=$(pwd)/spec"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/nylas/K2.git"
diff --git a/packages/local-sync/src/new-message-processor/spec/fixtures/1-99174-body.txt b/packages/local-sync/spec/fixtures/1-99174-body.txt
similarity index 100%
rename from packages/local-sync/src/new-message-processor/spec/fixtures/1-99174-body.txt
rename to packages/local-sync/spec/fixtures/1-99174-body.txt
diff --git a/packages/local-sync/src/new-message-processor/spec/fixtures/1-99174-headers.txt b/packages/local-sync/spec/fixtures/1-99174-headers.txt
similarity index 100%
rename from packages/local-sync/src/new-message-processor/spec/fixtures/1-99174-headers.txt
rename to packages/local-sync/spec/fixtures/1-99174-headers.txt
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json
new file mode 100644
index 000000000..5fc82c150
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json
@@ -0,0 +1,5 @@
+{
+  "imapMessage": {},
+  "desiredParts": {},
+  "result": {}
+}
diff --git a/packages/local-sync/src/new-message-processor/spec/fixtures/thread.js b/packages/local-sync/spec/fixtures/thread.js
similarity index 100%
rename from packages/local-sync/src/new-message-processor/spec/fixtures/thread.js
rename to packages/local-sync/spec/fixtures/thread.js
diff --git a/packages/local-sync/spec/local-sync-worker/fetch-folder-list-spec.js b/packages/local-sync/spec/local-sync-worker/fetch-folder-list-spec.js
deleted file mode 100644
index 75d1187ba..000000000
--- a/packages/local-sync/spec/local-sync-worker/fetch-folder-list-spec.js
+++ /dev/null
@@ -1,115 +0,0 @@
-const {PromiseUtils} = require('isomorphic-core');
-const mockDatabase = require('./mock-database');
-const FetchFolderList = require('../../src/local-sync-worker/imap/fetch-folder-list')
-
-const testCategoryRoles = (db, mailboxes) => {
-  const mockLogger = {
-    info: () => {},
-    debug: () => {},
-    error: () => {},
-  }
-  const mockImap = {
-    getBoxes: () => {
-      return Promise.resolve(mailboxes)
-    },
-  }
-  return new FetchFolderList('fakeProvider', mockLogger).run(db, mockImap).then(() => {
-    const {Folder, Label} = db;
-    return PromiseUtils.props({
-      folders: Folder.findAll(),
-      labels: Label.findAll(),
-    }).then(({folders, labels}) => {
-      const all = [].concat(folders, labels);
-      for (const category of all) {
-        expect(category.role).toEqual(mailboxes[category.name].role);
-      }
-    })
-  })
-};
-
-describe("FetchFolderList", () => {
-  beforeEach((done) => {
-    mockDatabase().then((db) => {
-      this.db = db;
-      done();
-    })
-  })
-
-  it("assigns roles when given a role attribute/flag", (done) => {
-    const mailboxes = {
-      'Sent': {attribs: ['\\Sent'], role: 'sent'},
-      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
-      'Spam': {attribs: ['\\Spam'], role: 'spam'},
-      'Trash': {attribs: ['\\Trash'], role: 'trash'},
-      'All Mail': {attribs: ['\\All'], role: 'all'},
-      'Important': {attribs: ['\\Important'], role: 'important'},
-      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
-      'Inbox': {attribs: ['\\Inbox'], role: 'inbox'},
-      'TestFolder': {attribs: [], role: null},
-      'Receipts': {attribs: [], role: null},
-    }
-
-    testCategoryRoles(this.db, mailboxes).then(done, done.fail);
-  })
-
-  it("assigns missing roles by localized display names", (done) => {
-    const mailboxes = {
-      'Sent': {attribs: [], role: 'sent'},
-      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
-      'Spam': {attribs: ['\\Spam'], role: 'spam'},
-      'Trash': {attribs: ['\\Trash'], role: 'trash'},
-      'All Mail': {attribs: ['\\All'], role: 'all'},
-      'Important': {attribs: ['\\Important'], role: 'important'},
-      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
-      'Inbox': {attribs: [], role: 'inbox'},
-    }
-
-    testCategoryRoles(this.db, mailboxes).then(done, done.fail);
-  })
-
-  it("doesn't assign a role more than once", (done) => {
-    const mailboxes = {
-      'Sent': {attribs: [], role: null},
-      'Sent Items': {attribs: [], role: null},
-      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
-      'Spam': {attribs: ['\\Spam'], role: 'spam'},
-      'Trash': {attribs: ['\\Trash'], role: 'trash'},
-      'All Mail': {attribs: ['\\All'], role: 'all'},
-      'Important': {attribs: ['\\Important'], role: 'important'},
-      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
-      'Mail': {attribs: ['\\Inbox'], role: 'inbox'},
-      'inbox': {attribs: [], role: null},
-    }
-
-    testCategoryRoles(this.db, mailboxes).then(done, done.fail);
-  })
-
-  it("updates role assignments if an assigned category is deleted", (done) => {
-    let mailboxes = {
-      'Sent': {attribs: [], role: null},
-      'Sent Items': {attribs: [], role: null},
-      'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
-      'Spam': {attribs: ['\\Spam'], role: 'spam'},
-      'Trash': {attribs: ['\\Trash'], role: 'trash'},
-      'All Mail': {attribs: ['\\All'], role: 'all'},
-      'Important': {attribs: ['\\Important'], role: 'important'},
-      'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
-      'Mail': {attribs: ['\\Inbox'], role: 'inbox'},
-    }
-
-    testCategoryRoles(this.db, mailboxes).then(() => {
-      mailboxes = {
-        'Sent Items': {attribs: [], role: 'sent'},
-        'Drafts': {attribs: ['\\Drafts'], role: 'drafts'},
-        'Spam': {attribs: ['\\Spam'], role: 'spam'},
-        'Trash': {attribs: ['\\Trash'], role: 'trash'},
-        'All Mail': {attribs: ['\\All'], role: 'all'},
-        'Important': {attribs: ['\\Important'], role: 'important'},
-        'Flagged': {attribs: ['\\Flagged'], role: 'flagged'},
-        'Mail': {attribs: ['\\Inbox'], role: 'inbox'},
-      }
-
-      return testCategoryRoles(this.db, mailboxes).then(done, done.fail);
-    }, done.fail);
-  })
-});
diff --git a/packages/local-sync/spec/local-sync-worker/mock-database.js b/packages/local-sync/spec/local-sync-worker/mock-database.js
deleted file mode 100644
index c2e20c47f..000000000
--- a/packages/local-sync/spec/local-sync-worker/mock-database.js
+++ /dev/null
@@ -1,55 +0,0 @@
-const LocalDatabaseConnector = require('../../src/shared/local-database-connector');
-
-/*
- * Mocks out various Model and Instance methods to prevent actually saving data
- * to the sequelize database. Note that with the current implementation, only
- * instances created with Model.build() are mocked out.
- *
- * Currently mocks out the following:
- *   Model
- *     .build()
- *     .findAll()
- *   Instance
- *     .destroy()
- *     .save()
- *
- */
-function mockDatabase() {
-  return LocalDatabaseConnector.forAccount(-1).then((db) => {
-    const data = {};
-    for (const modelName of Object.keys(db.sequelize.models)) {
-      const model = db.sequelize.models[modelName];
-      data[modelName] = {};
-
-      spyOn(model, 'findAll').and.callFake(() => {
-        return Promise.resolve(
-          Object.keys(data[modelName]).map(key => data[modelName][key])
-        );
-      });
-
-      const origBuild = model.build;
-      spyOn(model, 'build').and.callFake((...args) => {
-        const instance = origBuild.apply(model, args);
-
-        spyOn(instance, 'save').and.callFake(() => {
-          if (instance.id == null) {
-            const sortedIds = Object.keys(data[modelName]).sort();
-            const len = sortedIds.length;
-            instance.id = len ? +sortedIds[len - 1] + 1 : 0;
-          }
-          data[modelName][instance.id] = instance;
-        });
-
-        spyOn(instance, 'destroy').and.callFake(() => {
-          delete data[modelName][instance.id]
-        });
-
-        return instance;
-      })
-    }
-
-    return Promise.resolve(db);
-  });
-}
-
-module.exports = mockDatabase;
diff --git a/packages/local-sync/spec/message-factory-spec.js b/packages/local-sync/spec/message-factory-spec.js
new file mode 100644
index 000000000..c0643f26a
--- /dev/null
+++ b/packages/local-sync/spec/message-factory-spec.js
@@ -0,0 +1,41 @@
+const path = require('path');
+const fs = require('fs');
+const LocalDatabaseConnector = require('../src/shared/local-database-connector');
+const {parseFromImap} = require('../src/shared/message-factory');
+
+const FIXTURES_PATH = path.join(__dirname, 'fixtures')
+
+describe('MessageFactory', function MessageFactorySpecs() {
+  beforeEach(() => {
+    waitsForPromise(async () => {
+      const accountId = 'test-account-id';
+      await LocalDatabaseConnector.ensureAccountDatabase(accountId);
+      const db = await LocalDatabaseConnector.forAccount(accountId);
+      const folder = await db.Folder.create({
+        id: 'test-folder-id',
+        accountId: accountId,
+        version: 1,
+        name: 'Test Folder',
+        role: null,
+      });
+      this.options = { accountId, db, folder };
+    })
+  })
+
+  describe("parseFromImap", () => {
+    const fixturesDir = path.join(FIXTURES_PATH, 'MessageFactory', 'parseFromImap');
+    const filenames = fs.readdirSync(fixturesDir).filter(f => f.endsWith('.json'));
+
+    filenames.forEach((filename) => {
+      it(`should correctly build message properties for ${filename}`, () => {
+        const inJSON = JSON.parse(fs.readFileSync(path.join(fixturesDir, filename)));
+        const {imapMessage, desiredParts, result} = inJSON;
+
+        waitsForPromise(async () => {
+          const actual = await parseFromImap(imapMessage, desiredParts, this.options);
+          expect(actual).toEqual(result)
+        });
+      });
+    })
+  });
+});
diff --git a/packages/local-sync/spec/threading-spec.js b/packages/local-sync/spec/threading-spec.js
new file mode 100644
index 000000000..2313af432
--- /dev/null
+++ b/packages/local-sync/spec/threading-spec.js
@@ -0,0 +1,55 @@
+/* eslint global-require: 0 */
+/* eslint import/no-dynamic-require: 0 */
+// const path = require('path')
+// const BASE_PATH = path.join(__dirname, 'fixtures')
+
+xdescribe('threading', function threadingSpecs() {
+  // it('adds the message to the thread', (done) => {
+  //   const {message, reply} = require(`${BASE_PATH}/thread`)
+  //   const accountId = 'a-1'
+  //   const mockDb = {
+  //     Thread: {
+  //       findAll: () => {
+  //         return Promise.resolve([{
+  //           id: 1,
+  //           subject: "Loved your work and interests",
+  //           messages: [message],
+  //         }])
+  //       },
+  //       find: () => {
+  //         return Promise.resolve(null)
+  //       },
+  //       create: (thread) => {
+  //         thread.id = 1
+  //         thread.addMessage = (newMessage) => {
+  //           if (thread.messages) {
+  //             thread.messages.push(newMessage.id)
+  //           } else {
+  //             thread.messages = [newMessage.id]
+  //           }
+  //         }
+  //         return Promise.resolve(thread)
+  //       },
+  //     },
+  //     Message: {
+  //       findAll: () => {
+  //         return Promise.resolve([message, reply])
+  //       },
+  //       find: () => {
+  //         return Promise.resolve(reply)
+  //       },
+  //       create: () => {
+  //         message.setThread = (thread) => {
+  //           message.thread = thread.id
+  //         };
+  //         return Promise.resolve(message);
+  //       },
+  //     },
+  //   }
+  //
+  //   processMessage({db: mockDb, message: reply, accountId}).then((processed) => {
+  //     expect(processed.thread).toBe(1)
+  //     done()
+  //   })
+  // });
+});
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index b5d5c30bd..f6a628cc9 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -239,8 +239,8 @@ class FetchMessagesInFolder {
     try {
       const messageValues = await MessageFactory.parseFromImap(imapMessage, desiredParts, {
         db: this._db,
+        folder: this._folder,
         accountId: this._db.accountId,
-        folderId: this._folder.id,
       });
       const existingMessage = await Message.find({where: {id: messageValues.id}});
 
diff --git a/packages/local-sync/src/new-message-processor/spec/parsing-spec.js b/packages/local-sync/src/new-message-processor/spec/parsing-spec.js
deleted file mode 100644
index e8025c0b7..000000000
--- a/packages/local-sync/src/new-message-processor/spec/parsing-spec.js
+++ /dev/null
@@ -1,23 +0,0 @@
-const path = require('path')
-const fs = require('fs')
-const {processMessage} = require('../processors/parsing')
-
-const BASE_PATH = path.join(__dirname, 'fixtures')
-
-
-it('parses the message correctly', (done) => {
-  const bodyPath = path.join(BASE_PATH, '1-99174-body.txt')
-  const headersPath = path.join(BASE_PATH, '1-99174-headers.txt')
-  const rawBody = fs.readFileSync(bodyPath, 'utf8')
-  const rawHeaders = fs.readFileSync(headersPath, 'utf8')
-  const message = { rawHeaders, rawBody }
-  const bodyPart = `
In _data/apps.yml:
`
-
-  processMessage({message}).then((processed) => {
-    expect(processed.headers['in-reply-to']).toEqual('')
-    expect(processed.headerMessageId).toEqual('')
-    expect(processed.subject).toEqual('Re: [electron/electron.atom.io] Add Jasper app (#352)')
-    expect(processed.body.includes(bodyPart)).toBe(true)
-    done()
-  })
-})
diff --git a/packages/local-sync/src/new-message-processor/spec/run.js b/packages/local-sync/src/new-message-processor/spec/run.js
deleted file mode 100644
index 4eb56830f..000000000
--- a/packages/local-sync/src/new-message-processor/spec/run.js
+++ /dev/null
@@ -1,5 +0,0 @@
-import Jasmine from 'jasmine'
-
-const jasmine = new Jasmine()
-jasmine.loadConfigFile('spec/support/jasmine.json')
-jasmine.execute()
diff --git a/packages/local-sync/src/new-message-processor/spec/support/jasmine.json b/packages/local-sync/src/new-message-processor/spec/support/jasmine.json
deleted file mode 100644
index 6a2e779f4..000000000
--- a/packages/local-sync/src/new-message-processor/spec/support/jasmine.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
-    "spec_dir": "spec",
-    "spec_files": [
-        "**/*-spec.js"
-    ],
-    "helpers": [ ]
-}
diff --git a/packages/local-sync/src/new-message-processor/spec/threading-spec.js b/packages/local-sync/src/new-message-processor/spec/threading-spec.js
deleted file mode 100644
index 8fbe812cf..000000000
--- a/packages/local-sync/src/new-message-processor/spec/threading-spec.js
+++ /dev/null
@@ -1,55 +0,0 @@
-/* eslint global-require: 0 */
-/* eslint import/no-dynamic-require: 0 */
-const path = require('path')
-const {processMessage} = require('../processors/threading')
-
-const BASE_PATH = path.join(__dirname, 'fixtures')
-
-it('adds the message to the thread', (done) => {
-  const {message, reply} = require(`${BASE_PATH}/thread`)
-  const accountId = 'a-1'
-  const mockDb = {
-    Thread: {
-      findAll: () => {
-        return Promise.resolve([{
-          id: 1,
-          subject: "Loved your work and interests",
-          messages: [message],
-        }])
-      },
-      find: () => {
-        return Promise.resolve(null)
-      },
-      create: (thread) => {
-        thread.id = 1
-        thread.addMessage = (newMessage) => {
-          if (thread.messages) {
-            thread.messages.push(newMessage.id)
-          } else {
-            thread.messages = [newMessage.id]
-          }
-        }
-        return Promise.resolve(thread)
-      },
-    },
-    Message: {
-      findAll: () => {
-        return Promise.resolve([message, reply])
-      },
-      find: () => {
-        return Promise.resolve(reply)
-      },
-      create: () => {
-        message.setThread = (thread) => {
-          message.thread = thread.id
-        };
-        return Promise.resolve(message);
-      },
-    },
-  }
-
-  processMessage({db: mockDb, message: reply, accountId}).then((processed) => {
-    expect(processed.thread).toBe(1)
-    done()
-  })
-})
diff --git a/packages/local-sync/src/shared/local-database-connector.js b/packages/local-sync/src/shared/local-database-connector.js
index e6a8494f1..d1a25fd07 100644
--- a/packages/local-sync/src/shared/local-database-connector.js
+++ b/packages/local-sync/src/shared/local-database-connector.js
@@ -12,8 +12,9 @@ class LocalDatabaseConnector {
   }
 
   _sequelizePoolForDatabase(dbname) {
+    const storage = NylasEnv.inSpecMode() ? ':memory:' : path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`);
     return new Sequelize(dbname, '', '', {
-      storage: path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`),
+      storage: storage,
       dialect: "sqlite",
       logging: false,
     })
diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index 534e9466d..61375e801 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -12,8 +12,8 @@ function extractContacts(values = []) {
   })
 }
 
-function parseFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
-  const {Message, Label, Folder} = db
+async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder}) {
+  const {Message, Label} = db
   const body = {}
   const {headers, attributes} = imapMessage
   const xGmLabels = attributes['x-gm-labels']
@@ -51,7 +51,7 @@ function parseFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
     starred: attributes.flags.includes('\\Flagged'),
     date: attributes.date,
     folderImapUID: attributes.uid,
-    folderId: folderId,
+    folderId: folder.id,
     folder: null,
     labels: [],
     headers: parsedHeaders,
@@ -72,40 +72,15 @@ function parseFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
     values.snippet = values.body.substr(0, Math.min(values.body.length, SNIPPET_SIZE));
   }
 
-  return Folder.findById(folderId)
-  .then((folder) => {
-    values.folder = folder
+  values.folder = folder
+  if (xGmLabels) {
+    values.folderImapXGMLabels = JSON.stringify(xGmLabels)
+    values.labels = await Label.findXGMLabels(xGmLabels)
+  }
 
-    if (xGmLabels) {
-      values.folderImapXGMLabels = JSON.stringify(xGmLabels)
-      return Label.findXGMLabels(xGmLabels)
-      .then((labels) => {
-        values.labels = labels
-        return values
-      })
-    }
-    return Promise.resolve(values)
-  })
-}
-
-function buildFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
-  return parseFromImap(imapMessage, desiredParts, {db, accountId, folderId})
-  .then((messageValues) => db.Message.build(messageValues))
-}
-
-function createFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
-  return parseFromImap(imapMessage, desiredParts, {db, accountId, folderId})
-  .then((messageValues) => db.Message.create(messageValues))
-}
-
-function upsertFromImap(imapMessage, desiredParts, {db, accountId, folderId}) {
-  return parseFromImap(imapMessage, desiredParts, {db, accountId, folderId})
-  .then((messageValues) => db.Message.upsert(messageValues))
+  return values;
 }
 
 module.exports = {
   parseFromImap,
-  buildFromImap,
-  createFromImap,
-  upsertFromImap,
 }

From 63032a24492df3c2bcc2ae2829bd1cc5fc85e8bc Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Mon, 5 Dec 2016 12:55:14 -0800
Subject: [PATCH 458/800] [local-sync] Fix(send): change some validation fields
 for multi-send

---
 packages/local-sync/src/local-api/routes/send.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/local-sync/src/local-api/routes/send.js b/packages/local-sync/src/local-api/routes/send.js
index ec861ea5c..da98c6d09 100644
--- a/packages/local-sync/src/local-api/routes/send.js
+++ b/packages/local-sync/src/local-api/routes/send.js
@@ -65,12 +65,12 @@ module.exports = (server) => {
           account_id: Joi.string(),
           id: Joi.string(),
           object: Joi.string(),
-          metadata: Joi.array().items(Joi.string()),
+          metadata: Joi.array().items(Joi.object()),
           date: Joi.number(),
           files: Joi.array().items(Joi.string()),
-          file_ids: Joi.array().items(Joi.string()),
-          uploads: Joi.array().items(Joi.string()),
-          events: Joi.array().items(Joi.string()),
+          file_ids: Joi.array(),
+          uploads: Joi.array(),
+          events: Joi.array(),
           pristine: Joi.boolean(),
           categories: Joi.array().items(Joi.string()),
           draft: Joi.boolean(),

From 269d61171a782713ecfbfd0af9940e428adcfb1c Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Mon, 5 Dec 2016 15:00:21 -0800
Subject: [PATCH 459/800] [local-sync] fix(specs): New specs for threading

---
 packages/local-sync/package.json              |   2 +-
 .../local-sync/spec/fixtures/1-99174-body.txt |  51 -------
 .../spec/fixtures/1-99174-headers.txt         |  67 ---------
 .../fixtures/Threading/remote-thread-id-no.js |  53 ++++++++
 .../Threading/remote-thread-id-yes.js         |  53 ++++++++
 .../fixtures/Threading/subject-matching-no.js |  78 +++++++++++
 .../Threading/subject-matching-yes.js         |  78 +++++++++++
 packages/local-sync/spec/fixtures/thread.js   |  81 -----------
 packages/local-sync/spec/threading-spec.js    | 128 +++++++++++-------
 .../new-message-processor/detect-thread.js    |  66 +++++----
 .../src/shared/local-database-connector.js    |  10 +-
 11 files changed, 379 insertions(+), 288 deletions(-)
 delete mode 100644 packages/local-sync/spec/fixtures/1-99174-body.txt
 delete mode 100644 packages/local-sync/spec/fixtures/1-99174-headers.txt
 create mode 100644 packages/local-sync/spec/fixtures/Threading/remote-thread-id-no.js
 create mode 100644 packages/local-sync/spec/fixtures/Threading/remote-thread-id-yes.js
 create mode 100644 packages/local-sync/spec/fixtures/Threading/subject-matching-no.js
 create mode 100644 packages/local-sync/spec/fixtures/Threading/subject-matching-yes.js
 delete mode 100644 packages/local-sync/spec/fixtures/thread.js

diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index d77e66392..65ac6b351 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -27,7 +27,7 @@
     "vision": "4.1.0"
   },
   "scripts": {
-    "test": "../../../../node_modules/.bin/electron ../../../../ --test --spec-directory=$(pwd)/spec"
+    "test": "../../../../node_modules/.bin/electron ../../../../ --test --enable-logging --spec-directory=$(pwd)/spec"
   },
   "repository": {
     "type": "git",
diff --git a/packages/local-sync/spec/fixtures/1-99174-body.txt b/packages/local-sync/spec/fixtures/1-99174-body.txt
deleted file mode 100644
index c711338c0..000000000
--- a/packages/local-sync/spec/fixtures/1-99174-body.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-----==_mimepart_576812149912_70c83ff638d112bc812dd
-Content-Type: text/plain;
- charset=UTF-8
-Content-Transfer-Encoding: 7bit
-
-> @@ -1298,3 +1298,15 @@
->      - "Unity"
->      - "Scratch"
->    icon: "gausssense.png"
-> +-
-> +  name: "Jasper"
-> +  description: "The GitHub Issue Reader"
-> +  website: "https://jasperapp.io"
-> +  repository: "https://github.com/jasperapp/jasper"
-> +  license: ""
-
-Oh oops, one more thing, you can also just remove this line rather than leave it empty. 
-
----
-You are receiving this because you are subscribed to this thread.
-Reply to this email directly or view it on GitHub:
-https://github.com/electron/electron.atom.io/pull/352/files/c3af72fe5aea0add912cd5afe153a626619de222#r67715160
-----==_mimepart_576812149912_70c83ff638d112bc812dd
-Content-Type: text/html;
- charset=UTF-8
-Content-Transfer-Encoding: 7bit
-
-
In _data/apps.yml:

-
> @@ -1298,3 +1298,15 @@
->      - "Unity"
->      - "Scratch"
->    icon: "gausssense.png"
-> +-
-> +  name: "Jasper"
-> +  description: "The GitHub Issue Reader"
-> +  website: "https://jasperapp.io"
-> +  repository: "https://github.com/jasperapp/jasper"
-> +  license: ""
-

-
Oh oops, one more thing, you can also just remove this line rather than leave it empty. 

-
-
—
You are receiving this because you are subscribed to this thread.
Reply to this email directly, view it on GitHub, or mute the thread.

-

-

-  
-  
-

-
-

-
-----==_mimepart_576812149912_70c83ff638d112bc812dd--
diff --git a/packages/local-sync/spec/fixtures/1-99174-headers.txt b/packages/local-sync/spec/fixtures/1-99174-headers.txt
deleted file mode 100644
index 3658cf07b..000000000
--- a/packages/local-sync/spec/fixtures/1-99174-headers.txt
+++ /dev/null
@@ -1,67 +0,0 @@
-Delivered-To: bengotow@gmail.com
-Received: by 10.107.39.131 with SMTP id n125csp1566492ion; Mon, 20 Jun 2016
- 08:56:06 -0700 (PDT)
-X-Received: by 10.107.175.83 with SMTP id y80mr24643127ioe.70.1466438166869;
- Mon, 20 Jun 2016 08:56:06 -0700 (PDT)
-Return-Path: 
-Received: from o5.sgmail.github.com (o5.sgmail.github.com. [192.254.113.10])
- by mx.google.com with ESMTPS id g123si16992050ioe.194.2016.06.20.08.56.06 for
-  (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256
- bits=128/128); Mon, 20 Jun 2016 08:56:06 -0700 (PDT)
-Received-SPF: pass (google.com: best guess record for domain of
- bounces+848413-92ca-bengotow=gmail.com@sgmail.github.com designates
- 192.254.113.10 as permitted sender) client-ip=192.254.113.10;
-Authentication-Results: mx.google.com; dkim=pass header.i=@github.com;
- dkim=pass header.i=@sendgrid.info; spf=pass (google.com: best guess record
- for domain of bounces+848413-92ca-bengotow=gmail.com@sgmail.github.com
- designates 192.254.113.10 as permitted sender)
- smtp.mailfrom=bounces+848413-92ca-bengotow=gmail.com@sgmail.github.com;
- dmarc=pass (p=NONE dis=NONE) header.from=github.com
-DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=github.com;
- h=from:reply-to:to:cc:in-reply-to:references:subject:mime-version:content-type:content-transfer-encoding:list-id:list-archive:list-post:list-unsubscribe;
- s=s20150108; bh=5XfiJ/IF3qb2JfQruE1ZXDdHpzo=; b=pVyxGvcKaxEE3zix
- VALF4ugiEssCLrxxiuKOPttOz/WL7Xzf6wUl74cHRUbZ4ShEXT3pe8yUbiMuWJIy
- R/Kegxrf+ThIoFNNj4SLuZbtv+N7Ic2GrXZdxZLyHnR5sxNKdXk7r0Z6GkXdmavZ
- Y3GYWjJp9DO7FMFyQ88BqIPQXto=
-DKIM-Signature: v=1; a=rsa-sha1; c=relaxed; d=sendgrid.info;
- h=from:reply-to:to:cc:in-reply-to:references:subject:mime-version:content-type:content-transfer-encoding:list-id:list-archive:list-post:list-unsubscribe:x-feedback-id;
- s=smtpapi; bh=5XfiJ/IF3qb2JfQruE1ZXDdHpzo=; b=WHrBPVafQkvIJmwSPM
- SPpv2z0GcAQcM8IKTj/L4KDI92vwfoy8zclmTj+NB4XP5LH8Fcjd3Zj9yFJvb5O/
- 2+ZSEhQNSg9ALStUDQuGYR0kzspSQsvncx4jI7tHZPfP2oAZkQIu4IYLjVN80IEF
- SuFzrPpitO7BmwcRSiAy7SZLI=
-Received: by filter0441p1mdw1.sendgrid.net with SMTP id
- filter0441p1mdw1.13584.5768121421 2016-06-20 15:56:04.264672456 +0000 UTC
-Received: from github-smtp2b-ext-cp1-prd.iad.github.net
- (github-smtp2b-ext-cp1-prd.iad.github.net [192.30.253.17]) by
- ismtpd0004p1iad1.sendgrid.net (SG) with ESMTP id LipJElPMRZOlYFTbOZCHmg for
- ; Mon, 20 Jun 2016 15:56:04.173 +0000 (UTC)
-Date: Mon, 20 Jun 2016 08:56:04 -0700
-From: Jessica Lord 
-Reply-To: "electron/electron.atom.io"
- 
-To: "electron/electron.atom.io" 
-Cc: 
-Message-ID: 
-In-Reply-To: 
-References: 
-Subject: Re: [electron/electron.atom.io] Add Jasper app (#352)
-Mime-Version: 1.0
-Content-Type: multipart/alternative;
- boundary="--==_mimepart_576812149912_70c83ff638d112bc812dd"; charset=UTF-8
-Content-Transfer-Encoding: 7bit
-Precedence: list
-X-GitHub-Sender: jlord
-X-GitHub-Recipient: bengotow
-List-ID: electron/electron.atom.io 
-List-Archive: https://github.com/electron/electron.atom.io
-List-Post: 
-List-Unsubscribe: ,
- 
-X-Auto-Response-Suppress: All
-X-GitHub-Recipient-Address: bengotow@gmail.com
-X-SG-EID: zpvha59ROIAeS8NpfQSmmILEvCHiUBlaQ8TFhOrQHPl0QSHOgRoWHzGqgNGQ1PnCE+4nt5yi6b+iVd
- zi18AeK3NmcJMPjYoKBL9oylAFXcX7JSvmAI2o8GQMXjRfn6C9jjTKhzFhZ3brgLB13N+v++poucpZ
- K0ksN28YtdSO0PvYYktx2Lu+g8yKBRFzyXmcuJM5D9O3YDhd0/VRVk1WlanT5sFV/t1fjjJtGwaOvg
- 4=
-X-Feedback-ID: 848413:6xvVEJqleZlAW7/vhv7PzD/cv5tamo2SWZDKyvugGvg=:6xvVEJqleZlAW7/vhv7PzD/cv5tamo2SWZDKyvugGvg=:SG
-
diff --git a/packages/local-sync/spec/fixtures/Threading/remote-thread-id-no.js b/packages/local-sync/spec/fixtures/Threading/remote-thread-id-no.js
new file mode 100644
index 000000000..7b454629e
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/Threading/remote-thread-id-no.js
@@ -0,0 +1,53 @@
+module.exports = {
+  A: {
+    id: 1,
+    accountId: 'test-account-id',
+    subject: "This is an email",
+    body: "Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "x-gm-thrid": "GMAILTHREAD1",
+      "Delivered-To": "jackiehluo@gmail.com",
+      "Date": "Fri, 17 Jun 2016 09:38:44 -0700 (PDT)",
+      "Message-Id": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "X-Inbox-Id": "82y7eq1ipmadaxwcy6kr072bw-2147483647",
+    },
+    from: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    to: [{
+      name: "jackiehluo@gmail.com",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+    snippet: "Hi Jackie, While browsing Nylas themes, I stumbled upon your website and looked at your work. Great ",
+  },
+  B: {
+    id: 2,
+    accountId: 'test-account-id',
+    subject: "This is a reply with a different subject",
+    body: "Sagar,

Aw, glad to hear it! Thanks for getting in touch!

Jackie Luo
Software Engineer, Nylas

On Jun 17 2016, at 9:38 am, Sagar Sutar <sagar_s@nid.edu> wrote:
Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "x-gm-thrid": "GMAILTHREAD2",
+      "Date": "Fri, 17 Jun 2016 18:20:47 +0000",
+      "References": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "In-Reply-To": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "Message-Id": "",
+      "X-Inbox-Id": "cq08iqwatp00kai4qnff7zbaj-2147483647",
+    },
+    from: [{
+      name: "Jackie Luo",
+      email: "",
+    }],
+    to: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "",
+    snippet: "Sagar, Aw, glad to hear it! Thanks for getting in touch! Jackie Luo Software Engineer, Nylas",
+  },
+};
diff --git a/packages/local-sync/spec/fixtures/Threading/remote-thread-id-yes.js b/packages/local-sync/spec/fixtures/Threading/remote-thread-id-yes.js
new file mode 100644
index 000000000..a0a63fc5c
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/Threading/remote-thread-id-yes.js
@@ -0,0 +1,53 @@
+module.exports = {
+  A: {
+    id: 1,
+    accountId: 'test-account-id',
+    subject: "This is an email",
+    body: "Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "x-gm-thrid": "GMAILTHREAD1",
+      "Delivered-To": "jackiehluo@gmail.com",
+      "Date": "Fri, 17 Jun 2016 09:38:44 -0700 (PDT)",
+      "Message-Id": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "X-Inbox-Id": "82y7eq1ipmadaxwcy6kr072bw-2147483647",
+    },
+    from: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    to: [{
+      name: "jackiehluo@gmail.com",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+    snippet: "Hi Jackie, While browsing Nylas themes, I stumbled upon your website and looked at your work. Great ",
+  },
+  B: {
+    id: 2,
+    accountId: 'test-account-id',
+    subject: "This is a reply with a different subject",
+    body: "Sagar,

Aw, glad to hear it! Thanks for getting in touch!

Jackie Luo
Software Engineer, Nylas

On Jun 17 2016, at 9:38 am, Sagar Sutar <sagar_s@nid.edu> wrote:
Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "x-gm-thrid": "GMAILTHREAD1",
+      "Date": "Fri, 17 Jun 2016 18:20:47 +0000",
+      "References": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "In-Reply-To": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "Message-Id": "",
+      "X-Inbox-Id": "cq08iqwatp00kai4qnff7zbaj-2147483647",
+    },
+    from: [{
+      name: "Jackie Luo",
+      email: "",
+    }],
+    to: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "",
+    snippet: "Sagar, Aw, glad to hear it! Thanks for getting in touch! Jackie Luo Software Engineer, Nylas",
+  },
+};
diff --git a/packages/local-sync/spec/fixtures/Threading/subject-matching-no.js b/packages/local-sync/spec/fixtures/Threading/subject-matching-no.js
new file mode 100644
index 000000000..ddad93a6a
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/Threading/subject-matching-no.js
@@ -0,0 +1,78 @@
+module.exports = {
+  A: {
+    id: 1,
+    accountId: 'test-account-id',
+    subject: "Loved your work and interests",
+    body: "Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "Delivered-To": "jackiehluo@gmail.com",
+      "Received-SPF": `pass (google.com: domain of sagy26.1991@gmail.com
+        designates 209.85.192.174 as permitted sender) client-ip=209.85.192.174;`,
+      "Authentication-Results": `mx.google.com;
+         spf=pass (google.com: domain of sagy26.1991@gmail.com designates
+         209.85.192.174 as permitted sender) smtp.mailfrom=sagy26.1991@gmail.com`,
+      "X-Google-DKIM-Signature": `v=1; a=rsa-sha256; c=relaxed/relaxed;
+          d=1e100.net; s=20130820;
+          h=x-gm-message-state:date:user-agent:message-id:to:from:subject
+           :mime-version;
+          bh=to3fCB9g4R6V18kpAAKSAlUeTC+N0rg4JckFbiaILA4=;
+          b=WfI5viTYPjviUur9Bd2rJQfpHxIm2xYRdxrN64bJGuX0TQlb7p8bDvCBNNhY3mTXJx
+           lsQzRX9RA4FMuDk0oz0mpviWtkpkZsDeyjpSmA+ONcPgdyPAezzPDvSWRzMZY21fiHxS
+           hr4I5AeFKesGcbvwtJu+S0fMGhdveC8E35oTA010Xfave6Xd55qGXy7hW+4xCfvIesy4
+           01oOaXWDmLHqixKO3SXwmGCcDzqn/IKXhB7UXkF0efSTwh8yid6v9iXdW+ovJ2qg9peI
+           HSnPIilYk8SaKoPdGDgYZykfUIgNrSugtK/vvGG2aN+9lhURxPfzhniWdNqdsgR7G4E7
+           7XqA==`,
+      "X-Gm-Message-State": "ALyK8tIf7XyYaylyVf0qjzh8rhYz3rj/VQYaNLDjVq5ESH19ioJIgW7o9FbghP+wFYrBuw==",
+      "X-Received": `by 10.98.111.138 with SMTP id k132mr3246291pfc.105.1466181525186;
+          Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
+      "Return-Path": "",
+      "Received": `from [127.0.0.1] (ec2-52-36-99-221.us-west-2.compute.amazonaws.com. [52.36.99.221])
+          by smtp.gmail.com with ESMTPSA id d69sm64179062pfj.31.2016.06.17.09.38.44
+          for 
+          (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+          Fri, 17 Jun 2016 09:38:44 -0700 (PDT)`,
+      "Date": "Fri, 17 Jun 2016 09:38:44 -0700 (PDT)",
+      "User-Agent": "NylasMailer/0.4",
+      "Message-Id": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "X-Inbox-Id": "82y7eq1ipmadaxwcy6kr072bw-2147483647",
+    },
+    from: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    to: [{
+      name: "jackiehluo@gmail.com",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+    snippet: "Hi Jackie, While browsing Nylas themes, I stumbled upon your website and looked at your work. Great ",
+  },
+  B: {
+    id: 2,
+    accountId: 'test-account-id',
+    subject: "Re: Another unrelated email",
+    body: "Sagar,

Aw, glad to hear it! Thanks for getting in touch!

Jackie Luo
Software Engineer, Nylas

On Jun 17 2016, at 9:38 am, Sagar Sutar <sagar_s@nid.edu> wrote:
Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "Date": "Fri, 17 Jun 2016 18:20:47 +0000",
+      "References": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "In-Reply-To": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "User-Agent": "NylasMailer/0.4",
+      "Message-Id": "",
+      "X-Inbox-Id": "cq08iqwatp00kai4qnff7zbaj-2147483647",
+    },
+    from: [{
+      name: "Jackie Luo",
+      email: "",
+    }],
+    to: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "",
+    snippet: "Sagar, Aw, glad to hear it! Thanks for getting in touch! Jackie Luo Software Engineer, Nylas",
+  },
+};
diff --git a/packages/local-sync/spec/fixtures/Threading/subject-matching-yes.js b/packages/local-sync/spec/fixtures/Threading/subject-matching-yes.js
new file mode 100644
index 000000000..65b88ffa0
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/Threading/subject-matching-yes.js
@@ -0,0 +1,78 @@
+module.exports = {
+  A: {
+    id: 1,
+    accountId: 'test-account-id',
+    subject: "Loved your work and interests",
+    body: "Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "Delivered-To": "jackiehluo@gmail.com",
+      "Received-SPF": `pass (google.com: domain of sagy26.1991@gmail.com
+        designates 209.85.192.174 as permitted sender) client-ip=209.85.192.174;`,
+      "Authentication-Results": `mx.google.com;
+         spf=pass (google.com: domain of sagy26.1991@gmail.com designates
+         209.85.192.174 as permitted sender) smtp.mailfrom=sagy26.1991@gmail.com`,
+      "X-Google-DKIM-Signature": `v=1; a=rsa-sha256; c=relaxed/relaxed;
+          d=1e100.net; s=20130820;
+          h=x-gm-message-state:date:user-agent:message-id:to:from:subject
+           :mime-version;
+          bh=to3fCB9g4R6V18kpAAKSAlUeTC+N0rg4JckFbiaILA4=;
+          b=WfI5viTYPjviUur9Bd2rJQfpHxIm2xYRdxrN64bJGuX0TQlb7p8bDvCBNNhY3mTXJx
+           lsQzRX9RA4FMuDk0oz0mpviWtkpkZsDeyjpSmA+ONcPgdyPAezzPDvSWRzMZY21fiHxS
+           hr4I5AeFKesGcbvwtJu+S0fMGhdveC8E35oTA010Xfave6Xd55qGXy7hW+4xCfvIesy4
+           01oOaXWDmLHqixKO3SXwmGCcDzqn/IKXhB7UXkF0efSTwh8yid6v9iXdW+ovJ2qg9peI
+           HSnPIilYk8SaKoPdGDgYZykfUIgNrSugtK/vvGG2aN+9lhURxPfzhniWdNqdsgR7G4E7
+           7XqA==`,
+      "X-Gm-Message-State": "ALyK8tIf7XyYaylyVf0qjzh8rhYz3rj/VQYaNLDjVq5ESH19ioJIgW7o9FbghP+wFYrBuw==",
+      "X-Received": `by 10.98.111.138 with SMTP id k132mr3246291pfc.105.1466181525186;
+          Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
+      "Return-Path": "",
+      "Received": `from [127.0.0.1] (ec2-52-36-99-221.us-west-2.compute.amazonaws.com. [52.36.99.221])
+          by smtp.gmail.com with ESMTPSA id d69sm64179062pfj.31.2016.06.17.09.38.44
+          for 
+          (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
+          Fri, 17 Jun 2016 09:38:44 -0700 (PDT)`,
+      "Date": "Fri, 17 Jun 2016 09:38:44 -0700 (PDT)",
+      "User-Agent": "NylasMailer/0.4",
+      "Message-Id": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "X-Inbox-Id": "82y7eq1ipmadaxwcy6kr072bw-2147483647",
+    },
+    from: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    to: [{
+      name: "jackiehluo@gmail.com",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+    snippet: "Hi Jackie, While browsing Nylas themes, I stumbled upon your website and looked at your work. Great ",
+  },
+  B: {
+    id: 2,
+    accountId: 'test-account-id',
+    subject: "Re: Loved your work and interests",
+    body: "Sagar,

Aw, glad to hear it! Thanks for getting in touch!

Jackie Luo
Software Engineer, Nylas

On Jun 17 2016, at 9:38 am, Sagar Sutar <sagar_s@nid.edu> wrote:
Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
+    headers: {
+      "Date": "Fri, 17 Jun 2016 18:20:47 +0000",
+      "References": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "In-Reply-To": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
+      "User-Agent": "NylasMailer/0.4",
+      "Message-Id": "",
+      "X-Inbox-Id": "cq08iqwatp00kai4qnff7zbaj-2147483647",
+    },
+    from: [{
+      name: "Jackie Luo",
+      email: "",
+    }],
+    to: [{
+      name: "Sagar Sutar",
+      email: "",
+    }],
+    cc: [],
+    bcc: [],
+    headerMessageId: "",
+    snippet: "Sagar, Aw, glad to hear it! Thanks for getting in touch! Jackie Luo Software Engineer, Nylas",
+  },
+};
diff --git a/packages/local-sync/spec/fixtures/thread.js b/packages/local-sync/spec/fixtures/thread.js
deleted file mode 100644
index 82be1660b..000000000
--- a/packages/local-sync/spec/fixtures/thread.js
+++ /dev/null
@@ -1,81 +0,0 @@
-export const message = {
-  id: 1,
-  subject: "Loved your work and interests",
-  body: "Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
-  headers: {
-    "Delivered-To": "jackiehluo@gmail.com",
-    "Received-SPF": `pass (google.com: domain of sagy26.1991@gmail.com
-      designates 209.85.192.174 as permitted sender) client-ip=209.85.192.174;`,
-    "Authentication-Results": `mx.google.com;
-       spf=pass (google.com: domain of sagy26.1991@gmail.com designates
-       209.85.192.174 as permitted sender) smtp.mailfrom=sagy26.1991@gmail.com`,
-    "X-Google-DKIM-Signature": `v=1; a=rsa-sha256; c=relaxed/relaxed;
-        d=1e100.net; s=20130820;
-        h=x-gm-message-state:date:user-agent:message-id:to:from:subject
-         :mime-version;
-        bh=to3fCB9g4R6V18kpAAKSAlUeTC+N0rg4JckFbiaILA4=;
-        b=WfI5viTYPjviUur9Bd2rJQfpHxIm2xYRdxrN64bJGuX0TQlb7p8bDvCBNNhY3mTXJx
-         lsQzRX9RA4FMuDk0oz0mpviWtkpkZsDeyjpSmA+ONcPgdyPAezzPDvSWRzMZY21fiHxS
-         hr4I5AeFKesGcbvwtJu+S0fMGhdveC8E35oTA010Xfave6Xd55qGXy7hW+4xCfvIesy4
-         01oOaXWDmLHqixKO3SXwmGCcDzqn/IKXhB7UXkF0efSTwh8yid6v9iXdW+ovJ2qg9peI
-         HSnPIilYk8SaKoPdGDgYZykfUIgNrSugtK/vvGG2aN+9lhURxPfzhniWdNqdsgR7G4E7
-         7XqA==`,
-    "X-Gm-Message-State": "ALyK8tIf7XyYaylyVf0qjzh8rhYz3rj/VQYaNLDjVq5ESH19ioJIgW7o9FbghP+wFYrBuw==",
-    "X-Received": `by 10.98.111.138 with SMTP id k132mr3246291pfc.105.1466181525186;
-        Fri, 17 Jun 2016 09:38:45 -0700 (PDT)`,
-    "Return-Path": "",
-    "Received": `from [127.0.0.1] (ec2-52-36-99-221.us-west-2.compute.amazonaws.com. [52.36.99.221])
-        by smtp.gmail.com with ESMTPSA id d69sm64179062pfj.31.2016.06.17.09.38.44
-        for 
-        (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128);
-        Fri, 17 Jun 2016 09:38:44 -0700 (PDT)`,
-    "Date": "Fri, 17 Jun 2016 09:38:44 -0700 (PDT)",
-    "User-Agent": "NylasMailer/0.4",
-    "Message-Id": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
-    "X-Inbox-Id": "82y7eq1ipmadaxwcy6kr072bw-2147483647",
-  },
-  from: [{
-    name: "Sagar Sutar",
-    email: "",
-  }],
-  to: [{
-    name: "jackiehluo@gmail.com",
-    email: "",
-  }],
-  cc: [],
-  bcc: [],
-  headerMessageId: "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
-  snippet: "Hi Jackie, While browsing Nylas themes, I stumbled upon your website and looked at your work. Great ",
-  setThread: (thread) => {
-    message.thread = thread.id
-  },
-}
-
-export const reply = {
-  id: 2,
-  subject: "Re: Loved your work and interests",
-  body: "Sagar,

Aw, glad to hear it! Thanks for getting in touch!

Jackie Luo
Software Engineer, Nylas

On Jun 17 2016, at 9:38 am, Sagar Sutar <sagar_s@nid.edu> wrote:
Hi Jackie,
While browsing Nylas themes, I stumbled upon your website and looked at your work. 
Great work on projects, nice to see your multidisciplinary interests :)

Thanks, 
Sagar Sutar
thesagarsutar.me
",
-  headers: {
-    "Date": "Fri, 17 Jun 2016 18:20:47 +0000",
-    "References": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
-    "In-Reply-To": "<82y7eq1ipmadaxwcy6kr072bw-2147483647@mailer.nylas.com>",
-    "User-Agent": "NylasMailer/0.4",
-    "Message-Id": "",
-    "X-Inbox-Id": "cq08iqwatp00kai4qnff7zbaj-2147483647",
-  },
-  from: [{
-    name: "Jackie Luo",
-    email: "",
-  }],
-  to: [{
-    name: "Sagar Sutar",
-    email: "",
-  }],
-  cc: [],
-  bcc: [],
-  headerMessageId: "",
-  snippet: "Sagar, Aw, glad to hear it! Thanks for getting in touch! Jackie Luo Software Engineer, Nylas",
-  setThread: (thread) => {
-    reply.thread = thread.id
-  },
-}
diff --git a/packages/local-sync/spec/threading-spec.js b/packages/local-sync/spec/threading-spec.js
index 2313af432..22e6a4615 100644
--- a/packages/local-sync/spec/threading-spec.js
+++ b/packages/local-sync/spec/threading-spec.js
@@ -1,55 +1,81 @@
 /* eslint global-require: 0 */
 /* eslint import/no-dynamic-require: 0 */
-// const path = require('path')
-// const BASE_PATH = path.join(__dirname, 'fixtures')
+const path = require('path');
+const detectThread = require('../src/new-message-processor/detect-thread');
+const LocalDatabaseConnector = require('../src/shared/local-database-connector');
 
-xdescribe('threading', function threadingSpecs() {
-  // it('adds the message to the thread', (done) => {
-  //   const {message, reply} = require(`${BASE_PATH}/thread`)
-  //   const accountId = 'a-1'
-  //   const mockDb = {
-  //     Thread: {
-  //       findAll: () => {
-  //         return Promise.resolve([{
-  //           id: 1,
-  //           subject: "Loved your work and interests",
-  //           messages: [message],
-  //         }])
-  //       },
-  //       find: () => {
-  //         return Promise.resolve(null)
-  //       },
-  //       create: (thread) => {
-  //         thread.id = 1
-  //         thread.addMessage = (newMessage) => {
-  //           if (thread.messages) {
-  //             thread.messages.push(newMessage.id)
-  //           } else {
-  //             thread.messages = [newMessage.id]
-  //           }
-  //         }
-  //         return Promise.resolve(thread)
-  //       },
-  //     },
-  //     Message: {
-  //       findAll: () => {
-  //         return Promise.resolve([message, reply])
-  //       },
-  //       find: () => {
-  //         return Promise.resolve(reply)
-  //       },
-  //       create: () => {
-  //         message.setThread = (thread) => {
-  //           message.thread = thread.id
-  //         };
-  //         return Promise.resolve(message);
-  //       },
-  //     },
-  //   }
-  //
-  //   processMessage({db: mockDb, message: reply, accountId}).then((processed) => {
-  //     expect(processed.thread).toBe(1)
-  //     done()
-  //   })
-  // });
+const FIXTURES_PATH = path.join(__dirname, 'fixtures');
+const ACCOUNT_ID = 'test-account-threading';
+
+function messagesFromFixture({Message}, folder, name) {
+  const {A, B} = require(`${FIXTURES_PATH}/Threading/${name}`)
+
+  const msgA = Message.build(A);
+  msgA.folder = folder;
+  msgA.labels = [];
+
+  const msgB = Message.build(B);
+  msgB.folder = folder;
+  msgB.labels = [];
+
+  return {msgA, msgB};
+}
+
+describe('threading', function threadingSpecs() {
+  beforeEach(() => {
+    waitsForPromise({timeout: 1000}, async () => {
+      await LocalDatabaseConnector.ensureAccountDatabase(ACCOUNT_ID);
+      this.db = await LocalDatabaseConnector.forAccount(ACCOUNT_ID);
+      this.folder = await this.db.Folder.create({
+        id: 'test-folder-id',
+        accountId: ACCOUNT_ID,
+        version: 1,
+        name: 'Test Folder',
+        role: null,
+      });
+    });
+  });
+
+  afterEach(() => {
+    LocalDatabaseConnector.destroyAccountDatabase(ACCOUNT_ID)
+  })
+
+  describe("when remote thread ids are present", () => {
+    it('threads emails with the same gthreadid', () => {
+      waitsForPromise(async () => {
+        const {msgA, msgB} = messagesFromFixture(this.db, this.folder, 'remote-thread-id-yes');
+        const threadA = await detectThread({db: this.db, message: msgA});
+        const threadB = await detectThread({db: this.db, message: msgB});
+        expect(threadB.id).toEqual(threadA.id);
+      });
+    });
+
+    it('does not thread other emails', () => {
+      waitsForPromise(async () => {
+        const {msgA, msgB} = messagesFromFixture(this.db, this.folder, 'remote-thread-id-no');
+        const threadA = await detectThread({db: this.db, message: msgA});
+        const threadB = await detectThread({db: this.db, message: msgB});
+        expect(threadB.id).not.toEqual(threadA.id);
+      });
+    });
+  });
+  describe("when subject matching", () => {
+    it('threads emails with the same subject', () => {
+      waitsForPromise(async () => {
+        const {msgA, msgB} = messagesFromFixture(this.db, this.folder, 'subject-matching-yes');
+        const threadA = await detectThread({db: this.db, message: msgA});
+        const threadB = await detectThread({db: this.db, message: msgB});
+        expect(threadB.id).toEqual(threadA.id);
+      });
+    });
+
+    it('does not thread other emails', () => {
+      waitsForPromise(async () => {
+        const {msgA, msgB} = messagesFromFixture(this.db, this.folder, 'subject-matching-no');
+        const threadA = await detectThread({db: this.db, message: msgA});
+        const threadB = await detectThread({db: this.db, message: msgB});
+        expect(threadB.id).not.toEqual(threadA.id);
+      });
+    });
+  });
 });
diff --git a/packages/local-sync/src/new-message-processor/detect-thread.js b/packages/local-sync/src/new-message-processor/detect-thread.js
index b948b0472..4a69aa14d 100644
--- a/packages/local-sync/src/new-message-processor/detect-thread.js
+++ b/packages/local-sync/src/new-message-processor/detect-thread.js
@@ -36,10 +36,10 @@ function emptyThread({Thread, accountId}, options = {}) {
   t.folders = [];
   t.labels = [];
   t.participants = [];
-  return Promise.resolve(t)
+  return t;
 }
 
-function findOrBuildByMatching(db, message) {
+async function findOrBuildByMatching(db, message) {
   const {Thread, Label, Folder} = db
 
   // in the future, we should look at In-reply-to. Problem is it's a single-
@@ -47,7 +47,7 @@ function findOrBuildByMatching(db, message) {
   // but from newest->oldest, so when we ingest a message it's very unlikely
   // we have the "In-reply-to" message yet.
 
-  return Thread.findAll({
+  const possibleThreads = await Thread.findAll({
     where: {
       subject: cleanSubject(message.subject),
     },
@@ -56,56 +56,54 @@ function findOrBuildByMatching(db, message) {
     ],
     limit: 10,
     include: [{model: Label}, {model: Folder}],
-  }).then((threads) =>
-    pickMatchingThread(message, threads) || emptyThread(db, {})
-  )
+  });
+
+  return pickMatchingThread(message, possibleThreads) || emptyThread(db, {});
 }
 
-function findOrBuildByRemoteThreadId(db, remoteThreadId) {
+async function findOrBuildByRemoteThreadId(db, remoteThreadId) {
   const {Thread, Label, Folder} = db;
-  return Thread.find({
+  const existing = await Thread.find({
     where: {remoteThreadId},
     include: [{model: Label}, {model: Folder}],
-  }).then((thread) => {
-    return thread || emptyThread(db, {remoteThreadId})
-  })
+  });
+  return existing || emptyThread(db, {remoteThreadId});
 }
 
-function detectThread({db, message}) {
+async function detectThread({db, message}) {
   if (!(message.labels instanceof Array)) {
-    throw new Error("Threading processMessage expects labels to be an inflated array.");
+    throw new Error("detectThread expects labels to be an inflated array.");
   }
   if (!message.folder) {
-    throw new Error("Threading processMessage expects folder value to be present.");
+    throw new Error("detectThread expects folder value to be present.");
   }
 
-  let findOrBuildThread = null;
+  let thread = null;
   if (message.headers['x-gm-thrid']) {
-    findOrBuildThread = findOrBuildByRemoteThreadId(db, message.headers['x-gm-thrid'])
+    thread = await findOrBuildByRemoteThreadId(db, message.headers['x-gm-thrid'])
   } else {
-    findOrBuildThread = findOrBuildByMatching(db, message)
+    thread = await findOrBuildByMatching(db, message)
   }
 
-  return findOrBuildThread.then((thread) => {
-    if (!(thread.labels instanceof Array)) {
-      throw new Error("Threading processMessage expects thread.labels to be an inflated array.");
-    }
-    if (!(thread.folders instanceof Array)) {
-      throw new Error("Threading processMessage expects thread.folders to be an inflated array.");
-    }
+  if (!(thread.labels instanceof Array)) {
+    throw new Error("detectThread expects thread.labels to be an inflated array.");
+  }
+  if (!(thread.folders instanceof Array)) {
+    throw new Error("detectThread expects thread.folders to be an inflated array.");
+  }
 
-    // update the basic properties of the thread
-    thread.accountId = message.accountId;
+  // update the basic properties of the thread
+  thread.accountId = message.accountId;
 
-    // Threads may, locally, have the ID of any message within the thread
-    // (message IDs are globally unique, even across accounts!)
-    if (!thread.id) {
-      thread.id = `t:${message.id}`
-    }
+  // Threads may, locally, have the ID of any message within the thread
+  // (message IDs are globally unique, even across accounts!)
+  if (!thread.id) {
+    thread.id = `t:${message.id}`
+  }
 
-    thread.subject = cleanSubject(message.subject);
-    return thread.updateFromMessage(message);
-  });
+  thread.subject = cleanSubject(message.subject);
+  await thread.updateFromMessage(message);
+  return thread;
 }
 
 module.exports = detectThread
diff --git a/packages/local-sync/src/shared/local-database-connector.js b/packages/local-sync/src/shared/local-database-connector.js
index d1a25fd07..7d43eae69 100644
--- a/packages/local-sync/src/shared/local-database-connector.js
+++ b/packages/local-sync/src/shared/local-database-connector.js
@@ -66,9 +66,13 @@ class LocalDatabaseConnector {
     const dbname = `a-${accountId}`;
     const dbpath = path.join(process.env.NYLAS_HOME, `${dbname}.sqlite`);
 
-    const err = fs.accessSync(dbpath, fs.F_OK);
-    if (!err) {
-      fs.unlinkSync(dbpath);
+    try {
+      const err = fs.accessSync(dbpath, fs.F_OK);
+      if (!err) {
+        fs.unlinkSync(dbpath);
+      }
+    } catch (err) {
+      // Ignored
     }
 
     delete this._cache[accountId];

From 04b1763965c5307803ecc68d9ca54a666b2e183d Mon Sep 17 00:00:00 2001
From: Karim Hamidou 
Date: Mon, 5 Dec 2016 15:37:17 -0800
Subject: [PATCH 460/800] [fix] Fix transaction creation for contacts

Summary:
Fixes T7283. We weren't creating deltas for contacts because we were inserting contacts using `UPSERT`, which requires us to add another sequelize hook.  Unfortunately, support for this hook is only available on sequelize 4.x.

I didn't want to upgrade our sequelize version right now and cause of bunch of mysterious failures, so I've simply changed the `UPSERT` to be either an `INSERT`or `UPDATE`. We didn't really need it anyway.

Test Plan: Checked that N1 was receiving contact deltas.

Reviewers: bengotow

Reviewed By: bengotow

Maniphest Tasks: T7283

Differential Revision: https://phab.nylas.com/D3481
---
 .arcconfig                                    |  4 +++
 .gitignore                                    |  3 --
 .../src/hook-transaction-log.js               |  4 +++
 .../new-message-processor/extract-contacts.js | 30 +++++++++++--------
 4 files changed, 25 insertions(+), 16 deletions(-)
 create mode 100644 .arcconfig

diff --git a/.arcconfig b/.arcconfig
new file mode 100644
index 000000000..00ab68a97
--- /dev/null
+++ b/.arcconfig
@@ -0,0 +1,4 @@
+{
+  "project_id" : "K2",
+  "conduit_uri" : "https://phab.nylas.com/"
+}
diff --git a/.gitignore b/.gitignore
index d502bdbd8..06289f5de 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,3 @@
-.arcconfig
-.arclint
-arclib
 *.swp
 *~
 .DS_Store
diff --git a/packages/isomorphic-core/src/hook-transaction-log.js b/packages/isomorphic-core/src/hook-transaction-log.js
index df6e97a8c..699d6a770 100644
--- a/packages/isomorphic-core/src/hook-transaction-log.js
+++ b/packages/isomorphic-core/src/hook-transaction-log.js
@@ -46,5 +46,9 @@ module.exports = (db, sequelize, {only, onCreatedTransaction} = {}) => {
 
   sequelize.addHook("afterCreate", transactionLogger("create"))
   sequelize.addHook("afterUpdate", transactionLogger("modify"))
+
+  // NOTE: Hooking UPSERT requires Sequelize 4.x. We're
+  // on version 3 right now, but leaving this here for when we upgrade.
+  sequelize.addHook("afterUpsert", transactionLogger("modify"))
   sequelize.addHook("afterDelete", transactionLogger("delete"))
 }
diff --git a/packages/local-sync/src/new-message-processor/extract-contacts.js b/packages/local-sync/src/new-message-processor/extract-contacts.js
index 91fd3ccd5..4198bcb28 100644
--- a/packages/local-sync/src/new-message-processor/extract-contacts.js
+++ b/packages/local-sync/src/new-message-processor/extract-contacts.js
@@ -13,26 +13,30 @@ function isContactVerified(contact) {
   return true
 }
 
-function extractContacts({db, message}) {
-  const {Contact} = db;
-
+async function extractContacts({db, message}) {
   let allContacts = [];
   ['to', 'from', 'bcc', 'cc'].forEach((field) => {
     allContacts = allContacts.concat(message[field])
   })
 
   const verifiedContacts = allContacts.filter(c => isContactVerified(c));
-  return db.sequelize.transaction((transaction) => {
-    return Promise.all(verifiedContacts.map((contact) =>
-      Contact.upsert({
-        name: contact.name,
-        email: contact.email,
+  return db.sequelize.transaction(async (transaction) => {
+    for (const c of verifiedContacts) {
+      const id = cryptography.createHash('sha256').update(c.email, 'utf8').digest('hex');
+      let contact = await db.Contact.findById(id);
+      const cdata = {
+        name: c.name,
+        email: c.email,
         accountId: message.accountId,
-        id: cryptography.createHash('sha256').update(contact.email, 'utf8').digest('hex'),
-      }, {
-        transaction,
-      })
-    ))
+        id: id,
+      };
+      
+      if (!contact) {
+        contact = await db.Contact.create(cdata)
+      } else {
+        await contact.update(cdata);
+      }
+    }
   }).thenReturn(message)
 }
 

From 1e3b346c94bcfbfab8a980093e9e7b2ebf8d5129 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Mon, 5 Dec 2016 16:07:46 -0800
Subject: [PATCH 461/800] [local-sync] feat(specs): Add basic tests for folder
 sync

---
 .../FetchFolderList/gmail-bengotow.json       | 174 ++++++++++++++++++
 .../FetchFolderList/imap-inboxapptest1.json   | 158 ++++++++++++++++
 packages/local-sync/spec/helpers.js           |  28 +++
 .../spec/imap/fetch-folder-list-spec.js       |  45 +++++
 .../local-sync/spec/message-factory-spec.js   |  26 ++-
 packages/local-sync/spec/threading-spec.js    |   4 +-
 6 files changed, 417 insertions(+), 18 deletions(-)
 create mode 100644 packages/local-sync/spec/fixtures/FetchFolderList/gmail-bengotow.json
 create mode 100644 packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json
 create mode 100644 packages/local-sync/spec/helpers.js
 create mode 100644 packages/local-sync/spec/imap/fetch-folder-list-spec.js

diff --git a/packages/local-sync/spec/fixtures/FetchFolderList/gmail-bengotow.json b/packages/local-sync/spec/fixtures/FetchFolderList/gmail-bengotow.json
new file mode 100644
index 000000000..fc9b76d69
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/FetchFolderList/gmail-bengotow.json
@@ -0,0 +1,174 @@
+{
+	"boxes": {
+		"GitHub": {
+			"attribs": ["\\HasChildren"],
+			"delimiter": "/",
+			"children": {
+				"Electron": {
+					"attribs": ["\\HasNoChildren"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]"
+				},
+				"N1": {
+					"attribs": ["\\HasNoChildren"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]"
+				}
+			},
+			"parent": null
+		},
+		"INBOX": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Junk (Gmail)": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"N1-Snoozed": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Notes": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Receipts": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Recruiters": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Sentry": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"[Gmail]": {
+			"attribs": ["\\HasChildren", "\\Noselect"],
+			"delimiter": "/",
+			"children": {
+				"All Mail": {
+					"attribs": ["\\All", "\\HasNoChildren"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\All"
+				},
+				"Drafts": {
+					"attribs": ["\\Drafts", "\\HasNoChildren"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\Drafts"
+				},
+				"Important": {
+					"attribs": ["\\HasNoChildren", "\\Important"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\Important"
+				},
+				"Sent Mail": {
+					"attribs": ["\\HasNoChildren", "\\Sent"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\Sent"
+				},
+				"Spam": {
+					"attribs": ["\\HasNoChildren", "\\Junk"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\Junk"
+				},
+				"Starred": {
+					"attribs": ["\\Flagged", "\\HasNoChildren"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\Flagged"
+				},
+				"Trash": {
+					"attribs": ["\\HasNoChildren", "\\Trash"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]",
+					"special_use_attrib": "\\Trash"
+				}
+			},
+			"parent": null
+		}
+	},
+	"expectedFolders": [{
+		"role": "trash",
+		"name": "[Gmail]/Trash"
+	}, {
+		"role": "spam",
+		"name": "[Gmail]/Spam"
+	}, {
+		"role": "all",
+		"name": "[Gmail]/All Mail"
+	}],
+	"expectedLabels": [{
+		"role": "starred",
+		"name": "[Gmail]/Starred"
+	}, {
+		"role": "sent",
+		"name": "[Gmail]/Sent Mail"
+	}, {
+		"role": "important",
+		"name": "[Gmail]/Important"
+	}, {
+		"role": "drafts",
+		"name": "[Gmail]/Drafts"
+	}, {
+		"role": null,
+		"name": "Sentry"
+	}, {
+		"role": null,
+		"name": "Recruiters"
+	}, {
+		"role": null,
+		"name": "Receipts"
+	}, {
+		"role": null,
+		"name": "Notes"
+	}, {
+		"role": null,
+		"name": "N1-Snoozed"
+	}, {
+		"role": null,
+		"name": "Junk (Gmail)"
+	}, {
+		"role": "inbox",
+		"name": "INBOX"
+	}, {
+		"role": null,
+		"name": "GitHub"
+	}, {
+		"role": null,
+		"name": "GitHub/N1"
+	}, {
+		"role": null,
+		"name": "GitHub/Electron"
+	}]
+}
diff --git a/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json b/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json
new file mode 100644
index 000000000..2c2148fa8
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json
@@ -0,0 +1,158 @@
+{
+  "boxes": {
+  	"2016": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"INBOX": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"Archive": {
+  		"attribs": ["\\HasNoChildren", "\\Archive"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null,
+  		"special_use_attrib": "\\Archive"
+  	},
+  	"Arts": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"Boîte de réception": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"Drafts": {
+  		"attribs": ["\\HasNoChildren", "\\Drafts"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null,
+  		"special_use_attrib": "\\Drafts"
+  	},
+  	"Fondue": {
+  		"children": {
+  			"Savoyarde": {
+  				"attribs": ["\\HasNoChildren"],
+  				"delimiter": "/",
+  				"children": null,
+  				"parent": "[Circular]"
+  			}
+  		}
+  	},
+  	"Housse": {
+  		"children": {
+  			"De": {
+  				"children": {
+  					"Bateau": {
+  						"attribs": ["\\HasNoChildren"],
+  						"delimiter": "/",
+  						"children": null,
+  						"parent": "[Circular]"
+  					},
+  					"Rateau": {
+  						"attribs": ["\\HasNoChildren"],
+  						"delimiter": "/",
+  						"children": null,
+  						"parent": "[Circular]"
+  					}
+  				}
+  			}
+  		}
+  	},
+  	"JJJJJJJ JJJJJJJJ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"N1-Snoozed": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"Over the top": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"Sent": {
+  		"attribs": ["\\HasNoChildren", "\\Sent"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null,
+  		"special_use_attrib": "\\Sent"
+  	},
+  	"Spam": {
+  		"attribs": ["\\HasNoChildren", "\\Junk"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null,
+  		"special_use_attrib": "\\Junk"
+  	},
+  	"Taxes": {
+  		"attribs": ["\\HasNoChildren"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null
+  	},
+  	"Trash": {
+  		"attribs": ["\\HasNoChildren", "\\Trash"],
+  		"delimiter": "/",
+  		"children": null,
+  		"parent": null,
+  		"special_use_attrib": "\\Trash"
+  	}
+  },
+	"expectedFolders": [{
+		"role": "trash",
+		"name": "Trash"
+	}, {
+		"role": null,
+		"name": "Taxes"
+	}, {
+		"role": "spam",
+		"name": "Spam"
+	}, {
+		"role": "sent",
+		"name": "Sent"
+	}, {
+		"role": null,
+		"name": "Over the top"
+	}, {
+		"role": null,
+		"name": "N1-Snoozed"
+	}, {
+		"role": null,
+		"name": "JJJJJJJ JJJJJJJJ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1"
+	}, {
+		"role": "drafts",
+		"name": "Drafts"
+	}, {
+		"role": null,
+		"name": "Boîte de réception"
+	}, {
+		"role": null,
+		"name": "Arts"
+	}, {
+		"role": null,
+		"name": "Archive"
+	}, {
+		"role": "inbox",
+		"name": "INBOX"
+	}, {
+		"role": null,
+		"name": "2016"
+	}],
+	"expectedLabels": []
+}
diff --git a/packages/local-sync/spec/helpers.js b/packages/local-sync/spec/helpers.js
new file mode 100644
index 000000000..d767f5633
--- /dev/null
+++ b/packages/local-sync/spec/helpers.js
@@ -0,0 +1,28 @@
+const fs = require('fs');
+const path = require('path');
+
+const FIXTURES_PATH = path.join(__dirname, 'fixtures');
+const ACCOUNT_ID = 'test-account-id';
+
+function forEachJSONFixture(relativePath, callback) {
+  const fixturesDir = path.join(FIXTURES_PATH, relativePath);
+  const filenames = fs.readdirSync(fixturesDir).filter(f => f.endsWith('.json'));
+  filenames.forEach((filename) => {
+    const json = JSON.parse(fs.readFileSync(path.join(fixturesDir, filename)));
+    callback(filename, json);
+  });
+}
+
+const silentLogger = {
+  info: () => {},
+  warn: () => {},
+  debug: () => {},
+  error: () => {},
+}
+
+module.exports = {
+  FIXTURES_PATH,
+  ACCOUNT_ID,
+  silentLogger,
+  forEachJSONFixture,
+}
diff --git a/packages/local-sync/spec/imap/fetch-folder-list-spec.js b/packages/local-sync/spec/imap/fetch-folder-list-spec.js
new file mode 100644
index 000000000..ed2580ea1
--- /dev/null
+++ b/packages/local-sync/spec/imap/fetch-folder-list-spec.js
@@ -0,0 +1,45 @@
+
+const FetchFolderList = require('../../src/local-sync-worker/imap/fetch-folder-list');
+const LocalDatabaseConnector = require('../../src/shared/local-database-connector');
+const {forEachJSONFixture, ACCOUNT_ID, silentLogger} = require('../helpers');
+
+describe("FetchFolderList", function FetchFolderListSpecs() {
+  beforeEach(() => {
+    waitsForPromise(async () => {
+      await LocalDatabaseConnector.ensureAccountDatabase(ACCOUNT_ID);
+      this.db = await LocalDatabaseConnector.forAccount(ACCOUNT_ID);
+
+      this.stubImapBoxes = null;
+      this.imap = {
+        getBoxes: () => {
+          return Promise.resolve(this.stubImapBoxes);
+        },
+      };
+    });
+  });
+
+  afterEach(() => {
+    LocalDatabaseConnector.destroyAccountDatabase(ACCOUNT_ID)
+  })
+
+  describe("initial syncing", () => {
+    forEachJSONFixture('FetchFolderList', (filename, json) => {
+      it(`should create folders and labels correctly for boxes (${filename})`, () => {
+        waitsForPromise(async () => {
+          const {boxes, expectedFolders, expectedLabels} = json;
+          const provider = filename.split('-')[0];
+          this.stubImapBoxes = boxes;
+
+          const task = new FetchFolderList(provider, silentLogger);
+          await task.run(this.db, this.imap);
+
+          const folders = await this.db.Folder.findAll();
+          expect(folders.map((f) => { return {name: f.name, role: f.role} })).toEqual(expectedFolders);
+
+          const labels = await this.db.Label.findAll();
+          expect(labels.map(f => { return {name: f.name, role: f.role} })).toEqual(expectedLabels);
+        });
+      });
+    });
+  });
+});
diff --git a/packages/local-sync/spec/message-factory-spec.js b/packages/local-sync/spec/message-factory-spec.js
index c0643f26a..bd2ea44f0 100644
--- a/packages/local-sync/spec/message-factory-spec.js
+++ b/packages/local-sync/spec/message-factory-spec.js
@@ -1,35 +1,31 @@
-const path = require('path');
-const fs = require('fs');
 const LocalDatabaseConnector = require('../src/shared/local-database-connector');
 const {parseFromImap} = require('../src/shared/message-factory');
-
-const FIXTURES_PATH = path.join(__dirname, 'fixtures')
+const {forEachJSONFixture, ACCOUNT_ID} = require('./helpers');
 
 describe('MessageFactory', function MessageFactorySpecs() {
   beforeEach(() => {
     waitsForPromise(async () => {
-      const accountId = 'test-account-id';
-      await LocalDatabaseConnector.ensureAccountDatabase(accountId);
-      const db = await LocalDatabaseConnector.forAccount(accountId);
+      await LocalDatabaseConnector.ensureAccountDatabase(ACCOUNT_ID);
+      const db = await LocalDatabaseConnector.forAccount(ACCOUNT_ID);
       const folder = await db.Folder.create({
         id: 'test-folder-id',
-        accountId: accountId,
+        accountId: ACCOUNT_ID,
         version: 1,
         name: 'Test Folder',
         role: null,
       });
-      this.options = { accountId, db, folder };
+      this.options = { accountId: ACCOUNT_ID, db, folder };
     })
   })
 
-  describe("parseFromImap", () => {
-    const fixturesDir = path.join(FIXTURES_PATH, 'MessageFactory', 'parseFromImap');
-    const filenames = fs.readdirSync(fixturesDir).filter(f => f.endsWith('.json'));
+  afterEach(() => {
+    LocalDatabaseConnector.destroyAccountDatabase(ACCOUNT_ID)
+  })
 
-    filenames.forEach((filename) => {
+  describe("parseFromImap", () => {
+    forEachJSONFixture('MessageFactory/parseFromImap', (filename, json) => {
       it(`should correctly build message properties for ${filename}`, () => {
-        const inJSON = JSON.parse(fs.readFileSync(path.join(fixturesDir, filename)));
-        const {imapMessage, desiredParts, result} = inJSON;
+        const {imapMessage, desiredParts, result} = json;
 
         waitsForPromise(async () => {
           const actual = await parseFromImap(imapMessage, desiredParts, this.options);
diff --git a/packages/local-sync/spec/threading-spec.js b/packages/local-sync/spec/threading-spec.js
index 22e6a4615..f4d05ffa6 100644
--- a/packages/local-sync/spec/threading-spec.js
+++ b/packages/local-sync/spec/threading-spec.js
@@ -1,11 +1,9 @@
 /* eslint global-require: 0 */
 /* eslint import/no-dynamic-require: 0 */
-const path = require('path');
 const detectThread = require('../src/new-message-processor/detect-thread');
 const LocalDatabaseConnector = require('../src/shared/local-database-connector');
 
-const FIXTURES_PATH = path.join(__dirname, 'fixtures');
-const ACCOUNT_ID = 'test-account-threading';
+const {FIXTURES_PATH, ACCOUNT_ID} = require('./helpers')
 
 function messagesFromFixture({Message}, folder, name) {
   const {A, B} = require(`${FIXTURES_PATH}/Threading/${name}`)

From aed1d59916a54cfc62e27048ea5f82448daf6378 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Mon, 5 Dec 2016 16:20:17 -0800
Subject: [PATCH 462/800] [local-sync] fix(folder-list): Support children when
 parent has no attribs

---
 .../FetchFolderList/imap-inboxapptest1.json   | 291 +++++++++---------
 .../imap/fetch-folder-list.js                 |  21 +-
 2 files changed, 165 insertions(+), 147 deletions(-)

diff --git a/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json b/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json
index 2c2148fa8..00b5b3764 100644
--- a/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json
+++ b/packages/local-sync/spec/fixtures/FetchFolderList/imap-inboxapptest1.json
@@ -1,158 +1,167 @@
 {
-  "boxes": {
-  	"2016": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"INBOX": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"Archive": {
-  		"attribs": ["\\HasNoChildren", "\\Archive"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null,
-  		"special_use_attrib": "\\Archive"
-  	},
-  	"Arts": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"Boîte de réception": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"Drafts": {
-  		"attribs": ["\\HasNoChildren", "\\Drafts"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null,
-  		"special_use_attrib": "\\Drafts"
-  	},
-  	"Fondue": {
-  		"children": {
-  			"Savoyarde": {
-  				"attribs": ["\\HasNoChildren"],
-  				"delimiter": "/",
-  				"children": null,
-  				"parent": "[Circular]"
-  			}
-  		}
-  	},
-  	"Housse": {
-  		"children": {
-  			"De": {
-  				"children": {
-  					"Bateau": {
-  						"attribs": ["\\HasNoChildren"],
-  						"delimiter": "/",
-  						"children": null,
-  						"parent": "[Circular]"
-  					},
-  					"Rateau": {
-  						"attribs": ["\\HasNoChildren"],
-  						"delimiter": "/",
-  						"children": null,
-  						"parent": "[Circular]"
-  					}
-  				}
-  			}
-  		}
-  	},
-  	"JJJJJJJ JJJJJJJJ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"N1-Snoozed": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"Over the top": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"Sent": {
-  		"attribs": ["\\HasNoChildren", "\\Sent"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null,
-  		"special_use_attrib": "\\Sent"
-  	},
-  	"Spam": {
-  		"attribs": ["\\HasNoChildren", "\\Junk"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null,
-  		"special_use_attrib": "\\Junk"
-  	},
-  	"Taxes": {
-  		"attribs": ["\\HasNoChildren"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null
-  	},
-  	"Trash": {
-  		"attribs": ["\\HasNoChildren", "\\Trash"],
-  		"delimiter": "/",
-  		"children": null,
-  		"parent": null,
-  		"special_use_attrib": "\\Trash"
-  	}
-  },
+	"boxes": {
+		"2016": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"INBOX": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Archive": {
+			"attribs": ["\\HasNoChildren", "\\Archive"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null,
+			"special_use_attrib": "\\Archive"
+		},
+		"Arts": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Boîte de réception": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Drafts": {
+			"attribs": ["\\HasNoChildren", "\\Drafts"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null,
+			"special_use_attrib": "\\Drafts"
+		},
+		"Fondue": {
+			"children": {
+				"Savoyarde": {
+					"attribs": ["\\HasNoChildren"],
+					"delimiter": "/",
+					"children": null,
+					"parent": "[Circular]"
+				}
+			}
+		},
+		"Housse": {
+			"children": {
+				"De": {
+					"children": {
+						"Bateau": {
+							"attribs": ["\\HasNoChildren"],
+							"delimiter": "/",
+							"children": null,
+							"parent": "[Circular]"
+						},
+						"Rateau": {
+							"attribs": ["\\HasNoChildren"],
+							"delimiter": "/",
+							"children": null,
+							"parent": "[Circular]"
+						}
+					}
+				}
+			}
+		},
+		"JJJJJJJ JJJJJJJJ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"N1-Snoozed": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Over the top": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Sent": {
+			"attribs": ["\\HasNoChildren", "\\Sent"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null,
+			"special_use_attrib": "\\Sent"
+		},
+		"Spam": {
+			"attribs": ["\\HasNoChildren", "\\Junk"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null,
+			"special_use_attrib": "\\Junk"
+		},
+		"Taxes": {
+			"attribs": ["\\HasNoChildren"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null
+		},
+		"Trash": {
+			"attribs": ["\\HasNoChildren", "\\Trash"],
+			"delimiter": "/",
+			"children": null,
+			"parent": null,
+			"special_use_attrib": "\\Trash"
+		}
+	},
 	"expectedFolders": [{
-		"role": "trash",
-		"name": "Trash"
+		"name": "Trash",
+		"role": "trash"
 	}, {
-		"role": null,
-		"name": "Taxes"
+		"name": "Taxes",
+		"role": null
 	}, {
-		"role": "spam",
-		"name": "Spam"
+		"name": "Spam",
+		"role": "spam"
 	}, {
-		"role": "sent",
-		"name": "Sent"
+		"name": "Sent",
+		"role": "sent"
 	}, {
-		"role": null,
-		"name": "Over the top"
+		"name": "Over the top",
+		"role": null
 	}, {
-		"role": null,
-		"name": "N1-Snoozed"
+		"name": "N1-Snoozed",
+		"role": null
 	}, {
-		"role": null,
-		"name": "JJJJJJJ JJJJJJJJ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1"
+		"name": "JJJJJJJ JJJJJJJJ 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1",
+		"role": null
 	}, {
-		"role": "drafts",
-		"name": "Drafts"
+		"name": "Housse/De/Rateau",
+		"role": null
 	}, {
-		"role": null,
-		"name": "Boîte de réception"
+		"name": "Housse/De/Bateau",
+		"role": null
 	}, {
-		"role": null,
-		"name": "Arts"
+		"name": "Fondue/Savoyarde",
+		"role": null
 	}, {
-		"role": null,
-		"name": "Archive"
+		"name": "Drafts",
+		"role": "drafts"
 	}, {
-		"role": "inbox",
-		"name": "INBOX"
+		"name": "Boîte de réception",
+		"role": null
 	}, {
-		"role": null,
-		"name": "2016"
+		"name": "Arts",
+		"role": null
+	}, {
+		"name": "Archive",
+		"role": null
+	}, {
+		"name": "INBOX",
+		"role": "inbox"
+	}, {
+		"name": "2016",
+		"role": null
 	}],
 	"expectedLabels": []
 }
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index f3eb51eb2..b0ab88578 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -66,17 +66,26 @@ class FetchFolderList {
     const next = [];
 
     Object.keys(boxes).forEach((boxName) => {
-      stack.push([boxName, boxes[boxName]]);
+      stack.push([[boxName], boxes[boxName]]);
     });
 
     while (stack.length > 0) {
-      const [boxName, box] = stack.pop();
+      const [boxPath, box] = stack.pop();
+
       if (!box.attribs) {
-        // Some boxes seem to come back as partial objects. Not sure why, but
-        // I also can't access them via openMailbox. Possible node-imap i8n issue?
-        continue;
+        if (box.children) {
+          // In Fastmail, folders which are just containers for other folders
+          // have no attributes at all, just a children property. Add appropriate
+          // attribs so we can carry on.
+          box.attribs = ['\\HasChildren', '\\NoSelect'];
+        } else {
+          // Some boxes seem to come back as partial objects. Not sure why.
+          continue;
+        }
       }
 
+      const boxName = boxPath.join(box.delimiter);
+
       this._logger.info({
         box_name: boxName,
         attributes: JSON.stringify(box.attribs),
@@ -85,7 +94,7 @@ class FetchFolderList {
 
       if (box.children && box.attribs.includes('\\HasChildren')) {
         Object.keys(box.children).forEach((subname) => {
-          stack.push([`${boxName}${box.delimiter}${subname}`, box.children[subname]]);
+          stack.push([[].concat(boxPath, [subname]), box.children[subname]]);
         });
       }
 

From 1a2484006298c1a850e39480dc31631ac0003393 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Mon, 5 Dec 2016 18:50:11 -0800
Subject: [PATCH 463/800] [local-sync] Correctly sync folders and labels
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit will correctly keep track of folder and label ids when
creating them from N1.

Previously, when we sent the request to create a folder or label to our api,
we would immediately get back a serverId because it was created optimistically
in the back end— given that K2 is strictly non-optimistic, we won’t have a serverId
until some undetermined time in the future, and we need to somehow reference
the object that /was/ optimistically created in N1 to update the ui when
we do get the server id.

Since we can deterministically generate ids for folders and labels,
we "guess" what its going to be, and include it in the props of the syncback request
returned to N1. This is the simplest solution to get thing working
correctly right now, but we’ll need to revisit this in the future for
other types of objects (drafts, contacts, events), and revisit how we
will manage optimistic updates in N1 when we merge the 2 codebases
with K2 (given that K2 was designed to be non-optimisitc).
---
 .../local-sync/src/local-api/route-helpers.js |  17 +--
 .../src/local-api/routes/categories.js        | 102 ++++++++++++------
 .../imap/fetch-folder-list.js                 |   3 +-
 .../local-sync-worker/sync-process-manager.js |   2 +-
 .../src/local-sync-worker/sync-worker.js      |  13 +--
 ...k-factory.js => syncback-task-factory.es6} |  17 ++-
 .../syncback_tasks/create-category.imap.js    |  14 +++
 .../syncback_tasks/create-folder.imap.js      |  13 ---
 .../syncback_tasks/delete-folder.imap.js      |   9 +-
 .../syncback_tasks/delete-label.imap.js       |  14 +++
 .../syncback_tasks/rename-folder.imap.js      |   9 +-
 .../syncback_tasks/rename-label.imap.js       |  15 +++
 packages/local-sync/src/models/folder.js      |   8 +-
 packages/local-sync/src/models/label.js       |   6 ++
 .../local-sync/src/models/syncbackRequest.js  |   4 +-
 15 files changed, 167 insertions(+), 79 deletions(-)
 rename packages/local-sync/src/local-sync-worker/{syncback-task-factory.js => syncback-task-factory.es6} (82%)
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js
 delete mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/create-folder.imap.js
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js

diff --git a/packages/local-sync/src/local-api/route-helpers.js b/packages/local-sync/src/local-api/route-helpers.js
index 6dc9cc620..5f938d328 100644
--- a/packages/local-sync/src/local-api/route-helpers.js
+++ b/packages/local-sync/src/local-api/route-helpers.js
@@ -1,13 +1,14 @@
 const Serialization = require('./serialization');
+const SyncProcessManager = require('../local-sync-worker/sync-process-manager')
 
 module.exports = {
-  createSyncbackRequest: function createSyncbackRequest(request, reply, syncRequestArgs) {
-    request.getAccountDatabase().then((db) => {
-      const accountId = request.auth.credentials.id;
-      syncRequestArgs.accountId = accountId
-      db.SyncbackRequest.create(syncRequestArgs).then((syncbackRequest) => {
-        reply(Serialization.jsonStringify(syncbackRequest))
-      })
-    })
+  async createSyncbackRequest(request, reply, syncRequestArgs) {
+    const account = request.auth.credentials
+    syncRequestArgs.accountId = account.id
+
+    const db = await request.getAccountDatabase()
+    const syncbackRequest = await db.SyncbackRequest.create(syncRequestArgs)
+    SyncProcessManager.wakeWorkerForAccount(account)
+    reply(Serialization.jsonStringify(syncbackRequest))
   },
 }
diff --git a/packages/local-sync/src/local-api/routes/categories.js b/packages/local-sync/src/local-api/routes/categories.js
index c1e97de3a..0380c622c 100644
--- a/packages/local-sync/src/local-api/routes/categories.js
+++ b/packages/local-sync/src/local-api/routes/categories.js
@@ -25,16 +25,14 @@ module.exports = (server) => {
           ),
         },
       },
-      handler: (request, reply) => {
-        request.getAccountDatabase().then((db) => {
-          const Klass = db[klass];
-          Klass.findAll({
-            limit: request.query.limit,
-            offset: request.query.offset,
-          }).then((items) => {
-            reply(Serialization.jsonStringify(items));
-          })
+      async handler(request, reply) {
+        const db = await request.getAccountDatabase()
+        const Klass = db[klass];
+        const items = await Klass.findAll({
+          limit: request.query.limit,
+          offset: request.query.offset,
         })
+        reply(Serialization.jsonStringify(items));
       },
     });
 
@@ -47,17 +45,30 @@ module.exports = (server) => {
       config: {
         description: `Create ${term}`,
         tags: [term],
-        validate: {},
+        validate: {
+          params: {
+            payload: {
+              display_name: Joi.string().required(),
+            },
+          },
+        },
         response: {
           schema: Serialization.jsonSchema('SyncbackRequest'),
         },
       },
-      handler: (request, reply) => {
-        if (request.payload.display_name) {
+      async handler(request, reply) {
+        const {payload} = request
+        if (payload.display_name) {
+          const accountId = request.auth.credentials.id
+          const db = await request.getAccountDatabase()
+          const objectId = db[klass].hash({boxName: payload.display_name, accountId})
+
           createSyncbackRequest(request, reply, {
-            type: "CreateFolder",
+            type: "CreateCategory",
             props: {
-              displayName: request.payload.display_name,
+              objectId,
+              object: klass.toLowerCase(),
+              displayName: payload.display_name,
             },
           })
         }
@@ -72,22 +83,42 @@ module.exports = (server) => {
         tags: [term],
         validate: {
           params: {
-            id: Joi.string(),
+            id: Joi.string().required(),
+            payload: {
+              display_name: Joi.string().required(),
+            },
           },
         },
         response: {
           schema: Serialization.jsonSchema('SyncbackRequest'),
         },
       },
-      handler: (request, reply) => {
-        if (request.payload.display_name) {
-          createSyncbackRequest(request, reply, {
-            type: "RenameFolder",
-            props: {
-              displayName: request.payload.display_name,
-              id: request.params.id,
-            },
-          })
+      async handler(request, reply) {
+        const {payload} = request
+        if (payload.display_name) {
+          const accountId = request.auth.credentials.id
+          const db = await request.getAccountDatabase()
+          const objectId = db[klass].hash({boxName: payload.display_name, accountId})
+
+          if (klass === 'Label') {
+            createSyncbackRequest(request, reply, {
+              type: "RenameLabel",
+              props: {
+                objectId,
+                labelId: request.params.id,
+                displayName: payload.display_name,
+              },
+            })
+          } else {
+            createSyncbackRequest(request, reply, {
+              type: "RenameFolder",
+              props: {
+                objectId,
+                folderId: request.params.id,
+                displayName: payload.display_name,
+              },
+            })
+          }
         }
       },
     })
@@ -100,7 +131,7 @@ module.exports = (server) => {
         tags: [term],
         validate: {
           params: {
-            id: Joi.number().integer(),
+            id: Joi.string().required(),
           },
         },
         response: {
@@ -108,12 +139,21 @@ module.exports = (server) => {
         },
       },
       handler: (request, reply) => {
-        createSyncbackRequest(request, reply, {
-          type: "DeleteFolder",
-          props: {
-            id: request.params.id,
-          },
-        })
+        if (klass === 'Label') {
+          createSyncbackRequest(request, reply, {
+            type: "DeleteLabel",
+            props: {
+              labelId: request.params.id,
+            },
+          })
+        } else {
+          createSyncbackRequest(request, reply, {
+            type: "DeleteFolder",
+            props: {
+              folderId: request.params.id,
+            },
+          })
+        }
       },
     })
   });
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
index b0ab88578..40f4da679 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-folder-list.js
@@ -1,4 +1,3 @@
-const crypto = require('crypto')
 const {Provider, PromiseUtils} = require('isomorphic-core');
 const {localizedCategoryNames} = require('../sync-utils')
 
@@ -110,7 +109,7 @@ class FetchFolderList {
         const {accountId} = this._db
         category = Klass.build({
           accountId,
-          id: crypto.createHash('sha256').update(`${accountId}${boxName}`, 'utf8').digest('hex'),
+          id: Klass.hash({boxName, accountId}),
           name: boxName,
           role: role,
         });
diff --git a/packages/local-sync/src/local-sync-worker/sync-process-manager.js b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
index 3fcd2ac20..00f9bc892 100644
--- a/packages/local-sync/src/local-sync-worker/sync-process-manager.js
+++ b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
@@ -68,4 +68,4 @@ class SyncProcessManager {
   }
 }
 
-module.exports = new SyncProcessManager();
+module.exports = new SyncProcessManager()
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index d80532fd8..82f94886a 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -4,10 +4,7 @@ const {
   PromiseUtils,
 } = require('isomorphic-core');
 const LocalDatabaseConnector = require('../shared/local-database-connector')
-const {
-  jsonError,
-} = require('./sync-utils')
-
+const {jsonError} = require('./sync-utils')
 const FetchFolderList = require('./imap/fetch-folder-list')
 const FetchMessagesInFolder = require('./imap/fetch-messages-in-folder')
 const SyncbackTaskFactory = require('./syncback-task-factory')
@@ -112,8 +109,7 @@ class SyncWorker {
     conn.on('update', () => {
       this._onConnectionIdleUpdate();
     })
-    conn.on('queue-empty', () => {
-    });
+    conn.on('queue-empty', () => {});
 
     this._conn = conn;
     return await this._conn.connect();
@@ -123,9 +119,8 @@ class SyncWorker {
     const {SyncbackRequest} = this._db;
     const where = {where: {status: "NEW"}, limit: 100};
 
-    const tasks = (await SyncbackRequest.findAll(where)).map((req) =>
-      SyncbackTaskFactory.create(this._account, req)
-    );
+    const tasks = await SyncbackRequest.findAll(where)
+    .map((req) => SyncbackTaskFactory.create(this._account, req));
 
     return PromiseUtils.each(tasks, this.runSyncbackTask.bind(this));
   }
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js b/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
similarity index 82%
rename from packages/local-sync/src/local-sync-worker/syncback-task-factory.js
rename to packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
index 43d939074..dfd4257ef 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
@@ -4,6 +4,15 @@
  *
  */
 class SyncbackTaskFactory {
+
+  static TaskTypesAffectingMessageFolderUIDs = [
+    'MoveThreadToFolder',
+    'MoveMessageToFolder',
+    'SetThreadFolderAndLabels',
+    'RenameFolder',
+    'DeleteFolder',
+  ]
+
   static create(account, syncbackRequest) {
     let Task = null;
     switch (syncbackRequest.type) {
@@ -33,12 +42,16 @@ class SyncbackTaskFactory {
         Task = require('./syncback_tasks/star-message.imap'); break;
       case "UnstarMessage":
         Task = require('./syncback_tasks/unstar-message.imap'); break;
-      case "CreateFolder":
-        Task = require('./syncback_tasks/create-folder.imap'); break;
+      case "CreateCategory":
+        Task = require('./syncback_tasks/create-category.imap'); break;
       case "RenameFolder":
         Task = require('./syncback_tasks/rename-folder.imap'); break;
+      case "RenameLabel":
+        Task = require('./syncback_tasks/rename-label.imap'); break;
       case "DeleteFolder":
         Task = require('./syncback_tasks/delete-folder.imap'); break;
+      case "DeleteLabel":
+        Task = require('./syncback_tasks/delete-label.imap'); break;
       case "DeleteMessage":
         Task = require('./syncback_tasks/delete-message.imap'); break;
       case "SaveSentMessage":
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js
new file mode 100644
index 000000000..3c1e2b76f
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js
@@ -0,0 +1,14 @@
+const SyncbackTask = require('./syncback-task')
+
+class CreateCategoryIMAP extends SyncbackTask {
+  description() {
+    return `CreateCategory`;
+  }
+
+  async run(db, imap) {
+    const syncbackRequestObject = this.syncbackRequestObject()
+    const displayName = syncbackRequestObject.props.displayName
+    await imap.addBox(displayName)
+  }
+}
+module.exports = CreateCategoryIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/create-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/create-folder.imap.js
deleted file mode 100644
index c5ecec491..000000000
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/create-folder.imap.js
+++ /dev/null
@@ -1,13 +0,0 @@
-const SyncbackTask = require('./syncback-task')
-
-class CreateFolderIMAP extends SyncbackTask {
-  description() {
-    return `CreateFolder`;
-  }
-
-  run(db, imap) {
-    const folderName = this.syncbackRequestObject().props.displayName
-    return imap.addBox(folderName)
-  }
-}
-module.exports = CreateFolderIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js
index 005d6c9af..4629b0d3b 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js
@@ -5,11 +5,10 @@ class DeleteFolderIMAP extends SyncbackTask {
     return `DeleteFolder`;
   }
 
-  run(db, imap) {
-    const folderId = this.syncbackRequestObject().props.id
-    return db.Folder.findById(folderId).then((folder) => {
-      return imap.delBox(folder.name);
-    })
+  async run(db, imap) {
+    const folderId = this.syncbackRequestObject().props.folderId
+    const folder = await db.Folder.findById(folderId)
+    return imap.delBox(folder.name);
   }
 }
 module.exports = DeleteFolderIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js
new file mode 100644
index 000000000..498cc7c09
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js
@@ -0,0 +1,14 @@
+const SyncbackTask = require('./syncback-task')
+
+class DeleteLabelIMAP extends SyncbackTask {
+  description() {
+    return `DeleteLabel`;
+  }
+
+  async run(db, imap) {
+    const labelId = this.syncbackRequestObject().props.labelId
+    const label = await db.Label.findById(labelId)
+    return imap.delBox(label.name);
+  }
+}
+module.exports = DeleteLabelIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js
index 003e12bb3..70728afbe 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js
@@ -5,12 +5,11 @@ class RenameFolderIMAP extends SyncbackTask {
     return `RenameFolder`;
   }
 
-  run(db, imap) {
-    const folderId = this.syncbackRequestObject().props.id
+  async run(db, imap) {
+    const folderId = this.syncbackRequestObject().props.folderId
     const newFolderName = this.syncbackRequestObject().props.displayName
-    return db.Folder.findById(folderId).then((folder) => {
-      return imap.renameBox(folder.name, newFolderName);
-    })
+    const folder = await db.Folder.findById(folderId)
+    return imap.renameBox(folder.name, newFolderName);
   }
 }
 module.exports = RenameFolderIMAP
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js
new file mode 100644
index 000000000..b0779d8ab
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js
@@ -0,0 +1,15 @@
+const SyncbackTask = require('./syncback-task')
+
+class RenameLabelIMAP extends SyncbackTask {
+  description() {
+    return `RenameLabel`;
+  }
+
+  async run(db, imap) {
+    const labelId = this.syncbackRequestObject().props.labelId
+    const newLabelName = this.syncbackRequestObject().props.displayName
+    const folder = await db.Label.findById(labelId)
+    return imap.renameBox(folder.name, newLabelName);
+  }
+}
+module.exports = RenameLabelIMAP
diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.js
index 6a810c420..cea714496 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.js
@@ -1,3 +1,4 @@
+const crypto = require('crypto')
 const {DatabaseTypes: {buildJSONColumnOptions}} = require('isomorphic-core');
 const {formatImapPath} = require('../shared/imap-paths-utils');
 
@@ -21,10 +22,15 @@ module.exports = (sequelize, Sequelize) => {
       },
     ],
     classMethods: {
-      associate: ({Folder, Message, Thread}) => {
+      associate({Folder, Message, Thread}) {
         Folder.hasMany(Message)
         Folder.belongsToMany(Thread, {through: 'thread_folders'})
       },
+
+      hash({boxName, accountId}) {
+        const cleanName = formatImapPath(boxName)
+        return crypto.createHash('sha256').update(`${accountId}${cleanName}`, 'utf8').digest('hex')
+      },
     },
     instanceMethods: {
       toJSON: function toJSON() {
diff --git a/packages/local-sync/src/models/label.js b/packages/local-sync/src/models/label.js
index 5f1315bd9..a893b9830 100644
--- a/packages/local-sync/src/models/label.js
+++ b/packages/local-sync/src/models/label.js
@@ -1,3 +1,4 @@
+const crypto = require('crypto')
 const {formatImapPath} = require('../shared/imap-paths-utils');
 
 module.exports = (sequelize, Sequelize) => {
@@ -40,6 +41,11 @@ module.exports = (sequelize, Sequelize) => {
           where: sequelize.or({name: labelNames}, {role: labelRoles}),
         })
       },
+
+      hash({boxName, accountId}) {
+        const cleanName = formatImapPath(boxName)
+        return crypto.createHash('sha256').update(`${accountId}${cleanName}`, 'utf8').digest('hex')
+      },
     },
     instanceMethods: {
       imapLabelIdentifier() {
diff --git a/packages/local-sync/src/models/syncbackRequest.js b/packages/local-sync/src/models/syncbackRequest.js
index 4cfdaac21..4907b9232 100644
--- a/packages/local-sync/src/models/syncbackRequest.js
+++ b/packages/local-sync/src/models/syncbackRequest.js
@@ -13,14 +13,14 @@ module.exports = (sequelize, Sequelize) => {
     accountId: { type: Sequelize.STRING, allowNull: false },
   }, {
     instanceMethods: {
-      toJSON: function toJSON() {
+      toJSON() {
         return {
           id: `${this.id}`,
           type: this.type,
           error: JSON.stringify(this.error || {}),
           props: this.props,
           status: this.status,
-          object: 'providerSyncbackRequest',
+          object: 'syncbackRequest',
           account_id: this.accountId,
         }
       },

From e785d73bdc998e8caf977a005597175d4675334d Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Tue, 6 Dec 2016 11:34:30 -0800
Subject: [PATCH 464/800] [local-sync, iso-core] feat(send): add finishing
 touches to gmail multi-send

Delete the sent messages that gmail automatically creates and save our
generic form of the draft as a sent message.
---
 packages/isomorphic-core/src/imap-box.js      | 19 +++++++++++
 .../local-sync/src/local-api/routes/send.js   |  8 ++---
 .../src/local-api/sendmail-client.js          |  8 ++---
 .../syncback-task-factory.es6                 |  2 ++
 .../syncback_tasks/delete-message.imap.js     |  9 ++---
 .../delete-sent-message.gmail.js              | 33 +++++++++++++++++++
 .../perm-delete-message.imap.js               | 24 --------------
 .../syncback_tasks/save-sent-message.imap.js  | 26 ++++++++++++---
 8 files changed, 87 insertions(+), 42 deletions(-)
 create mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js
 delete mode 100644 packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js

diff --git a/packages/isomorphic-core/src/imap-box.js b/packages/isomorphic-core/src/imap-box.js
index ab91f5be4..01e17f4b7 100644
--- a/packages/isomorphic-core/src/imap-box.js
+++ b/packages/isomorphic-core/src/imap-box.js
@@ -1,4 +1,5 @@
 const _ = require('underscore');
+const PromiseUtils = require('./promise-utils')
 
 const {
   IMAPConnectionNotReadyError,
@@ -173,6 +174,24 @@ class IMAPBox {
     return this._conn._imap.delLabelsAsync(range, labels)
   }
 
+  append(rawMime, options) {
+    if (!this._conn._imap) {
+      throw new IMAPConnectionNotReadyError(`IMAPBox::append`)
+    }
+    return PromiseUtils.promisify(this._conn._imap.append).call(
+      this._conn._imap, rawMime, options
+    );
+  }
+
+  search(criteria) {
+    if (!this._conn._imap) {
+      throw new IMAPConnectionNotReadyError(`IMAPBox::search`)
+    }
+    return PromiseUtils.promisify(this._conn._imap.search).call(
+      this._conn._imap, criteria
+    );
+  }
+
   closeBox({expunge = true} = {}) {
     if (!this._conn._imap) {
       throw new IMAPConnectionNotReadyError(`IMAPBox::closeBox`)
diff --git a/packages/local-sync/src/local-api/routes/send.js b/packages/local-sync/src/local-api/routes/send.js
index da98c6d09..42cf7aaf8 100644
--- a/packages/local-sync/src/local-api/routes/send.js
+++ b/packages/local-sync/src/local-api/routes/send.js
@@ -168,10 +168,10 @@ module.exports = (server) => {
         // gmail creates sent messages for each one, go through and delete them
         if (account.provider === 'gmail') {
           try {
-            // TODO: use type: "PermananentDeleteMessage" once it's fully implemented
             await db.SyncbackRequest.create({
-              type: "DeleteMessage",
-              props: { messageId: draft.id },
+              accountId: account.id,
+              type: "DeleteSentMessage",
+              props: { messageId: `${draft.id}@nylas.com` },
             });
           } catch (err) {
             // Even if this fails, we need to finish the multi-send session,
@@ -185,7 +185,7 @@ module.exports = (server) => {
         await db.SyncbackRequest.create({
           accountId: account.id,
           type: "SaveSentMessage",
-          props: {rawMime},
+          props: {rawMime, messageId: `${draft.id}@nylas.com`},
         });
 
         await (draft.isSent = true);
diff --git a/packages/local-sync/src/local-api/sendmail-client.js b/packages/local-sync/src/local-api/sendmail-client.js
index e8da66dd6..659cf85b9 100644
--- a/packages/local-sync/src/local-api/sendmail-client.js
+++ b/packages/local-sync/src/local-api/sendmail-client.js
@@ -57,6 +57,7 @@ class SendmailClient {
     }
     msgData.subject = draft.subject;
     msgData.html = draft.body;
+    msgData.messageId = `${draft.id}@nylas.com`;
 
     // TODO: attachments
 
@@ -69,18 +70,17 @@ class SendmailClient {
     msgData.headers = draft.headers;
     msgData.headers['User-Agent'] = `NylasMailer-K2`
 
-    // TODO: do we want to set messageId or date?
-
     return msgData;
   }
 
   async buildMime(draft) {
     const builder = mailcomposer(this._draftToMsgData(draft))
-    return new Promise((resolve, reject) => {
+    const mimeNode = await (new Promise((resolve, reject) => {
       builder.build((error, result) => {
         error ? reject(error) : resolve(result)
       })
-    })
+    }));
+    return mimeNode.toString('ascii')
   }
 
   async send(draft) {
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6 b/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
index dfd4257ef..f86ca3a4c 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
@@ -56,6 +56,8 @@ class SyncbackTaskFactory {
         Task = require('./syncback_tasks/delete-message.imap'); break;
       case "SaveSentMessage":
         Task = require('./syncback_tasks/save-sent-message.imap'); break;
+      case "DeleteSentMessage":
+        Task = require('./syncback_tasks/delete-sent-message.gmail'); break;
       default:
         throw new Error(`Task type not defined in syncback-task-factory: ${syncbackRequest.type}`)
     }
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
index 5ec2d6d4c..a204e5b85 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
@@ -6,13 +6,10 @@ class DeleteMessageIMAP extends SyncbackTask {
     return `DeleteMessage`;
   }
 
-  run(db, imap) {
+  async run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
-
-    return TaskHelpers.openMessageBox({messageId, db, imap})
-      .then(({box, message}) => {
-        return box.addFlags(message.folderImapUID, 'DELETED')
-      })
+    const {box, message} = await TaskHelpers.openMessageBox({messageId, db, imap})
+    return box.addFlags(message.folderImapUID, ['DELETED'])
   }
 }
 module.exports = DeleteMessageIMAP;
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js
new file mode 100644
index 000000000..e34467baf
--- /dev/null
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js
@@ -0,0 +1,33 @@
+const SyncbackTask = require('./syncback-task')
+
+class DeleteSentMessageGMAIL extends SyncbackTask {
+  description() {
+    return `DeleteSentMessage`;
+  }
+
+  async run(db, imap) {
+    const {messageId} = this.syncbackRequestObject().props
+
+    const trash = await db.Folder.find({where: {role: 'trash'}});
+    if (!trash) { throw new Error(`Could not find folder with role 'trash'.`) }
+
+    const allMail = await db.Folder.find({where: {role: 'all'}});
+    if (!allMail) { throw new Error(`Could not find folder with role 'all'.`) }
+
+    // Move the message from all mail to trash and then delete it from there
+    const steps = [
+      {folder: allMail, deleteFn: (box, uid) => box.moveFromBox(uid, trash.name)},
+      {folder: trash, deleteFn: (box, uid) => box.addFlags(uid, 'DELETED')},
+    ]
+
+    for (const {folder, deleteFn} of steps) {
+      const box = await imap.openBox(folder.name);
+      const uids = await box.search([['HEADER', 'Message-ID', messageId]])
+      for (const uid of uids) {
+        await deleteFn(box, uid);
+      }
+      box.closeBox();
+    }
+  }
+}
+module.exports = DeleteSentMessageGMAIL;
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js
deleted file mode 100644
index 96488595c..000000000
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/perm-delete-message.imap.js
+++ /dev/null
@@ -1,24 +0,0 @@
-const SyncbackTask = require('./syncback-task')
-
-class PermanentlyDeleteMessageIMAP extends SyncbackTask {
-  description() {
-    return `PermanentlyDeleteMessage`;
-  }
-
-  async run(db, imap) {
-    const messageId = this.syncbackRequestObject().props.messageId
-    const message = await db.Message.findById(messageId);
-    const folder = await db.Folder.findById(message.folderId);
-    const box = await imap.openBox(folder.name);
-    const result = await box.addFlags(message.folderImapUID, 'DELETED');
-    return result;
-
-    // TODO: We need to also delete the message from the trash
-    // if (folder.role === 'trash') { return result; }
-    //
-    // const trash = await db.Folder.find({where: {role: 'trash'}});
-    // const trashBox = await imap.openBox(trash.name);
-    // return await trashBox.addFlags(message.folderImapUID, 'DELETED');
-  }
-}
-module.exports = PermanentlyDeleteMessageIMAP;
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
index 2eb0cb6c1..ec3871e59 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
@@ -6,10 +6,28 @@ class SaveSentMessageIMAP extends SyncbackTask {
   }
 
   async run(db, imap) {
-    // TODO: gmail doesn't have a sent folder
-    const folder = await db.Folder.find({where: {role: 'sent'}});
-    const box = await imap.openBox(folder.name);
-    return box.append(this.syncbackRequestObject().props.rawMime);
+    const {rawMime, messageId} = this.syncbackRequestObject().props;
+
+    // Non-gmail
+    const sentFolder = await db.Folder.find({where: {role: 'sent'}});
+    if (sentFolder) {
+      const box = await imap.openBox(sentFolder.name);
+      return box.append(rawMime);
+    }
+
+    // Gmail, we need to add the message to all mail and add the sent label
+    const sentLabel = await db.Label.find({where: {role: 'sent'}});
+    const allMail = await db.Folder.find({where: {role: 'all'}});
+    if (sentLabel && allMail) {
+      let box = await imap.openBox(allMail.name);
+      await box.append(rawMime, {flags: 'SEEN'})
+      const uids = await box.search([['HEADER', 'Message-ID', messageId]])
+      // There should only be one uid in the array
+      return box.setLabels(uids[0], '\\Sent');
+    }
+
+    throw new Error('Could not save message to sent folder.')
   }
 }
+
 module.exports = SaveSentMessageIMAP;

From fda7fe18cc2615ebb7e7ab2840b2668c87120ae1 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Tue, 6 Dec 2016 15:34:28 -0800
Subject: [PATCH 465/800] [local-sync] Ensure order correct order when running
 syncback requests

When running syncback requests, if a task is meant to change the UID of
a message (e.g. move it to a new folder), that task should be run after
other tasks that don't affect the UID. Otherwise, when trying to run the
other tasks, they would reference a UID that is no longer valid.

This commit will make sure that we run any tasks that will change message uids last,
/and/ make sure that we don't run more than 1 task that will affect the uids of
the same messages in a row (i.e. without running a sync loop in between)
---
 .../src/local-sync-worker/sync-worker.js      | 60 +++++++++++++++++--
 ...k-factory.es6 => syncback-task-factory.js} |  8 ---
 .../syncback_tasks/create-category.imap.js    |  4 ++
 .../syncback_tasks/delete-folder.imap.js      |  4 ++
 .../syncback_tasks/delete-label.imap.js       |  4 ++
 .../syncback_tasks/delete-message.imap.js     |  4 ++
 .../mark-message-as-read.imap.js              |  4 ++
 .../mark-message-as-unread.imap.js            |  4 ++
 .../mark-thread-as-read.imap.js               |  4 ++
 .../mark-thread-as-unread.imap.js             |  4 ++
 .../move-message-to-folder.imap.js            |  4 ++
 .../move-thread-to-folder.imap.js             |  4 ++
 .../syncback_tasks/rename-folder.imap.js      |  4 ++
 .../syncback_tasks/rename-label.imap.js       |  4 ++
 .../syncback_tasks/save-sent-message.imap.js  |  4 ++
 .../syncback_tasks/set-message-labels.imap.js |  4 ++
 .../set-thread-folder-and-labels.imap.js      |  4 ++
 .../syncback_tasks/set-thread-labels.imap.js  |  4 ++
 .../syncback_tasks/star-message.imap.js       |  4 ++
 .../syncback_tasks/star-thread.imap.js        |  4 ++
 .../syncback_tasks/syncback-task.js           |  4 ++
 .../syncback_tasks/unstar-message.imap.js     |  4 ++
 .../syncback_tasks/unstar-thread.imap.js      |  4 ++
 23 files changed, 138 insertions(+), 14 deletions(-)
 rename packages/local-sync/src/local-sync-worker/{syncback-task-factory.es6 => syncback-task-factory.js} (93%)

diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index 82f94886a..c70fce3ad 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -115,14 +115,62 @@ class SyncWorker {
     return await this._conn.connect();
   }
 
-  async syncbackMessageActions() {
-    const {SyncbackRequest} = this._db;
-    const where = {where: {status: "NEW"}, limit: 100};
+  async runNewSyncbackRequests() {
+    const {SyncbackRequest, Message} = this._db;
+    const where = {
+      limit: 100,
+      where: {status: "NEW"},
+      order: [['createdAt', 'ASC']],
+    };
 
+    // Make sure we run the tasks that affect IMAP uids last, and that we don't
+    // run 2 tasks that will affect the same set of UIDS together (i.e. without
+    // running a sync loop in between)
     const tasks = await SyncbackRequest.findAll(where)
-    .map((req) => SyncbackTaskFactory.create(this._account, req));
+    .map((req) => SyncbackTaskFactory.create(this._account, req))
 
-    return PromiseUtils.each(tasks, this.runSyncbackTask.bind(this));
+    if (tasks.length === 0) { return Promise.resolve() }
+
+    const tasksToProcess = tasks.filter(t => !t.affectsImapMessageUIDs())
+    const tasksAffectingUIDs = tasks.filter(t => t.affectsImapMessageUIDs())
+
+    const changeFolderTasks = tasksAffectingUIDs.filter(t =>
+      t.description() === 'RenameFolder' || t.description() === 'DeleteFolder'
+    )
+    if (changeFolderTasks.length > 0) {
+      // If we are renaming or deleting folders, those are the only tasks we
+      // want to process before executing any other tasks that may change UIDs
+      const affectedFolderIds = new Set()
+      changeFolderTasks.forEach((task) => {
+        const {props: {folderId}} = task.syncbackRequestObject()
+        if (folderId && !affectedFolderIds.has(folderId)) {
+          tasksToProcess.push(task)
+          affectedFolderIds.add(folderId)
+        }
+      })
+      return PromiseUtils.each(tasks, (task) => this.runSyncbackTask(task));
+    }
+
+    // Otherwise, make sure that we don't process more than 1 task that will affect
+    // the UID of the same message
+    const affectedMessageIds = new Set()
+    for (const task of tasksAffectingUIDs) {
+      const {props: {messageId, threadId}} = task.syncbackRequestObject()
+      if (messageId) {
+        if (!affectedMessageIds.has(messageId)) {
+          tasksToProcess.push(task)
+          affectedMessageIds.add(messageId)
+        }
+      } else if (threadId) {
+        const messageIds = await Message.findAll({where: {threadId}}).map(m => m.id)
+        const shouldIncludeTask = messageIds.every(id => !affectedMessageIds.has(id))
+        if (shouldIncludeTask) {
+          tasksToProcess.push(task)
+          messageIds.forEach(id => affectedMessageIds.add(id))
+        }
+      }
+    }
+    return PromiseUtils.each(tasks, (task) => this.runSyncbackTask(task));
   }
 
   async runSyncbackTask(task) {
@@ -176,7 +224,7 @@ class SyncWorker {
     try {
       await this._account.update({syncError: null});
       await this.ensureConnection();
-      await this.syncbackMessageActions();
+      await this.runNewSyncbackRequests();
       await this._conn.runOperation(new FetchFolderList(this._account.provider, this._logger));
       await this.syncMessagesInAllFolders();
       await this.onSyncDidComplete();
diff --git a/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6 b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
similarity index 93%
rename from packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
rename to packages/local-sync/src/local-sync-worker/syncback-task-factory.js
index f86ca3a4c..8a24a630b 100644
--- a/packages/local-sync/src/local-sync-worker/syncback-task-factory.es6
+++ b/packages/local-sync/src/local-sync-worker/syncback-task-factory.js
@@ -5,14 +5,6 @@
  */
 class SyncbackTaskFactory {
 
-  static TaskTypesAffectingMessageFolderUIDs = [
-    'MoveThreadToFolder',
-    'MoveMessageToFolder',
-    'SetThreadFolderAndLabels',
-    'RenameFolder',
-    'DeleteFolder',
-  ]
-
   static create(account, syncbackRequest) {
     let Task = null;
     switch (syncbackRequest.type) {
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js
index 3c1e2b76f..869f2a3a3 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/create-category.imap.js
@@ -5,6 +5,10 @@ class CreateCategoryIMAP extends SyncbackTask {
     return `CreateCategory`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const syncbackRequestObject = this.syncbackRequestObject()
     const displayName = syncbackRequestObject.props.displayName
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js
index 4629b0d3b..2771aff61 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-folder.imap.js
@@ -5,6 +5,10 @@ class DeleteFolderIMAP extends SyncbackTask {
     return `DeleteFolder`;
   }
 
+  affectsImapMessageUIDs() {
+    return true
+  }
+
   async run(db, imap) {
     const folderId = this.syncbackRequestObject().props.folderId
     const folder = await db.Folder.findById(folderId)
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js
index 498cc7c09..a269e0f11 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-label.imap.js
@@ -5,6 +5,10 @@ class DeleteLabelIMAP extends SyncbackTask {
     return `DeleteLabel`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const labelId = this.syncbackRequestObject().props.labelId
     const label = await db.Label.findById(labelId)
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
index a204e5b85..516f84080 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-message.imap.js
@@ -6,6 +6,10 @@ class DeleteMessageIMAP extends SyncbackTask {
     return `DeleteMessage`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
     const {box, message} = await TaskHelpers.openMessageBox({messageId, db, imap})
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-read.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-read.imap.js
index 26dfb6faf..f1c8c8995 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-read.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-read.imap.js
@@ -6,6 +6,10 @@ class MarkMessageAsReadIMAP extends SyncbackTask {
     return `MarkMessageAsRead`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-unread.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-unread.imap.js
index f7f7a3484..ec4688914 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-unread.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-message-as-unread.imap.js
@@ -6,6 +6,10 @@ class MarkMessageAsUnreadIMAP extends SyncbackTask {
     return `MarkMessageAsUnread`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-read.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-read.imap.js
index c74ac249c..b90b107cf 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-read.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-read.imap.js
@@ -6,6 +6,10 @@ class MarkThreadAsRead extends SyncbackTask {
     return `MarkThreadAsRead`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-unread.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-unread.imap.js
index 72ae4068a..e0f214003 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-unread.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/mark-thread-as-unread.imap.js
@@ -6,6 +6,10 @@ class MarkThreadAsUnread extends SyncbackTask {
     return `MarkThreadAsUnread`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js
index 5beecb32f..53376edfd 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-message-to-folder.imap.js
@@ -6,6 +6,10 @@ class MoveMessageToFolderIMAP extends SyncbackTask {
     return `MoveMessageToFolder`;
   }
 
+  affectsImapMessageUIDs() {
+    return true
+  }
+
   async run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
     const targetFolderId = this.syncbackRequestObject().props.folderId
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
index 1ab698215..345bdb72e 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/move-thread-to-folder.imap.js
@@ -6,6 +6,10 @@ class MoveThreadToFolderIMAP extends SyncbackTask {
     return `MoveThreadToFolder`;
   }
 
+  affectsImapMessageUIDs() {
+    return true
+  }
+
   async run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
     const targetFolderId = this.syncbackRequestObject().props.folderId
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js
index 70728afbe..aea8f53df 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-folder.imap.js
@@ -5,6 +5,10 @@ class RenameFolderIMAP extends SyncbackTask {
     return `RenameFolder`;
   }
 
+  affectsImapMessageUIDs() {
+    return true
+  }
+
   async run(db, imap) {
     const folderId = this.syncbackRequestObject().props.folderId
     const newFolderName = this.syncbackRequestObject().props.displayName
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js
index b0779d8ab..c5ca7448d 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/rename-label.imap.js
@@ -5,6 +5,10 @@ class RenameLabelIMAP extends SyncbackTask {
     return `RenameLabel`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const labelId = this.syncbackRequestObject().props.labelId
     const newLabelName = this.syncbackRequestObject().props.displayName
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
index ec3871e59..4f5a5e22c 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/save-sent-message.imap.js
@@ -5,6 +5,10 @@ class SaveSentMessageIMAP extends SyncbackTask {
     return `SaveSentMessage`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const {rawMime, messageId} = this.syncbackRequestObject().props;
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js
index 19aa1bf09..0b749dc1e 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-message-labels.imap.js
@@ -6,6 +6,10 @@ class SetMessageLabelsIMAP extends SyncbackTask {
     return `SetMessageLabels`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
     const labelIds = this.syncbackRequestObject().props.labelIds
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js
index 15f0adc0d..51dad5684 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-folder-and-labels.imap.js
@@ -6,6 +6,10 @@ class SetThreadFolderAndLabelsIMAP extends SyncbackTask {
     return `SetThreadFolderAndLabels`;
   }
 
+  affectsImapMessageUIDs() {
+    return true
+  }
+
   async run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
     const labelIds = this.syncbackRequestObject().props.labelIds
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
index 4bc6c1351..b1de79855 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/set-thread-labels.imap.js
@@ -6,6 +6,10 @@ class SetThreadLabelsIMAP extends SyncbackTask {
     return `SetThreadLabels`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   async run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
     const labelIds = this.syncbackRequestObject().props.labelIds
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/star-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/star-message.imap.js
index 8ae5a4f59..5ff066546 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/star-message.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/star-message.imap.js
@@ -6,6 +6,10 @@ class StarMessageIMAP extends SyncbackTask {
     return `StarMessage`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/star-thread.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/star-thread.imap.js
index 71083f96f..d6fd83072 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/star-thread.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/star-thread.imap.js
@@ -6,6 +6,10 @@ class StarThread extends SyncbackTask {
     return `StarThread`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/syncback-task.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/syncback-task.js
index 6379bc11b..c4d20bf74 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/syncback-task.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/syncback-task.js
@@ -12,6 +12,10 @@ class SyncbackTask {
     throw new Error("Must return a description")
   }
 
+  affectsImapMessageUIDs() {
+    throw new Error("Must implement `affectsImapMessageUIDs`")
+  }
+
   run() {
     throw new Error("Must implement a run method")
   }
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-message.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-message.imap.js
index c4f9cbd85..944035dfd 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-message.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-message.imap.js
@@ -6,6 +6,10 @@ class UnstarMessageIMAP extends SyncbackTask {
     return `UnstarMessage`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const messageId = this.syncbackRequestObject().props.messageId
 
diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-thread.imap.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-thread.imap.js
index c1652318c..ce91472ae 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-thread.imap.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/unstar-thread.imap.js
@@ -6,6 +6,10 @@ class UnstarThread extends SyncbackTask {
     return `UnstarThread`;
   }
 
+  affectsImapMessageUIDs() {
+    return false
+  }
+
   run(db, imap) {
     const threadId = this.syncbackRequestObject().props.threadId
 

From 7468e8123e06c8c7b7bd237afdbbabb1f22ab91f Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Tue, 6 Dec 2016 15:54:45 -0800
Subject: [PATCH 466/800] [local-sync] Update DeleteSentMessage task for
 ordering

---
 .../syncback_tasks/delete-sent-message.gmail.js               | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js
index e34467baf..66378e8c1 100644
--- a/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js
+++ b/packages/local-sync/src/local-sync-worker/syncback_tasks/delete-sent-message.gmail.js
@@ -5,6 +5,10 @@ class DeleteSentMessageGMAIL extends SyncbackTask {
     return `DeleteSentMessage`;
   }
 
+  affectsImapMessageUIDs() {
+    return true
+  }
+
   async run(db, imap) {
     const {messageId} = this.syncbackRequestObject().props
 

From 896f981408be6a686d46e47d59c7018aa75f869f Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Mon, 5 Dec 2016 17:59:47 -0800
Subject: [PATCH 467/800] [isomorphic-core] extract AuthHelpers to DRY

---
 packages/isomorphic-core/index.js             |  5 +-
 packages/isomorphic-core/package.json         |  1 +
 packages/isomorphic-core/src/auth-helpers.js  | 41 +++++++++++++++++
 .../local-sync/src/local-api/routes/auth.js   | 46 ++-----------------
 4 files changed, 48 insertions(+), 45 deletions(-)
 create mode 100644 packages/isomorphic-core/src/auth-helpers.js

diff --git a/packages/isomorphic-core/index.js b/packages/isomorphic-core/index.js
index 67721a8b6..ef22e6018 100644
--- a/packages/isomorphic-core/index.js
+++ b/packages/isomorphic-core/index.js
@@ -5,11 +5,12 @@ module.exports = {
     IMAP: 'imap',
   },
   Imap: require('imap'),
-  IMAPConnection: require('./src/imap-connection'),
   IMAPErrors: require('./src/imap-errors'),
+  loadModels: require('./src/load-models'),
+  AuthHelpers: require('./src/auth-helpers'),
   PromiseUtils: require('./src/promise-utils'),
   DatabaseTypes: require('./src/database-types'),
-  loadModels: require('./src/load-models'),
+  IMAPConnection: require('./src/imap-connection'),
   DeltaStreamBuilder: require('./src/delta-stream-builder'),
   HookTransactionLog: require('./src/hook-transaction-log'),
   HookIncrementVersionOnSave: require('./src/hook-increment-version-on-save'),
diff --git a/packages/isomorphic-core/package.json b/packages/isomorphic-core/package.json
index 6a5c04e21..a1a41509b 100644
--- a/packages/isomorphic-core/package.json
+++ b/packages/isomorphic-core/package.json
@@ -5,6 +5,7 @@
   "main": "index.js",
   "dependencies": {
     "imap": "0.8.18",
+    "joi": "8.4.2",
     "promise-props": "1.0.0",
     "promise.prototype.finally": "1.0.1",
     "rx": "4.1.0",
diff --git a/packages/isomorphic-core/src/auth-helpers.js b/packages/isomorphic-core/src/auth-helpers.js
new file mode 100644
index 000000000..b60f646cc
--- /dev/null
+++ b/packages/isomorphic-core/src/auth-helpers.js
@@ -0,0 +1,41 @@
+const Joi = require('joi');
+
+const imapSmtpSettings = Joi.object().keys({
+  imap_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
+  imap_port: Joi.number().integer().required(),
+  imap_username: Joi.string().required(),
+  imap_password: Joi.string().required(),
+  smtp_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
+  smtp_port: Joi.number().integer().required(),
+  smtp_username: Joi.string().required(),
+  smtp_password: Joi.string().required(),
+  ssl_required: Joi.boolean().required(),
+}).required();
+
+const resolvedGmailSettings = Joi.object().keys({
+  xoauth2: Joi.string().required(),
+}).required();
+
+const exchangeSettings = Joi.object().keys({
+  username: Joi.string().required(),
+  password: Joi.string().required(),
+  eas_server_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
+}).required();
+
+module.exports = {
+  authPostConfig() {
+    return {
+      description: 'Authenticates a new account.',
+      tags: ['accounts'],
+      auth: false,
+      validate: {
+        payload: {
+          email: Joi.string().email().required(),
+          name: Joi.string().required(),
+          provider: Joi.string().valid('imap', 'gmail').required(),
+          settings: Joi.alternatives().try(imapSmtpSettings, exchangeSettings, resolvedGmailSettings),
+        },
+      },
+    }
+  },
+}
diff --git a/packages/local-sync/src/local-api/routes/auth.js b/packages/local-sync/src/local-api/routes/auth.js
index ca9633231..948901832 100644
--- a/packages/local-sync/src/local-api/routes/auth.js
+++ b/packages/local-sync/src/local-api/routes/auth.js
@@ -3,35 +3,14 @@ const _ = require('underscore');
 const crypto = require('crypto');
 const Serialization = require('../serialization');
 const {
-  IMAPConnection,
   IMAPErrors,
+  AuthHelpers,
+  IMAPConnection,
 } = require('isomorphic-core');
 const DefaultSyncPolicy = require('../default-sync-policy')
 const LocalDatabaseConnector = require('../../shared/local-database-connector')
 const SyncProcessManager = require('../../local-sync-worker/sync-process-manager')
 
-const imapSmtpSettings = Joi.object().keys({
-  imap_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
-  imap_port: Joi.number().integer().required(),
-  imap_username: Joi.string().required(),
-  imap_password: Joi.string().required(),
-  smtp_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
-  smtp_port: Joi.number().integer().required(),
-  smtp_username: Joi.string().required(),
-  smtp_password: Joi.string().required(),
-  ssl_required: Joi.boolean().required(),
-}).required();
-
-const resolvedGmailSettings = Joi.object().keys({
-  xoauth2: Joi.string().required(),
-}).required();
-
-const exchangeSettings = Joi.object().keys({
-  username: Joi.string().required(),
-  password: Joi.string().required(),
-  eas_server_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
-}).required();
-
 const buildAccountWith = ({name, email, provider, settings, credentials}) => {
   return LocalDatabaseConnector.forShared().then((db) => {
     const {AccountToken, Account} = db;
@@ -69,26 +48,7 @@ module.exports = (server) => {
   server.route({
     method: 'POST',
     path: '/auth',
-    config: {
-      description: 'Authenticates a new account.',
-      notes: 'Notes go here',
-      tags: ['accounts'],
-      auth: false,
-      validate: {
-        payload: {
-          email: Joi.string().email().required(),
-          name: Joi.string().required(),
-          provider: Joi.string().valid('imap', 'gmail').required(),
-          settings: Joi.alternatives().try(imapSmtpSettings, exchangeSettings, resolvedGmailSettings),
-        },
-      },
-      response: {
-        schema: Joi.alternatives().try(
-          Serialization.jsonSchema('Account'),
-          Serialization.jsonSchema('Error')
-        ),
-      },
-    },
+    config: AuthHelpers.authPostConfig(),
     handler: (request, reply) => {
       const dbStub = {};
       const connectionChecks = [];

From 2cbb90bb3bbb73539a672322ccdffa14e85a00a8 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 6 Dec 2016 10:42:32 -0800
Subject: [PATCH 468/800] [*] DRY Auth

---
 packages/isomorphic-core/src/auth-helpers.js  |  71 ++++++++++-
 .../isomorphic-core/src/models/account.js     |  22 +++-
 .../src/local-api/routes/accounts.js          |   2 +-
 .../local-sync/src/local-api/routes/auth.js   | 113 ++----------------
 4 files changed, 102 insertions(+), 106 deletions(-)

diff --git a/packages/isomorphic-core/src/auth-helpers.js b/packages/isomorphic-core/src/auth-helpers.js
index b60f646cc..3204316d3 100644
--- a/packages/isomorphic-core/src/auth-helpers.js
+++ b/packages/isomorphic-core/src/auth-helpers.js
@@ -1,4 +1,7 @@
+const _ = require('underscore')
 const Joi = require('joi');
+const IMAPErrors = require('./imap-errors')
+const IMAPConnection = require('./imap-connection')
 
 const imapSmtpSettings = Joi.object().keys({
   imap_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
@@ -23,7 +26,7 @@ const exchangeSettings = Joi.object().keys({
 }).required();
 
 module.exports = {
-  authPostConfig() {
+  imapAuthRouteConfig() {
     return {
       description: 'Authenticates a new account.',
       tags: ['accounts'],
@@ -38,4 +41,70 @@ module.exports = {
       },
     }
   },
+
+  imapAuthHandler(accountBuildFn) {
+    return (request, reply) => {
+      const dbStub = {};
+      const connectionChecks = [];
+      const {settings, email, provider, name} = request.payload;
+
+      let connectionSettings = null;
+      let connectionCredentials = null;
+
+      if (provider === 'imap') {
+        connectionSettings = _.pick(settings, [
+          'imap_host', 'imap_port',
+          'smtp_host', 'smtp_port',
+          'ssl_required',
+        ]);
+        connectionCredentials = _.pick(settings, [
+          'imap_username', 'imap_password',
+          'smtp_username', 'smtp_password',
+        ]);
+      }
+
+      if (provider === 'gmail') {
+        connectionSettings = {
+          imap_username: email,
+          imap_host: 'imap.gmail.com',
+          imap_port: 993,
+          smtp_username: email,
+          smtp_host: 'smtp.gmail.com',
+          smtp_port: 465,
+          ssl_required: true,
+        }
+        connectionCredentials = {
+          xoauth2: settings.xoauth2,
+        }
+      }
+
+      connectionChecks.push(IMAPConnection.connect({
+        settings: Object.assign({}, connectionSettings, connectionCredentials),
+        logger: request.logger,
+        db: dbStub,
+      }));
+
+      Promise.all(connectionChecks).then((conns) => {
+        for (const conn of conns) {
+          if (conn) { conn.end(); }
+        }
+        const accountParams = {
+          name: name,
+          provider: provider,
+          emailAddress: email,
+          connectionSettings: connectionSettings,
+        }
+        return accountBuildFn(accountParams, connectionCredentials)
+      })
+      .then(({account, token}) => {
+        const response = account.toJSON();
+        response.account_token = token.value;
+        reply(JSON.stringify(response));
+      })
+      .catch((err) => {
+        const code = err instanceof IMAPErrors.IMAPAuthenticationError ? 401 : 400
+        reply({message: err.message, type: "api_error"}).code(code);
+      })
+    }
+  },
 }
diff --git a/packages/isomorphic-core/src/models/account.js b/packages/isomorphic-core/src/models/account.js
index 557fd86de..8577cfb89 100644
--- a/packages/isomorphic-core/src/models/account.js
+++ b/packages/isomorphic-core/src/models/account.js
@@ -3,6 +3,8 @@ const {buildJSONColumnOptions, buildJSONARRAYColumnOptions} = require('../databa
 
 const {DB_ENCRYPTION_ALGORITHM, DB_ENCRYPTION_PASSWORD} = process.env;
 
+const AccountToken = require('./account-token')
+
 module.exports = (sequelize, Sequelize) => {
   const Account = sequelize.define('account', {
     id: { type: Sequelize.STRING(65), primaryKey: true },
@@ -27,8 +29,24 @@ module.exports = (sequelize, Sequelize) => {
       },
     ],
     classMethods: {
-      associate: ({AccountToken}) => {
-        Account.hasMany(AccountToken, {as: 'tokens'})
+      associate: (data = {}) => {
+        Account.hasMany(data.AccountToken, {as: 'tokens'})
+      },
+      upsertWithCredentials: (accountParams, credentials) => {
+        const idString = `${accountParams.email}${JSON.stringify(accountParams.settings)}`
+        const id = crypto.createHash('sha256').update(idString, 'utf8').digest('hex')
+        return Account.findById(id).then((existing) => {
+          const account = existing || Account.build(Object.assign({id}, accountParams))
+
+          // always update with the latest credentials
+          account.setCredentials(credentials);
+
+          return account.save().then((saved) =>
+            AccountToken.create({accountId: saved.id}).then((token) =>
+              Promise.resolve({account: saved, token: token})
+            )
+          );
+        });
       },
     },
     instanceMethods: {
diff --git a/packages/local-sync/src/local-api/routes/accounts.js b/packages/local-sync/src/local-api/routes/accounts.js
index 1f13d4b6b..d77776e68 100644
--- a/packages/local-sync/src/local-api/routes/accounts.js
+++ b/packages/local-sync/src/local-api/routes/accounts.js
@@ -23,7 +23,7 @@ module.exports = (server) => {
               accounts.map(({id}) => AccountToken.create({accountId: id, value: id})
             )).finally(() =>
               reply(accounts.map((account) =>
-                Object.assign(account.toJSON(), {id: `${account.id}`, auth_token: `${account.id}`})
+                Object.assign(account.toJSON(), {id: `${account.id}`, account_token: `${account.id}`})
               ))
             )
           });
diff --git a/packages/local-sync/src/local-api/routes/auth.js b/packages/local-sync/src/local-api/routes/auth.js
index 948901832..71c9e1750 100644
--- a/packages/local-sync/src/local-api/routes/auth.js
+++ b/packages/local-sync/src/local-api/routes/auth.js
@@ -1,46 +1,17 @@
-const Joi = require('joi');
-const _ = require('underscore');
-const crypto = require('crypto');
-const Serialization = require('../serialization');
-const {
-  IMAPErrors,
-  AuthHelpers,
-  IMAPConnection,
-} = require('isomorphic-core');
+const { AuthHelpers } = require('isomorphic-core');
 const DefaultSyncPolicy = require('../default-sync-policy')
 const LocalDatabaseConnector = require('../../shared/local-database-connector')
 const SyncProcessManager = require('../../local-sync-worker/sync-process-manager')
 
-const buildAccountWith = ({name, email, provider, settings, credentials}) => {
-  return LocalDatabaseConnector.forShared().then((db) => {
-    const {AccountToken, Account} = db;
+const accountBuildFn = (accountParams, credentials) => {
+  return LocalDatabaseConnector.forShared().then(({Account}) => {
+    accountParams.syncPolicy = DefaultSyncPolicy
+    accountParams.lastSyncCompletions = []
 
-    const idString = `${email}${JSON.stringify(settings)}`
-    const id = crypto.createHash('sha256').update(idString, 'utf8').digest('hex')
-    return Account.findById(id).then((existing) => {
-      const account = existing || Account.build({
-        id,
-        name: name,
-        provider: provider,
-        emailAddress: email,
-        connectionSettings: settings,
-        syncPolicy: DefaultSyncPolicy,
-        lastSyncCompletions: [],
-      })
-
-      // always update with the latest credentials
-      account.setCredentials(credentials);
-
-      return account.save().then((saved) =>
-        AccountToken.create({accountId: saved.id}).then((token) => {
-          SyncProcessManager.addWorkerForAccount(saved);
-          return Promise.resolve({
-            account: saved,
-            token: token,
-          });
-        })
-      );
-    });
+    return Account.upsertWithCredentials(accountParams, credentials)
+    .then(({account}) => {
+      SyncProcessManager.addWorkerForAccount(account)
+    })
   });
 }
 
@@ -48,69 +19,7 @@ module.exports = (server) => {
   server.route({
     method: 'POST',
     path: '/auth',
-    config: AuthHelpers.authPostConfig(),
-    handler: (request, reply) => {
-      const dbStub = {};
-      const connectionChecks = [];
-      const {settings, email, provider, name} = request.payload;
-
-      let connectionSettings = null;
-      let connectionCredentials = null;
-
-      if (provider === 'imap') {
-        connectionSettings = _.pick(settings, [
-          'imap_host', 'imap_port',
-          'smtp_host', 'smtp_port',
-          'ssl_required',
-        ]);
-        connectionCredentials = _.pick(settings, [
-          'imap_username', 'imap_password',
-          'smtp_username', 'smtp_password',
-        ]);
-      }
-
-      if (provider === 'gmail') {
-        connectionSettings = {
-          imap_username: email,
-          imap_host: 'imap.gmail.com',
-          imap_port: 993,
-          smtp_username: email,
-          smtp_host: 'smtp.gmail.com',
-          smtp_port: 465,
-          ssl_required: true,
-        }
-        connectionCredentials = {
-          xoauth2: settings.xoauth2,
-        }
-      }
-
-      connectionChecks.push(IMAPConnection.connect({
-        settings: Object.assign({}, connectionSettings, connectionCredentials),
-        logger: request.logger,
-        db: dbStub,
-      }));
-
-      Promise.all(connectionChecks).then((conns) => {
-        for (const conn of conns) {
-          if (conn) { conn.end(); }
-        }
-        return buildAccountWith({
-          name: name,
-          email: email,
-          provider: provider,
-          settings: connectionSettings,
-          credentials: connectionCredentials,
-        })
-      })
-      .then(({account, token}) => {
-        const response = account.toJSON();
-        response.auth_token = token.value;
-        reply(Serialization.jsonStringify(response));
-      })
-      .catch((err) => {
-        const code = err instanceof IMAPErrors.IMAPAuthenticationError ? 401 : 400
-        reply({message: err.message, type: "api_error"}).code(code);
-      })
-    },
+    config: AuthHelpers.imapAuthRouteConfig(),
+    handler: AuthHelpers.imapAuthHandler(accountBuildFn),
   });
 }

From 4c53247df1a840ec41500c4d06063ff9901716e4 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 6 Dec 2016 12:31:00 -0800
Subject: [PATCH 469/800] [*] update lerna to 2.0.0-beta.30

---
 lerna.json   | 2 +-
 package.json | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lerna.json b/lerna.json
index 7a61c20d8..898e42b26 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,4 +1,4 @@
 {
-  "lerna": "2.0.0-beta.23",
+  "lerna": "2.0.0-beta.30",
   "version": "0.0.1"
 }
diff --git a/package.json b/package.json
index 803b53356..b9557314b 100644
--- a/package.json
+++ b/package.json
@@ -4,7 +4,7 @@
   "description": "The local sync engine for Nylas N1",
   "main": "",
   "dependencies": {
-    "lerna": "2.0.0-beta.23",
+    "lerna": "2.0.0-beta.30",
     "pm2": "^1.1.3"
   },
   "devDependencies": {
@@ -17,7 +17,9 @@
     "eslint-plugin-react": "6.7.1"
   },
   "scripts": {
-    "start": "pm2 start ./pm2-dev.yml --no-daemon",
+    "start": "pm2 stop all; pm2 delete all; pm2 start ./pm2-dev.yml --no-daemon",
+    "stop": "pm2 stop all; pm2 delete all",
+    "restart": "pm2 restart all",
     "postinstall": "lerna bootstrap"
   },
   "repository": {

From 8cbda7505a95fdd9501ecdb008ea132969222119 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 6 Dec 2016 14:53:24 -0800
Subject: [PATCH 470/800] [*] fix auth

---
 packages/isomorphic-core/src/auth-helpers.js     | 12 +++++++++---
 packages/isomorphic-core/src/models/account.js   | 12 +++++-------
 packages/local-sync/src/local-api/routes/auth.js |  3 ++-
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/packages/isomorphic-core/src/auth-helpers.js b/packages/isomorphic-core/src/auth-helpers.js
index 3204316d3..b2840e496 100644
--- a/packages/isomorphic-core/src/auth-helpers.js
+++ b/packages/isomorphic-core/src/auth-helpers.js
@@ -25,6 +25,8 @@ const exchangeSettings = Joi.object().keys({
   eas_server_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
 }).required();
 
+const AUTH_500_USER_MESSAGE = "Please contact support@nylas.com. An unforseen error has occurred."
+
 module.exports = {
   imapAuthRouteConfig() {
     return {
@@ -99,11 +101,15 @@ module.exports = {
       .then(({account, token}) => {
         const response = account.toJSON();
         response.account_token = token.value;
-        reply(JSON.stringify(response));
+        console.log(response)
+        return reply(JSON.stringify(response));
       })
       .catch((err) => {
-        const code = err instanceof IMAPErrors.IMAPAuthenticationError ? 401 : 400
-        reply({message: err.message, type: "api_error"}).code(code);
+        if (err instanceof IMAPErrors.IMAPAuthenticationError) {
+          return reply({message: err.message, type: "api_error"}).code(401);
+        }
+        request.logger.error(err)
+        return reply({message: AUTH_500_USER_MESSAGE, type: "api_error"}).code(500);
       })
     }
   },
diff --git a/packages/isomorphic-core/src/models/account.js b/packages/isomorphic-core/src/models/account.js
index 8577cfb89..1147046f3 100644
--- a/packages/isomorphic-core/src/models/account.js
+++ b/packages/isomorphic-core/src/models/account.js
@@ -3,8 +3,6 @@ const {buildJSONColumnOptions, buildJSONARRAYColumnOptions} = require('../databa
 
 const {DB_ENCRYPTION_ALGORITHM, DB_ENCRYPTION_PASSWORD} = process.env;
 
-const AccountToken = require('./account-token')
-
 module.exports = (sequelize, Sequelize) => {
   const Account = sequelize.define('account', {
     id: { type: Sequelize.STRING(65), primaryKey: true },
@@ -41,11 +39,11 @@ module.exports = (sequelize, Sequelize) => {
           // always update with the latest credentials
           account.setCredentials(credentials);
 
-          return account.save().then((saved) =>
-            AccountToken.create({accountId: saved.id}).then((token) =>
-              Promise.resolve({account: saved, token: token})
-            )
-          );
+          return account.save().then((saved) => {
+            return sequelize.models.accountToken.create({accountId: saved.id}).then((token) => {
+              return Promise.resolve({account: saved, token: token})
+            })
+          });
         });
       },
     },
diff --git a/packages/local-sync/src/local-api/routes/auth.js b/packages/local-sync/src/local-api/routes/auth.js
index 71c9e1750..e21097141 100644
--- a/packages/local-sync/src/local-api/routes/auth.js
+++ b/packages/local-sync/src/local-api/routes/auth.js
@@ -9,8 +9,9 @@ const accountBuildFn = (accountParams, credentials) => {
     accountParams.lastSyncCompletions = []
 
     return Account.upsertWithCredentials(accountParams, credentials)
-    .then(({account}) => {
+    .then(({account, token}) => {
       SyncProcessManager.addWorkerForAccount(account)
+      return {account, token}
     })
   });
 }

From 83ef47d049bb03665ac985eafc863cf4856d72aa Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 6 Dec 2016 15:28:49 -0800
Subject: [PATCH 471/800] [*] package.json updates from lerna

---
 packages/local-sync/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index 65ac6b351..0dc03c7d1 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -16,14 +16,14 @@
     "joi": "8.4.2",
     "mimelib": "0.2.19",
     "nodemailer": "2.5.0",
-    "quoted-printable": "^1.0.1",
+    "quoted-printable": "1.0.1",
     "request": "2.79.0",
     "rx": "4.1.0",
     "sequelize": "3.27.0",
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz",
     "striptags": "2.1.1",
     "underscore": "1.8.3",
-    "utf7": "^1.0.2",
+    "utf7": "1.0.2",
     "vision": "4.1.0"
   },
   "scripts": {

From 7e485228af1bdb710511b061220a8bd1bc664e7b Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 6 Dec 2016 16:02:58 -0800
Subject: [PATCH 472/800] [cloud-api] fix param names

---
 packages/isomorphic-core/src/models/account.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/isomorphic-core/src/models/account.js b/packages/isomorphic-core/src/models/account.js
index 1147046f3..c1175e888 100644
--- a/packages/isomorphic-core/src/models/account.js
+++ b/packages/isomorphic-core/src/models/account.js
@@ -31,7 +31,10 @@ module.exports = (sequelize, Sequelize) => {
         Account.hasMany(data.AccountToken, {as: 'tokens'})
       },
       upsertWithCredentials: (accountParams, credentials) => {
-        const idString = `${accountParams.email}${JSON.stringify(accountParams.settings)}`
+        if (!accountParams || !credentials || !accountParams.emailAddress) {
+          throw new Error("Need to pass accountParams and credentials to upsertWithCredentials")
+        }
+        const idString = `${accountParams.emailAddress}${JSON.stringify(accountParams.connectionSettings)}`;
         const id = crypto.createHash('sha256').update(idString, 'utf8').digest('hex')
         return Account.findById(id).then((existing) => {
           const account = existing || Account.build(Object.assign({id}, accountParams))

From 6ac46d10795d018092f828ac353f9f2fa6f374a0 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Tue, 6 Dec 2016 17:20:43 -0800
Subject: [PATCH 473/800] remove console log

---
 packages/isomorphic-core/src/auth-helpers.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/isomorphic-core/src/auth-helpers.js b/packages/isomorphic-core/src/auth-helpers.js
index b2840e496..d9f9b8e68 100644
--- a/packages/isomorphic-core/src/auth-helpers.js
+++ b/packages/isomorphic-core/src/auth-helpers.js
@@ -101,7 +101,6 @@ module.exports = {
       .then(({account, token}) => {
         const response = account.toJSON();
         response.account_token = token.value;
-        console.log(response)
         return reply(JSON.stringify(response));
       })
       .catch((err) => {

From c3bd3dc2975df869154ea7cdb9ddb000c0b27097 Mon Sep 17 00:00:00 2001
From: Ben Gotow 
Date: Tue, 6 Dec 2016 17:51:12 -0800
Subject: [PATCH 474/800] =?UTF-8?q?[local-sync]=20Don=E2=80=99t=20update?=
 =?UTF-8?q?=20contacts=20if=20name=20is=20the=20same?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../new-message-processor/extract-contacts.js | 29 ++++++++++++-------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/packages/local-sync/src/new-message-processor/extract-contacts.js b/packages/local-sync/src/new-message-processor/extract-contacts.js
index 4198bcb28..34b9c70ed 100644
--- a/packages/local-sync/src/new-message-processor/extract-contacts.js
+++ b/packages/local-sync/src/new-message-processor/extract-contacts.js
@@ -1,6 +1,6 @@
 const cryptography = require('crypto');
 
-function isContactVerified(contact) {
+function isContactMeaningful(contact) {
   // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
   const regex = new RegExp(/^(noreply|no-reply|donotreply|mailer|support|webmaster|news(letter)?@)/ig)
 
@@ -19,25 +19,34 @@ async function extractContacts({db, message}) {
     allContacts = allContacts.concat(message[field])
   })
 
-  const verifiedContacts = allContacts.filter(c => isContactVerified(c));
-  return db.sequelize.transaction(async (transaction) => {
-    for (const c of verifiedContacts) {
+  const meaningfulContacts = allContacts.filter(c => isContactMeaningful(c));
+
+  await db.sequelize.transaction(async (transaction) => {
+    const promises = [];
+
+    for (const c of meaningfulContacts) {
       const id = cryptography.createHash('sha256').update(c.email, 'utf8').digest('hex');
-      let contact = await db.Contact.findById(id);
+      const existing = await db.Contact.findById(id);
       const cdata = {
         name: c.name,
         email: c.email,
         accountId: message.accountId,
         id: id,
       };
-      
-      if (!contact) {
-        contact = await db.Contact.create(cdata)
+
+      if (!existing) {
+        promises.push(db.Contact.create(cdata, {transaction}));
       } else {
-        await contact.update(cdata);
+        const updateRequired = (cdata.name !== existing.name);
+        if (updateRequired) {
+          promises.push(existing.update(cdata, {transaction}));
+        }
       }
     }
-  }).thenReturn(message)
+    await Promise.all(promises);
+  })
+
+  return message;
 }
 
 module.exports = extractContacts

From a23c68092e5b77ea95da17941af578acfb3ca3e4 Mon Sep 17 00:00:00 2001
From: Christine Spang 
Date: Tue, 6 Dec 2016 11:19:39 -0800
Subject: [PATCH 475/800] [local-sync] Add specs for message parsing

Summary:
This commit also fixes snippets for HTML-only messages to strip out HTML
tags, and makes us preserve whitespace for plaintext emails by
displaying them in 
 tags, and makes us log
messages that fail to parse at all to a tempdir.

The only issue I found with using 
 tags for plaintext email was
that some lines may trigger scrolling, so there is an associated commit
(D3484) that changes the CSS for 
 to wrap
lines.

In the future, we can add regression tests to this test suite whenever
we fix parsing bugs.

Test Plan: unit tests included

Reviewers: bengotow

Reviewed By: bengotow

Differential Revision: https://phab.nylas.com/D3483
---
 packages/local-sync/package.json                   |  3 ++-
 .../MessageFactory/parseFromImap/base-case.json    |  5 -----
 .../parseFromImap/crypto-gram-ascii-plaintext.json |  1 +
 .../parseFromImap/eff-plaintext-no-mime.json       |  1 +
 .../hacker-newsletter-multipart-alternative.json   |  1 +
 .../parseFromImap/mileageplus-mime-html-only.json  |  1 +
 .../node-streamtest-windows-1252.json              |  1 +
 .../spam-mime-html-base64-encoded.json             |  1 +
 ...imm-multipart-alternative-quoted-printable.json |  1 +
 packages/local-sync/spec/message-factory-spec.js   |  9 ++++++++-
 .../imap/fetch-messages-in-folder.js               |  8 ++++++++
 packages/local-sync/src/shared/message-factory.js  | 14 +++++++++++---
 12 files changed, 36 insertions(+), 10 deletions(-)
 delete mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/crypto-gram-ascii-plaintext.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/eff-plaintext-no-mime.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/hacker-newsletter-multipart-alternative.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/mileageplus-mime-html-only.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/node-streamtest-windows-1252.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/spam-mime-html-base64-encoded.json
 create mode 100644 packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/theskimm-multipart-alternative-quoted-printable.json

diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index 0dc03c7d1..a24ccce17 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -23,7 +23,8 @@
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz",
     "striptags": "2.1.1",
     "underscore": "1.8.3",
-    "utf7": "1.0.2",
+    "utf7": "^1.0.2",
+    "striptags": "2.1.1",
     "vision": "4.1.0"
   },
   "scripts": {
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json
deleted file mode 100644
index 5fc82c150..000000000
--- a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/base-case.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "imapMessage": {},
-  "desiredParts": {},
-  "result": {}
-}
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/crypto-gram-ascii-plaintext.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/crypto-gram-ascii-plaintext.json
new file mode 100644
index 000000000..2ce6b4d3b
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/crypto-gram-ascii-plaintext.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"partID":"1","type":"text","subtype":"plain","params":{"charset":"us-ascii","format":"flowed"},"id":null,"description":null,"encoding":"7BIT","size":42161,"lines":820,"md5":null,"disposition":null,"language":null}],"date":"2016-11-15T07:50:26.000Z","flags":["\\Seen"],"uid":345982,"modseq":"8120006","x-gm-labels":["\\Inbox"],"x-gm-msgid":"1551049662245032910","x-gm-thrid":"1551049662245032910"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.31.185.141 with SMTP id j135csp15122vkf; Mon, 14 Nov 2016\r\n 23:50:26 -0800 (PST)\r\nX-Received: by 10.37.220.66 with SMTP id y63mr6697075ybe.190.1479196226438;\r\n Mon, 14 Nov 2016 23:50:26 -0800 (PST)\r\nReturn-Path: \r\nReceived: from schneier.modwest.com (schneier.modwest.com. [204.11.247.92]) by\r\n mx.google.com with ESMTPS id i126si6507480ybb.7.2016.11.14.23.50.26 for\r\n  (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256\r\n bits=128/128); Mon, 14 Nov 2016 23:50:26 -0800 (PST)\r\nReceived-SPF: pass (google.com: domain of\r\n crypto-gram-bounces@lists.schneier.com designates 204.11.247.92 as permitted\r\n sender) client-ip=204.11.247.92;\r\nAuthentication-Results: mx.google.com; spf=pass (google.com: domain of\r\n crypto-gram-bounces@lists.schneier.com designates 204.11.247.92 as permitted\r\n sender) smtp.mailfrom=crypto-gram-bounces@lists.schneier.com\r\nReceived: from schneier.modwest.com (localhost [127.0.0.1]) by\r\n schneier.modwest.com (Postfix) with ESMTP id A57D33A66E for\r\n ; Tue, 15 Nov 2016 00:48:53 -0700 (MST)\r\nX-Original-To: crypto-gram@lists.schneier.com\r\nDelivered-To: crypto-gram@lists.schneier.com\r\nReceived: from webmail.schneier.com (localhost [127.0.0.1]) by\r\n schneier.modwest.com (Postfix) with ESMTPA id 735B038F18; Tue, 15 Nov 2016\r\n 00:27:10 -0700 (MST)\r\nMIME-Version: 1.0\r\nDate: Tue, 15 Nov 2016 01:27:10 -0600\r\nFrom: Bruce Schneier \r\nSubject: CRYPTO-GRAM, November 15, 2016\r\nMessage-ID: <76bcad7045e1f498eb00e27fc969ee53@schneier.com>\r\nX-Sender: schneier@schneier.com\r\nUser-Agent: Roundcube Webmail/0.9.5\r\nX-Mailman-Approved-At: Tue, 15 Nov 2016 00:45:13 -0700\r\nX-BeenThere: crypto-gram@lists.schneier.com\r\nX-Mailman-Version: 2.1.15\r\nPrecedence: list\r\nCc: Crypto-Gram Mailing List \r\nList-Id: Crypto-Gram Mailing List \r\nList-Unsubscribe: ,\r\n \r\nList-Post: \r\nList-Help: \r\nList-Subscribe: ,\r\n \r\nContent-Transfer-Encoding: 7bit\r\nContent-Type: text/plain; charset=\"us-ascii\"; Format=\"flowed\"\r\nTo: christine@spang.cc\r\nErrors-To: crypto-gram-bounces@lists.schneier.com\r\nSender: \"Crypto-Gram\" \r\n\r\n","parts":{"1":"\r\n             CRYPTO-GRAM\r\n\r\n          November 15, 2016\r\n\r\n          by Bruce Schneier\r\n    CTO, Resilient, an IBM Company\r\n        schneier@schneier.com\r\n       https://www.schneier.com\r\n\r\n\r\nA free monthly newsletter providing summaries, analyses, insights, and \r\ncommentaries on security: computer and otherwise.\r\n\r\nFor back issues, or to subscribe, visit \r\n.\r\n\r\nYou can read this issue on the web at \r\n. These \r\nsame essays and news items appear in the \"Schneier on Security\" blog at \r\n, along with a lively and intelligent \r\ncomment section. An RSS feed is available.\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\nIn this issue:\r\n      Election Security\r\n      News\r\n      Lessons From the Dyn DDoS Attack\r\n      Regulation of the Internet of Things\r\n      Schneier News\r\n      Virtual Kidnapping\r\n      Intelligence Oversight and How It Can Fail\r\n      Whistleblower Investigative Report on NSA Suite B Cryptography\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Election Security\r\n\r\n\r\n\r\nIt's over. The voting went smoothly. As of the time of writing, there \r\nare no serious fraud allegations, nor credible evidence that anyone \r\ntampered with voting rolls or voting machines. And most important, the \r\nresults are not in doubt.\r\n\r\nWhile we may breathe a collective sigh of relief about that, we can't \r\nignore the issue until the next election. The risks remain.\r\n\r\nAs computer security experts have been saying for years, our newly \r\ncomputerized voting systems are vulnerable to attack by both individual \r\nhackers and government-sponsored cyberwarriors. It is only a matter of \r\ntime before such an attack happens.\r\n\r\nElectronic voting machines can be hacked, and those machines that do not \r\ninclude a paper ballot that can verify each voter's choice can be hacked \r\nundetectably. Voting rolls are also vulnerable; they are all \r\ncomputerized databases whose entries can be deleted or changed to sow \r\nchaos on Election Day.\r\n\r\nThe largely ad hoc system in states for collecting and tabulating \r\nindividual voting results is vulnerable as well. While the difference \r\nbetween theoretical if demonstrable vulnerabilities and an actual attack \r\non Election Day is considerable, we got lucky this year. Not just \r\npresidential elections are at risk, but state and local elections, too.\r\n\r\nTo be very clear, this is not about voter fraud. The risks of ineligible \r\npeople voting, or people voting twice, have been repeatedly shown to be \r\nvirtually nonexistent, and \"solutions\" to this problem are largely \r\nvoter-suppression measures. Election fraud, however, is both far more \r\nfeasible and much more worrisome.\r\n\r\nHere's my worry. On the day after an election, someone claims that a \r\nresult was hacked. Maybe one of the candidates points to a wide \r\ndiscrepancy between the most recent polls and the actual results. Maybe \r\nan anonymous person announces that he hacked a particular brand of \r\nvoting machine, describing in detail how. Or maybe it's a system failure \r\nduring Election Day: voting machines recording significantly fewer votes \r\nthan there were voters, or zero votes for one candidate or another. \r\n(These are not theoretical occurrences; they have both happened in the \r\nUnited States before, though because of error, not malice.)\r\n\r\nWe have no procedures for how to proceed if any of these things happen. \r\nThere's no manual, no national panel of experts, no regulatory body to \r\nsteer us through this crisis. How do we figure out if someone hacked the \r\nvote? Can we recover the true votes, or are they lost? What do we do \r\nthen?\r\n\r\nFirst, we need to do more to secure our elections system. We should \r\ndeclare our voting systems to be critical national infrastructure. This \r\nis largely symbolic, but it demonstrates a commitment to secure \r\nelections and makes funding and other resources available to states.\r\n\r\nWe need national security standards for voting machines, and funding for \r\nstates to procure machines that comply with those standards. \r\nVoting-security experts can deal with the technical details, but such \r\nmachines must include a paper ballot that provides a record verifiable \r\nby voters. The simplest and most reliable way to do that is already \r\npracticed in 37 states: optical-scan paper ballots, marked by the \r\nvoters, counted by computer but recountable by hand. And we need a \r\nsystem of pre-election and postelection security audits to increase \r\nconfidence in the system.\r\n\r\nSecond, election tampering, either by a foreign power or by a domestic \r\nactor, is inevitable, so we need detailed procedures to follow -- both \r\ntechnical procedures to figure out what happened, and legal procedures \r\nto figure out what to do -- that will efficiently get us to a fair and \r\nequitable election resolution. There should be a board of independent \r\ncomputer-security experts to unravel what happened, and a board of \r\nindependent election officials, either at the Federal Election \r\nCommission or elsewhere, empowered to determine and put in place an \r\nappropriate response.\r\n\r\nIn the absence of such impartial measures, people rush to defend their \r\ncandidate and their party. Florida in 2000 was a perfect example. What \r\ncould have been a purely technical issue of determining the intent of \r\nevery voter became a battle for who would win the presidency. The \r\ndebates about hanging chads and spoiled ballots and how broad the \r\nrecount should be were contested by people angling for a particular \r\noutcome. In the same way, after a hacked election, partisan politics \r\nwill place tremendous pressure on officials to make decisions that \r\noverride fairness and accuracy.\r\n\r\nThat is why we need to agree on policies to deal with future election \r\nfraud. We need procedures to evaluate claims of voting-machine hacking. \r\nWe need a fair and robust vote-auditing process. And we need all of this \r\nin place before an election is hacked and battle lines are drawn.\r\n\r\nIn response to Florida, the Help America Vote Act of 2002 required each \r\nstate to publish its own guidelines on what constitutes a vote. Some \r\nstates -- Indiana, in particular -- set up a \"war room\" of public and \r\nprivate cybersecurity experts ready to help if anything did occur. While \r\nthe Department of Homeland Security is assisting some states with \r\nelection security, and the F.B.I. and the Justice Department made some \r\npreparations this year, the approach is too piecemeal.\r\n\r\nElections serve two purposes. First, and most obvious, they are how we \r\nchoose a winner. But second, and equally important, they convince the \r\nloser -- and all the supporters -- that he or she lost. To achieve the \r\nfirst purpose, the voting system must be fair and accurate. To achieve \r\nthe second one, it must be *shown* to be fair and accurate.\r\n\r\nWe need to have these conversations before something happens, when \r\neveryone can be calm and rational about the issues. The integrity of our \r\nelections is at stake, which means our democracy is at stake.\r\n\r\nThis essay previously appeared in the New York Times.\r\nhttp://www.nytimes.com/2016/11/09/opinion/american-elections-will-be-hacked.html\r\n\r\nElection-machine vulnerabilities:\r\nhttps://www.washingtonpost.com/posteverything/wp/2016/07/27/by-november-russian-hackers-could-target-voting-machines/\r\n\r\nElections are hard to rig:\r\nhttps://www.washingtonpost.com/news/the-fix/wp/2016/08/03/one-reason-to-doubt-the-presidential-election-will-be-rigged-its-a-lot-harder-than-it-seems/\r\n\r\nVoting systems as critical infrastructure:\r\nhttps://papers.ssrn.com/sol3/papers.cfm?abstract_id=2852461\r\n\r\nVoting machine security:\r\nhttps://www.verifiedvoting.org/\r\nhttp://votingmachines.procon.org/view.answers.php?questionID=000291\r\nhttp://votingmachines.procon.org/view.answers.php?questionID=000291\r\n\r\nElection-defense preparations for 2016:\r\nhttp://www.usatoday.com/story/tech/news/2016/11/05/election-2016-cyber-hack-issues-homeland-security-indiana-pennsylvania-election-protection-verified-voter/93262960/\r\nhttp://www.nbcnews.com/storyline/2016-election-day/all-hands-deck-protect-election-hack-say-officials-n679271\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      News\r\n\r\n\r\n\r\nLance Spitzner looks at the safety features of a power saw and tries to \r\napply them to Internet security.\r\nhttps://securingthehuman.sans.org/blog/2016/10/18/what-iot-and-security-needs-to-learn-from-the-dewalt-mitre-saw\r\n\r\nResearchers discover a clever attack that bypasses the address space \r\nlayout randomization (ALSR) on Intel's CPUs.\r\nhttp://arstechnica.com/security/2016/10/flaw-in-intel-chips-could-make-malware-attacks-more-potent/\r\nhttp://www.cs.ucr.edu/~nael/pubs/micro16.pdf\r\n\r\nIn an interviw in Wired, President Obama talks about AI risk, \r\ncybersecurity, and more.\r\nhttps://www.wired.com/2016/10/president-obama-mit-joi-ito-interview/\r\n\r\nPrivacy makes workers more productive. Interesting research.\r\nhttps://www.psychologytoday.com/blog/the-outsourced-mind/201604/want-people-behave-better-give-them-more-privacy\r\n\r\nNews about the DDOS attacks against Dyn.\r\nhttps://motherboard.vice.com/read/twitter-reddit-spotify-were-collateral-damage-in-major-internet-attack\r\nhttps://krebsonsecurity.com/2016/10/ddos-on-dyn-impacts-twitter-spotify-reddit/\r\nhttps://motherboard.vice.com/read/blame-the-internet-of-things-for-destroying-the-internet-today\r\n\r\nJosephine Wolff examines different Internet governance stakeholders and \r\nhow they frame security debates.\r\nhttps://policyreview.info/articles/analysis/what-we-talk-about-when-we-talk-about-cybersecurity-security-internet-governance\r\n\r\nThe UK is admitting \"offensive cyber\" operations against ISIS/Daesh. I \r\nthink this might be the first time it has been openly acknowledged.\r\nhttps://www.theguardian.com/politics/blog/live/2016/oct/20/philip-green-knighthood-commons-set-to-debate-stripping-philip-green-of-his-knighthood-politics-live\r\n\r\nIt's not hard to imagine the criminal possibilities of automation, \r\nautonomy, and artificial intelligence. But the imaginings are becoming \r\nmainstream -- and the future isn't too far off.\r\nhttp://www.nytimes.com/2016/10/24/technology/artificial-intelligence-evolves-with-its-criminal-potential.html\r\n\r\nAlong similar lines, computers are able to predict court verdicts. My \r\nguess is that the real use here isn't to predict actual court verdicts, \r\nbut for well-paid defense teams to test various defensive tactics.\r\nhttp://www.telegraph.co.uk/science/2016/10/23/artifically-intelligent-judge-developed-which-can-predict-court/\r\n\r\nGood long article on the 2015 attack against the US Office of Personnel \r\nManagement.\r\nhttps://www.wired.com/2016/10/inside-cyberattack-shocked-us-government/\r\n\r\nHow Powell's and Podesta's e-mail accounts were hacked. It was phishing.\r\nhttps://motherboard.vice.com/read/how-hackers-broke-into-john-podesta-and-colin-powells-gmail-accounts\r\n\r\nA year and a half ago, I wrote about hardware bit-flipping attacks, \r\nwhich were then largely theoretical. Now, they can be used to root \r\nAndroid phones.\r\nhttp://arstechnica.com/security/2016/10/using-rowhammer-bitflips-to-root-android-phones-is-now-a-thing/\r\nhttps://vvdveen.com/publications/drammer.pdf\r\nhttps://www.vusec.net/projects/drammer/\r\n\r\nEavesdropping on typing while connected over VoIP.\r\nhttps://arxiv.org/pdf/1609.09359.pdf\r\nhttps://news.uci.edu/research/typing-while-skyping-could-compromise-privacy/\r\n\r\nAn impressive Chinese device that automatically reads marked cards in \r\norder to cheat at poker and other card games.\r\nhttps://www.elie.net/blog/security/fuller-house-exposing-high-end-poker-cheating-devices\r\n\r\nA useful guide on how to avoid kidnapping children on Halloween.\r\nhttp://reductress.com/post/how-to-not-kidnap-any-kids-on-halloween-not-even-one/\r\n\r\nA card game based on the iterated prisoner's dilemma.\r\nhttps://opinionatedgamers.com/2016/10/26/h-m-s-dolores-game-review-by-chris-wray/\r\n\r\nThere's another leak of NSA hacking tools and data from the Shadow \r\nBrokers. This one includes a list of hacked sites. The data is old, but \r\nyou can see if you've been hacked.\r\nhttp://arstechnica.co.uk/security/2016/10/new-leak-may-show-if-you-were-hacked-by-the-nsa/\r\nHonestly, I am surprised by this release. I thought that the original \r\nShadow Brokers dump was everything. Now that we know they held things \r\nback, there could easily be more releases.\r\nhttp://www.networkworld.com/article/3137065/security/shadow-brokers-leak-list-of-nsa-targets-and-compromised-servers.html\r\nNote that the Hague-based Organization for the Prohibition of Chemical \r\nWeapons is on the list, hacked in 2000.\r\nhttps://boingboing.net/2016/11/06/in-2000-the-nsa-hacked-the-ha.html\r\n\r\nFree cybersecurity MOOC from F-Secure and the University of Finland.\r\nhttp://mooc.fi/courses/2016/cybersecurity/\r\n\r\nResearchers have trained a neural network to encrypt its communications. \r\nThis story is more about AI and neural networks than it is about \r\ncryptography. The algorithm isn't any good, but is a perfect example of \r\nwhat I've heard called \"Schneier's Law\": Anyone can design a cipher that \r\nthey themselves cannot break.\r\nhttps://www.newscientist.com/article/2110522-googles-neural-networks-invent-their-own-encryption/\r\nhttp://arstechnica.com/information-technology/2016/10/google-ai-neural-network-cryptography/\r\nhttps://www.engadget.com/2016/10/28/google-ai-created-its-own-form-of-encryption/\r\nhttps://arxiv.org/pdf/1610.06918v1.pdf\r\nSchneier's Law:\r\nhttps://www.schneier.com/blog/archives/2011/04/schneiers_law.html\r\n\r\nGoogle now links anonymous browser tracking with identifiable tracking. \r\nThe article also explains how to opt out.\r\nhttps://www.propublica.org/article/google-has-quietly-dropped-ban-on-personally-identifiable-web-tracking\r\n\r\nNew Atlas has a great three-part feature on the history of hacking as \r\nportrayed in films, including video clips. The 1980s. The 1990s. The \r\n2000s.\r\nhttp://newatlas.com/history-hollywood-hacking-1980s/45482/\r\nhttp://newatlas.com/hollywood-hacking-movies-1990s/45623/\r\nhttp://newatlas.com/hollywood-hacking-2000s/45965\r\n\r\nFor years, the DMCA has been used to stifle legitimate research into the \r\nsecurity of embedded systems. Finally, the research exemption to the \r\nDMCA is in effect (for two years, but we can hope it'll be extended \r\nforever).\r\nhttps://www.wired.com/2016/10/hacking-car-pacemaker-toaster-just-became-legal/\r\nhttps://www.eff.org/deeplinks/2016/10/why-did-we-have-wait-year-fix-our-cars\r\n\r\nFirefox is removing the battery status API, citing privacy concerns.\r\nhttps://www.fxsitecompat.com/en-CA/docs/2016/battery-status-api-has-been-removed/\r\nhttps://eprint.iacr.org/2015/616.pdf\r\nW3C is updating the spec.\r\nhttps://www.w3.org/TR/battery-status/#acknowledgements\r\nHere's a battery tracker found in the wild.\r\nhttp://randomwalker.info/publications/OpenWPM_1_million_site_tracking_measurement.pdf\r\n\r\nElection-day humor from 2004, but still relevent.\r\nhttp://www.ganssle.com/tem/tem316.html#article2\r\n\r\nA self-propagating smart light bulb worm.\r\nhttp://iotworm.eyalro.net/\r\nhttps://boingboing.net/2016/11/09/a-lightbulb-worm-could-take-ov.html\r\nhttps://tech.slashdot.org/story/16/11/09/0041201/researchers-hack-philips-hue-smart-bulbs-using-a-drone\r\nThis is exactly the sort of Internet-of-Things attack that has me \r\nworried.\r\n\r\nAd networks are surreptitiously using ultrasonic communications to jump \r\nfrom device to device. It should come as no surprise that this \r\ncommunications channel can be used to hack devices as well.\r\nhttps://www.newscientist.com/article/2110762-your-homes-online-gadgets-could-be-hacked-by-ultrasound/\r\nhttps://www.schneier.com/blog/archives/2015/11/ads_surreptitio.html\r\n\r\nThis is some interesting research. You can fool facial recognition \r\nsystems by wearing glasses printed with elements of other peoples' \r\nfaces.\r\nhttps://www.cs.cmu.edu/~sbhagava/papers/face-rec-ccs16.pdf\r\nhttp://qz.com/823820/carnegie-mellon-made-a-special-pair-of-glasses-that-lets-you-steal-a-digital-identity/\r\nhttps://boingboing.net/2016/11/02/researchers-trick-facial-recog.html\r\n\r\nInteresting research: \"Using Artificial Intelligence to Identify State \r\nSecrets,\" https://arxiv.org/abs/1611.00356\r\n\r\nThere's a Kickstarter for a sticker that you can stick on a glove and \r\nthen register with a biometric access system like an iPhone. It's an \r\ninteresting security trade-off: swapping something you are (the \r\nbiometric) with something you have (the glove).\r\nhttps://www.kickstarter.com/projects/nanotips/taps-touchscreen-sticker-w-touch-id-ships-before-x?token=5b586aa6\r\nhttps://gizmodo.com/these-fake-fingerprint-stickers-let-you-access-a-protec-1788710313\r\n\r\nJulian Oliver has designed and built a cellular eavesdropping device \r\nthat's disguised as an old HP printer. It's more of a conceptual art \r\npiece than an actual piece of eavesdropping equipment, but it still \r\nmakes the point.\r\nhttps://julianoliver.com/output/stealth-cell-tower\r\nhttps://www.wired.com/2016/11/evil-office-printer-hijacks-cellphone-connection/\r\nhttps://boingboing.net/2016/11/03/a-fake-hp-printer-thats-actu.html\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Lessons From the Dyn DDoS Attack\r\n\r\n\r\n\r\nA week ago Friday, someone took down numerous popular websites in a \r\nmassive distributed denial-of-service (DDoS) attack against the domain \r\nname provider Dyn. DDoS attacks are neither new nor sophisticated. The \r\nattacker sends a massive amount of traffic, causing the victim's system \r\nto slow to a crawl and eventually crash. There are more or less clever \r\nvariants, but basically, it's a datapipe-size battle between attacker \r\nand victim. If the defender has a larger capacity to receive and process \r\ndata, he or she will win. If the attacker can throw more data than the \r\nvictim can process, he or she will win.\r\n\r\nThe attacker can build a giant data cannon, but that's expensive. It is \r\nmuch smarter to recruit millions of innocent computers on the internet. \r\nThis is the \"distributed\" part of the DDoS attack, and pretty much how \r\nit's worked for decades. Cybercriminals infect innocent computers around \r\nthe internet and recruit them into a botnet. They then target that \r\nbotnet against a single victim.\r\n\r\nYou can imagine how it might work in the real world. If I can trick tens \r\nof thousands of others to order pizzas to be delivered to your house at \r\nthe same time, I can clog up your street and prevent any legitimate \r\ntraffic from getting through. If I can trick many millions, I might be \r\nable to crush your house from the weight. That's a DDoS attack -- it's \r\nsimple brute force.\r\n\r\nAs you'd expect, DDoSers have various motives. The attacks started out \r\nas a way to show off, then quickly transitioned to a method of \r\nintimidation -- or a way of just getting back at someone you didn't \r\nlike. More recently, they've become vehicles of protest. In 2013, the \r\nhacker group Anonymous petitioned the White House to recognize DDoS \r\nattacks as a legitimate form of protest. Criminals have used these \r\nattacks as a means of extortion, although one group found that just the \r\nfear of attack was enough. Military agencies are also thinking about \r\nDDoS as a tool in their cyberwar arsenals. A 2007 DDoS attack against \r\nEstonia was blamed on Russia and widely called an act of cyberwar.\r\n\r\nThe DDoS attack against Dyn two weeks ago was nothing new, but it \r\nillustrated several important trends in computer security.\r\n\r\nThese attack techniques are broadly available. Fully capable DDoS attack \r\ntools are available for free download. Criminal groups offer DDoS \r\nservices for hire. The particular attack technique used against Dyn was \r\nfirst used a month earlier. It's called Mirai, and since the source code \r\nwas released four weeks ago, over a dozen botnets have incorporated the \r\ncode.\r\n\r\nThe Dyn attacks were probably not originated by a government. The \r\nperpetrators were most likely hackers mad at Dyn for helping Brian Krebs \r\nidentify -- and the FBI arrest -- two Israeli hackers who were running a \r\nDDoS-for-hire ring. Recently I have written about probing DDoS attacks \r\nagainst internet infrastructure companies that appear to be perpetrated \r\nby a nation-state. But, honestly, we don't know for sure.\r\n\r\nThis is important. Software spreads capabilities. The smartest attacker \r\nneeds to figure out the attack and write the software. After that, \r\nanyone can use it. There's not even much of a difference between \r\ngovernment and criminal attacks. In December 2014, there was a \r\nlegitimate debate in the security community as to whether the massive \r\nattack against Sony had been perpetrated by a nation-state with a $20 \r\nbillion military budget or a couple of guys in a basement somewhere. The \r\ninternet is the only place where we can't tell the difference. Everyone \r\nuses the same tools, the same techniques and the same tactics.\r\n\r\nThese attacks are getting larger. The Dyn DDoS attack set a record at \r\n1.2 Tbps. The previous record holder was the attack against \r\ncybersecurity journalist Brian Krebs a month prior at 620 Gbps. This is \r\nmuch larger than required to knock the typical website offline. A year \r\nago, it was unheard of. Now it occurs regularly.\r\n\r\nThe botnets attacking Dyn and Brian Krebs consisted largely of unsecure \r\nInternet of Things (IoT) devices -- webcams, digital video recorders, \r\nrouters and so on. This isn't new, either. We've already seen \r\ninternet-enabled refrigerators and TVs used in DDoS botnets. But again, \r\nthe scale is bigger now. In 2014, the news was hundreds of thousands of \r\nIoT devices -- the Dyn attack used millions. Analysts expect the IoT to \r\nincrease the number of things on the internet by a factor of 10 or more. \r\nExpect these attacks to similarly increase.\r\n\r\nThe problem is that these IoT devices are unsecure and likely to remain \r\nthat way. The economics of internet security don't trickle down to the \r\nIoT. Commenting on the Krebs attack last month, I wrote:\r\n\r\n     The market can't fix this because neither the buyer nor the\r\n     seller cares. Think of all the CCTV cameras and DVRs used in\r\n     the attack against Brian Krebs. The owners of those devices\r\n     don't care. Their devices were cheap to buy, they still work,\r\n     and they don't even know Brian. The sellers of those devices\r\n     don't care: They're now selling newer and better models, and\r\n     the original buyers only cared about price and features. There\r\n     is no market solution because the insecurity is what economists\r\n     call an externality: It's an effect of the purchasing decision\r\n     that affects other people. Think of it kind of like invisible\r\n     pollution.\r\n\r\nTo be fair, one company that made some of the unsecure things used in \r\nthese attacks recalled its unsecure webcams. But this is more of a \r\npublicity stunt than anything else. I would be surprised if the company \r\ngot many devices back. We already know that the reputational damage from \r\nhaving your unsecure software made public isn't large and doesn't last. \r\nAt this point, the market still largely rewards sacrificing security in \r\nfavor of price and time-to-market.\r\n\r\nDDoS prevention works best deep in the network, where the pipes are the \r\nlargest and the capability to identify and block the attacks is the most \r\nevident. But the backbone providers have no incentive to do this. They \r\ndon't feel the pain when the attacks occur and they have no way of \r\nbilling for the service when they provide it. So they let the attacks \r\nthrough and force the victims to defend themselves. In many ways, this \r\nis similar to the spam problem. It, too, is best dealt with in the \r\nbackbone, but similar economics dump the problem onto the endpoints.\r\n\r\nWe're unlikely to get any regulation forcing backbone companies to clean \r\nup either DDoS attacks or spam, just as we are unlikely to get any \r\nregulations forcing IoT manufacturers to make their systems secure. This \r\nis me again:\r\n\r\n     What this all means is that the IoT will remain insecure unless\r\n     government steps in and fixes the problem. When we have market\r\n     failures, government is the only solution. The government could\r\n     impose security regulations on IoT manufacturers, forcing them\r\n     to make their devices secure even though their customers don't\r\n     care. They could impose liabilities on manufacturers, allowing\r\n     people like Brian Krebs to sue them. Any of these would raise\r\n     the cost of insecurity and give companies incentives to spend\r\n     money making their devices secure.\r\n\r\nThat leaves the victims to pay. This is where we are in much of computer \r\nsecurity. Because the hardware, software and networks we use are so \r\nunsecure, we have to pay an entire industry to provide after-the-fact \r\nsecurity.\r\n\r\nThere are solutions you can buy. Many companies offer DDoS protection, \r\nalthough they're generally calibrated to the older, smaller attacks. We \r\ncan safely assume that they'll up their offerings, although the cost \r\nmight be prohibitive for many users. Understand your risks. Buy \r\nmitigation if you need it, but understand its limitations. Know the \r\nattacks are possible and will succeed if large enough. And the attacks \r\nare getting larger all the time. Prepare for that.\r\n\r\nThis essay previously appeared on the SecurityIntelligence website.\r\nhttps://securityintelligence.com/lessons-from-the-dyn-ddos-attack/\r\n\r\nhttps://securityintelligence.com/news/multi-phased-ddos-attack-causes-hours-long-outages/\r\nhttp://arstechnica.com/information-technology/2016/10/inside-the-machine-uprising-how-cameras-dvrs-took-down-parts-of-the-internet/\r\nhttps://www.theguardian.com/technology/2016/oct/26/ddos-attack-dyn-mirai-botnet\r\nhttp://searchsecurity.techtarget.com/news/450401962/Details-emerging-on-Dyn-DNS-DDoS-attack-Mirai-IoT-botnet\r\nhttp://hub.dyn.com/static/hub.dyn.com/dyn-blog/dyn-statement-on-10-21-2016-ddos-attack.html\r\n\r\nDDoS petition:\r\nhttp://www.huffingtonpost.com/2013/01/12/anonymous-ddos-petition-white-house_n_2463009.html\r\n\r\nDDoS extortion:\r\nhttps://securityintelligence.com/ddos-extortion-easy-and-lucrative/\r\nhttp://www.computerworld.com/article/3061813/security/empty-ddos-threats-deliver-100k-to-extortion-group.html\r\n\r\nDDoS against Estonia:\r\nhttp://www.iar-gwu.org/node/65\r\n\r\nDDoS for hire:\r\nhttp://www.forbes.com/sites/thomasbrewster/2016/10/23/massive-ddos-iot-botnet-for-hire-twitter-dyn-amazon/#11f82518c915\r\n\r\nMirai:\r\nhttps://www.arbornetworks.com/blog/asert/mirai-iot-botnet-description-ddos-attack-mitigation/\r\nhttps://krebsonsecurity.com/2016/10/source-code-for-iot-botnet-mirai-released/\r\nhttps://threatpost.com/mirai-bots-more-than-double-since-source-code-release/121368/\r\n\r\nKrebs:\r\nhttp://krebsonsecurity.com/2016/09/israeli-online-attack-service-vdos-earned-600000-in-two-years/\r\nhttp://www.theverge.com/2016/9/11/12878692/vdos-israeli-teens-ddos-cyberattack-service-arrested\r\nhttps://krebsonsecurity.com/2016/09/krebsonsecurity-hit-with-record-ddos/\r\nhttp://www.businessinsider.com/akamai-brian-krebs-ddos-attack-2016-9\r\n\r\nNation-state DDoS Attacks:\r\nhttps://www.schneier.com/blog/archives/2016/09/someone_is_lear.html\r\n\r\nNorth Korea and Sony:\r\nhttps://www.theatlantic.com/international/archive/2014/12/did-north-korea-really-attack-sony/383973/\r\n\r\nInternet of Things (IoT) security:\r\nhttps://securityintelligence.com/will-internet-things-leveraged-ruin-companys-day-understanding-iot-security/\r\nhttps://thehackernews.com/2014/01/100000-refrigerators-and-other-home.html\r\n\r\nEver larger DDoS Attacks:\r\nhttp://www.ibtimes.co.uk/biggest-ever-terabit-scale-ddos-attack-yet-could-be-horizon-experts-warn-1588364\r\n\r\nMy previous essay on this:\r\nhttps://www.schneier.com/essays/archives/2016/10/we_need_to_save_the_.html\r\n\r\nrecalled:\r\nhttp://www.zdnet.com/article/chinese-tech-giant-recalls-webcams-used-in-dyn-cyberattack/\r\n\r\nidentify and block the attacks:\r\nhttp://www.ibm.com/security/threat-protection/\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Regulation of the Internet of Things\r\n\r\n\r\n\r\nLate last month, popular websites like Twitter, Pinterest, Reddit and \r\nPayPal went down for most of a day. The distributed denial-of-service \r\nattack that caused the outages, and the vulnerabilities that made the \r\nattack possible, was as much a failure of market and policy as it was of \r\ntechnology. If we want to secure our increasingly computerized and \r\nconnected world, we need more government involvement in the security of \r\nthe \"Internet of Things\" and increased regulation of what are now \r\ncritical and life-threatening technologies. It's no longer a question of \r\nif, it's a question of when.\r\n\r\nFirst, the facts. Those websites went down because their domain name \r\nprovider -- a company named Dyn -- was forced offline. We don't know who \r\nperpetrated that attack, but it could have easily been a lone hacker. \r\nWhoever it was launched a distributed denial-of-service attack against \r\nDyn by exploiting a vulnerability in large numbers -- possibly millions \r\n-- of Internet-of-Things devices like webcams and digital video \r\nrecorders, then recruiting them all into a single botnet. The botnet \r\nbombarded Dyn with traffic, so much that it went down. And when it went \r\ndown, so did dozens of websites.\r\n\r\nYour security on the Internet depends on the security of millions of \r\nInternet-enabled devices, designed and sold by companies you've never \r\nheard of to consumers who don't care about your security.\r\n\r\nThe technical reason these devices are insecure is complicated, but \r\nthere is a market failure at work. The Internet of Things is bringing \r\ncomputerization and connectivity to many tens of millions of devices \r\nworldwide. These devices will affect every aspect of our lives, because \r\nthey're things like cars, home appliances, thermostats, lightbulbs, \r\nfitness trackers, medical devices, smart streetlights and sidewalk \r\nsquares. Many of these devices are low-cost, designed and built \r\noffshore, then rebranded and resold. The teams building these devices \r\ndon't have the security expertise we've come to expect from the major \r\ncomputer and smartphone manufacturers, simply because the market won't \r\nstand for the additional costs that would require. These devices don't \r\nget security updates like our more expensive computers, and many don't \r\neven have a way to be patched. And, unlike our computers and phones, \r\nthey stay around for years and decades.\r\n\r\nAn additional market failure illustrated by the Dyn attack is that \r\nneither the seller nor the buyer of those devices cares about fixing the \r\nvulnerability. The owners of those devices don't care. They wanted a \r\nwebcam -- or thermostat, or refrigerator -- with nice features at a good \r\nprice. Even after they were recruited into this botnet, they still work \r\nfine -- you can't even tell they were used in the attack. The sellers of \r\nthose devices don't care: They've already moved on to selling newer and \r\nbetter models. There is no market solution because the insecurity \r\nprimarily affects other people. It's a form of invisible pollution.\r\n\r\nAnd, like pollution, the only solution is to regulate. The government \r\ncould impose minimum security standards on IoT manufacturers, forcing \r\nthem to make their devices secure even though their customers don't \r\ncare. They could impose liabilities on manufacturers, allowing companies \r\nlike Dyn to sue them if their devices are used in DDoS attacks. The \r\ndetails would need to be carefully scoped, but either of these options \r\nwould raise the cost of insecurity and give companies incentives to \r\nspend money making their devices secure.\r\n\r\nIt's true that this is a domestic solution to an international problem \r\nand that there's no U.S. regulation that will affect, say, an Asian-made \r\nproduct sold in South America, even though that product could still be \r\nused to take down U.S. websites. But the main costs in making software \r\ncome from development. If the United States and perhaps a few other \r\nmajor markets implement strong Internet-security regulations on IoT \r\ndevices, manufacturers will be forced to upgrade their security if they \r\nwant to sell to those markets. And any improvements they make in their \r\nsoftware will be available in their products wherever they are sold, \r\nsimply because it makes no sense to maintain two different versions of \r\nthe software. This is truly an area where the actions of a few countries \r\ncan drive worldwide change.\r\n\r\nRegardless of what you think about regulation vs. market solutions, I \r\nbelieve there is no choice. Governments will get involved in the IoT, \r\nbecause the risks are too great and the stakes are too high. Computers \r\nare now able to affect our world in a direct and physical manner.\r\n\r\nSecurity researchers have demonstrated the ability to remotely take \r\ncontrol of Internet-enabled cars. They've demonstrated ransomware \r\nagainst home thermostats and exposed vulnerabilities in implanted \r\nmedical devices. They've hacked voting machines and power plants. In one \r\nrecent paper, researchers showed how a vulnerability in smart lightbulbs \r\ncould be used to start a chain reaction, resulting in them *all* being \r\ncontrolled by the attackers -- that's every one in a city. Security \r\nflaws in these things could mean people dying and property being \r\ndestroyed.\r\n\r\nNothing motivates the U.S. government like fear. Remember 2001? A \r\nsmall-government Republican president created the Department of Homeland \r\nSecurity in the wake of the Sept. 11 terrorist attacks: a rushed and \r\nill-thought-out decision that we've been trying to fix for more than a \r\ndecade. A fatal IoT disaster will similarly spur our government into \r\naction, and it's unlikely to be well-considered and thoughtful action. \r\nOur choice isn't between government involvement and no government \r\ninvolvement. Our choice is between smarter government involvement and \r\nstupider government involvement. We have to start thinking about this \r\nnow. Regulations are necessary, important and complex -- and they're \r\ncoming. We can't afford to ignore these issues until it's too late.\r\n\r\nIn general, the software market demands that products be fast and cheap \r\nand that security be a secondary consideration. That was okay when \r\nsoftware didn't matter -- it was okay that your spreadsheet crashed once \r\nin a while. But a software bug that literally crashes your car is \r\nanother thing altogether. The security vulnerabilities in the Internet \r\nof Things are deep and pervasive, and they won't get fixed if the market \r\nis left to sort it out for itself. We need to proactively discuss good \r\nregulatory solutions; otherwise, a disaster will impose bad ones on us.\r\n\r\nThis essay previously appeared in the Washington Post.\r\nhttps://www.washingtonpost.com/posteverything/wp/2016/11/03/your-wifi-connected-thermostat-can-take-down-the-whole-internet-we-need-new-regulations/\r\n\r\n\r\nDDoS:\r\nhttps://www.washingtonpost.com/news/the-switch/wp/2016/10/21/someone-attacked-a-major-part-of-the-internets-infrastructure/\r\n\r\nIoT and DDoS:\r\nhttps://krebsonsecurity.com/2016/10/hacked-cameras-dvrs-powered-todays-massive-internet-outage/\r\n\r\nThe IoT market failure and regulation:\r\nhttps://www.schneier.com/essays/archives/2016/10/we_need_to_save_the_.html\r\nhttps://www.wired.com/2014/01/theres-no-good-way-to-patch-the-internet-of-things-and-thats-a-huge-problem/\r\nhttp://www.computerworld.com/article/3136650/security/after-ddos-attack-senator-seeks-industry-led-security-standards-for-iot-devices.html\r\n\r\nIoT ransomware:\r\nhttps://motherboard.vice.com/read/internet-of-things-ransomware-smart-thermostat\r\nmedical:\r\n\r\nHacking medical devices:\r\nhttp://motherboard.vice.com/read/hackers-killed-a-simulated-human-by-turning-off-its-pacemaker\r\nhttp://abcnews.go.com/US/vice-president-dick-cheney-feared-pacemaker-hacking/story?id=20621434\r\n\r\nHacking voting machines:\r\nhttp://www.politico.com/magazine/story/2016/08/2016-elections-russia-hack-how-to-hack-an-election-in-seven-minutes-214144\r\n\r\nHacking power plants:\r\nhttps://www.wired.com/2016/01/everything-we-know-about-ukraines-power-plant-hack/\r\n\r\nHacking light bulbs:\r\nhttp://iotworm.eyalro.net\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Schneier News\r\n\r\n\r\nI am speaking in Cambridge, MA on November 15 at the Harvard Big-Data \r\nClub.\r\nhttp://harvardbigdata.com/event/keynote-lecture-bruce-schneier\r\n\r\nI am speaking in Palm Springs, CA on November 30 at the TEDMED \r\nConference.\r\nhttp://www.tedmed.com/speakers/show?id=627300\r\n\r\nI am participating in the Resilient end-of-year webinar on December 8.\r\nhttp://info.resilientsystems.com/webinar-eoy-cybersecurity-2016-review-2017-predictions\r\n\r\nI am speaking on December 14 in Accra at the University of Ghana.\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Virtual Kidnapping\r\n\r\n\r\n\r\nThis is a harrowing story of a scam artist that convinced a mother that \r\nher daughter had been kidnapped. It's unclear if these virtual \r\nkidnappers use data about their victims, or just call people at random \r\nand hope to get lucky. Still, it's a new criminal use of smartphones and \r\nubiquitous information. Reminds me of the scammers who call low-wage \r\nworkers at retail establishments late at night and convince them to do \r\noutlandish and occasionally dangerous things.\r\nhttps://www.washingtonpost.com/local/we-have-your-daughter-a-virtual-kidnapping-and-a-mothers-five-hours-of-hell/2016/10/03/8f082690-8963-11e6-875e-2c1bfe943b66_story.html\r\nMore stories are here.\r\nhttp://www.nbcwashington.com/investigations/Several-Virtual-Kidnapping-Attempts-in-Maryland-Recently-375792991.html\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Intelligence Oversight and How It Can Fail\r\n\r\n\r\n\r\nFormer NSA attorneys John DeLong and Susan Hennessay have written a \r\nfascinating article describing a particular incident of oversight \r\nfailure inside the NSA. Technically, the story hinges on a definitional \r\ndifference between the NSA and the FISA court meaning of the word \r\n\"archived.\" (For the record, I would have defaulted to the NSA's \r\ninterpretation, which feels more accurate technically.) But while the \r\nstory is worth reading, what's especially interesting are the broader \r\nissues about how a nontechnical judiciary can provide oversight over a \r\nvery technical data collection-and-analysis organization -- especially \r\nif the oversight must largely be conducted in secret.\r\n\r\nIn many places I have separated different kinds of oversight: are we \r\ndoing things right versus are we doing the right things? This is very \r\nmuch about the first: is the NSA complying with the rules the courts \r\nimpose on them? I believe that the NSA tries very hard to follow the \r\nrules it's given, while at the same time being very aggressive about how \r\nit interprets any kind of ambiguities and using its nonadversarial \r\nrelationship with its overseers to its advantage.\r\n\r\nThe only possible solution I can see to all of this is more public \r\nscrutiny. Secrecy is toxic here.\r\n\r\nhttps://www.lawfareblog.com/understanding-footnote-14-nsa-lawyering-oversight-and-compliance\r\n\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Whistleblower Investigative Report on NSA Suite B Cryptography\r\n\r\n\r\n\r\nThe NSA has been abandoning secret and proprietary cryptographic \r\nalgorithms in favor of commercial public algorithms, generally known as \r\n\"Suite B.\" In 2010, an NSA employee filed some sort of whistleblower \r\ncomplaint, alleging that this move is both insecure and wasteful.  The \r\nUS DoD Inspector General investigated and wrote a report in 2011.\r\n\r\nThe report -- slightly redacted and declassified -- found that there was \r\nno wrongdoing. But the report is an interesting window into the NSA's \r\nsystem of algorithm selection and testing (pages 5 and 6), as well as \r\nhow they investigate whistleblower complaints.\r\n\r\nhttp://www.dodig.mil/FOIA/err/11-INTEL-06%20(Redacted).pdf\r\n\r\nSuite B Cryptography:\r\nhttp://csrc.nist.gov/groups/SMA/ispab/documents/minutes/2006-03/E_Barker-March2006-ISPAB.pdf\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\nSince 1998, CRYPTO-GRAM has been a free monthly newsletter providing \r\nsummaries, analyses, insights, and commentaries on security: computer \r\nand otherwise. You can subscribe, unsubscribe, or change your address on \r\nthe Web at . Back issues are \r\nalso available at that URL.\r\n\r\nPlease feel free to forward CRYPTO-GRAM, in whole or in part, to \r\ncolleagues and friends who will find it valuable. Permission is also \r\ngranted to reprint CRYPTO-GRAM, as long as it is reprinted in its \r\nentirety.\r\n\r\nCRYPTO-GRAM is written by Bruce Schneier. Bruce Schneier is an \r\ninternationally renowned security technologist, called a \"security guru\" \r\nby The Economist. He is the author of 13 books -- including his latest, \r\n\"Data and Goliath\" -- as well as hundreds of articles, essays, and \r\nacademic papers. His influential newsletter \"Crypto-Gram\" and his blog \r\n\"Schneier on Security\" are read by over 250,000 people. He has testified \r\nbefore Congress, is a frequent guest on television and radio, has served \r\non several government committees, and is regularly quoted in the press. \r\nSchneier is a fellow at the Berkman Center for Internet and Society at \r\nHarvard Law School, a program fellow at the New America Foundation's \r\nOpen Technology Institute, a board member of the Electronic Frontier \r\nFoundation, an Advisory Board Member of the Electronic Privacy \r\nInformation Center, and the Chief Technology Officer at Resilient, an \r\nIBM Company.  See .\r\n\r\nCrypto-Gram is a personal newsletter. Opinions expressed are not \r\nnecessarily those of Resilient, an IBM Company.\r\n\r\nCopyright (c) 2016 by Bruce Schneier.\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n\r\n\r\n\r\n\r\nTo unsubscribe from Crypto-Gram, click this link:\r\n\r\nhttps://lists.schneier.com/cgi-bin/mailman/options/crypto-gram/christine%40spang.cc?login-unsub=Unsubscribe\r\n\r\nYou will be e-mailed a confirmation message.  Follow the instructions in that message to confirm your removal from the list.\r\n"}},"desiredParts":[{"id":"1","encoding":"7BIT","mimetype":"text/plain"}],"result":{"id":"06cee3acb8fb7682fb622fcbac850cf25579be8a9b0afea9eed9f7454b53eb42","to":[{"name":"","email":"christine@spang.cc"}],"cc":[{"name":"Crypto-Gram Mailing List","email":"crypto-gram@lists.schneier.com"}],"bcc":[],"from":[{"name":"Bruce Schneier","email":"schneier@schneier.com"}],"replyTo":[],"accountId":"test-account-id","body":"
\r\n             CRYPTO-GRAM\r\n\r\n          November 15, 2016\r\n\r\n          by Bruce Schneier\r\n    CTO, Resilient, an IBM Company\r\n        schneier@schneier.com\r\n       https://www.schneier.com\r\n\r\n\r\nA free monthly newsletter providing summaries, analyses, insights, and \r\ncommentaries on security: computer and otherwise.\r\n\r\nFor back issues, or to subscribe, visit \r\n.\r\n\r\nYou can read this issue on the web at \r\n. These \r\nsame essays and news items appear in the \"Schneier on Security\" blog at \r\n, along with a lively and intelligent \r\ncomment section. An RSS feed is available.\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\nIn this issue:\r\n      Election Security\r\n      News\r\n      Lessons From the Dyn DDoS Attack\r\n      Regulation of the Internet of Things\r\n      Schneier News\r\n      Virtual Kidnapping\r\n      Intelligence Oversight and How It Can Fail\r\n      Whistleblower Investigative Report on NSA Suite B Cryptography\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Election Security\r\n\r\n\r\n\r\nIt's over. The voting went smoothly. As of the time of writing, there \r\nare no serious fraud allegations, nor credible evidence that anyone \r\ntampered with voting rolls or voting machines. And most important, the \r\nresults are not in doubt.\r\n\r\nWhile we may breathe a collective sigh of relief about that, we can't \r\nignore the issue until the next election. The risks remain.\r\n\r\nAs computer security experts have been saying for years, our newly \r\ncomputerized voting systems are vulnerable to attack by both individual \r\nhackers and government-sponsored cyberwarriors. It is only a matter of \r\ntime before such an attack happens.\r\n\r\nElectronic voting machines can be hacked, and those machines that do not \r\ninclude a paper ballot that can verify each voter's choice can be hacked \r\nundetectably. Voting rolls are also vulnerable; they are all \r\ncomputerized databases whose entries can be deleted or changed to sow \r\nchaos on Election Day.\r\n\r\nThe largely ad hoc system in states for collecting and tabulating \r\nindividual voting results is vulnerable as well. While the difference \r\nbetween theoretical if demonstrable vulnerabilities and an actual attack \r\non Election Day is considerable, we got lucky this year. Not just \r\npresidential elections are at risk, but state and local elections, too.\r\n\r\nTo be very clear, this is not about voter fraud. The risks of ineligible \r\npeople voting, or people voting twice, have been repeatedly shown to be \r\nvirtually nonexistent, and \"solutions\" to this problem are largely \r\nvoter-suppression measures. Election fraud, however, is both far more \r\nfeasible and much more worrisome.\r\n\r\nHere's my worry. On the day after an election, someone claims that a \r\nresult was hacked. Maybe one of the candidates points to a wide \r\ndiscrepancy between the most recent polls and the actual results. Maybe \r\nan anonymous person announces that he hacked a particular brand of \r\nvoting machine, describing in detail how. Or maybe it's a system failure \r\nduring Election Day: voting machines recording significantly fewer votes \r\nthan there were voters, or zero votes for one candidate or another. \r\n(These are not theoretical occurrences; they have both happened in the \r\nUnited States before, though because of error, not malice.)\r\n\r\nWe have no procedures for how to proceed if any of these things happen. \r\nThere's no manual, no national panel of experts, no regulatory body to \r\nsteer us through this crisis. How do we figure out if someone hacked the \r\nvote? Can we recover the true votes, or are they lost? What do we do \r\nthen?\r\n\r\nFirst, we need to do more to secure our elections system. We should \r\ndeclare our voting systems to be critical national infrastructure. This \r\nis largely symbolic, but it demonstrates a commitment to secure \r\nelections and makes funding and other resources available to states.\r\n\r\nWe need national security standards for voting machines, and funding for \r\nstates to procure machines that comply with those standards. \r\nVoting-security experts can deal with the technical details, but such \r\nmachines must include a paper ballot that provides a record verifiable \r\nby voters. The simplest and most reliable way to do that is already \r\npracticed in 37 states: optical-scan paper ballots, marked by the \r\nvoters, counted by computer but recountable by hand. And we need a \r\nsystem of pre-election and postelection security audits to increase \r\nconfidence in the system.\r\n\r\nSecond, election tampering, either by a foreign power or by a domestic \r\nactor, is inevitable, so we need detailed procedures to follow -- both \r\ntechnical procedures to figure out what happened, and legal procedures \r\nto figure out what to do -- that will efficiently get us to a fair and \r\nequitable election resolution. There should be a board of independent \r\ncomputer-security experts to unravel what happened, and a board of \r\nindependent election officials, either at the Federal Election \r\nCommission or elsewhere, empowered to determine and put in place an \r\nappropriate response.\r\n\r\nIn the absence of such impartial measures, people rush to defend their \r\ncandidate and their party. Florida in 2000 was a perfect example. What \r\ncould have been a purely technical issue of determining the intent of \r\nevery voter became a battle for who would win the presidency. The \r\ndebates about hanging chads and spoiled ballots and how broad the \r\nrecount should be were contested by people angling for a particular \r\noutcome. In the same way, after a hacked election, partisan politics \r\nwill place tremendous pressure on officials to make decisions that \r\noverride fairness and accuracy.\r\n\r\nThat is why we need to agree on policies to deal with future election \r\nfraud. We need procedures to evaluate claims of voting-machine hacking. \r\nWe need a fair and robust vote-auditing process. And we need all of this \r\nin place before an election is hacked and battle lines are drawn.\r\n\r\nIn response to Florida, the Help America Vote Act of 2002 required each \r\nstate to publish its own guidelines on what constitutes a vote. Some \r\nstates -- Indiana, in particular -- set up a \"war room\" of public and \r\nprivate cybersecurity experts ready to help if anything did occur. While \r\nthe Department of Homeland Security is assisting some states with \r\nelection security, and the F.B.I. and the Justice Department made some \r\npreparations this year, the approach is too piecemeal.\r\n\r\nElections serve two purposes. First, and most obvious, they are how we \r\nchoose a winner. But second, and equally important, they convince the \r\nloser -- and all the supporters -- that he or she lost. To achieve the \r\nfirst purpose, the voting system must be fair and accurate. To achieve \r\nthe second one, it must be *shown* to be fair and accurate.\r\n\r\nWe need to have these conversations before something happens, when \r\neveryone can be calm and rational about the issues. The integrity of our \r\nelections is at stake, which means our democracy is at stake.\r\n\r\nThis essay previously appeared in the New York Times.\r\nhttp://www.nytimes.com/2016/11/09/opinion/american-elections-will-be-hacked.html\r\n\r\nElection-machine vulnerabilities:\r\nhttps://www.washingtonpost.com/posteverything/wp/2016/07/27/by-november-russian-hackers-could-target-voting-machines/\r\n\r\nElections are hard to rig:\r\nhttps://www.washingtonpost.com/news/the-fix/wp/2016/08/03/one-reason-to-doubt-the-presidential-election-will-be-rigged-its-a-lot-harder-than-it-seems/\r\n\r\nVoting systems as critical infrastructure:\r\nhttps://papers.ssrn.com/sol3/papers.cfm?abstract_id=2852461\r\n\r\nVoting machine security:\r\nhttps://www.verifiedvoting.org/\r\nhttp://votingmachines.procon.org/view.answers.php?questionID=000291\r\nhttp://votingmachines.procon.org/view.answers.php?questionID=000291\r\n\r\nElection-defense preparations for 2016:\r\nhttp://www.usatoday.com/story/tech/news/2016/11/05/election-2016-cyber-hack-issues-homeland-security-indiana-pennsylvania-election-protection-verified-voter/93262960/\r\nhttp://www.nbcnews.com/storyline/2016-election-day/all-hands-deck-protect-election-hack-say-officials-n679271\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      News\r\n\r\n\r\n\r\nLance Spitzner looks at the safety features of a power saw and tries to \r\napply them to Internet security.\r\nhttps://securingthehuman.sans.org/blog/2016/10/18/what-iot-and-security-needs-to-learn-from-the-dewalt-mitre-saw\r\n\r\nResearchers discover a clever attack that bypasses the address space \r\nlayout randomization (ALSR) on Intel's CPUs.\r\nhttp://arstechnica.com/security/2016/10/flaw-in-intel-chips-could-make-malware-attacks-more-potent/\r\nhttp://www.cs.ucr.edu/~nael/pubs/micro16.pdf\r\n\r\nIn an interviw in Wired, President Obama talks about AI risk, \r\ncybersecurity, and more.\r\nhttps://www.wired.com/2016/10/president-obama-mit-joi-ito-interview/\r\n\r\nPrivacy makes workers more productive. Interesting research.\r\nhttps://www.psychologytoday.com/blog/the-outsourced-mind/201604/want-people-behave-better-give-them-more-privacy\r\n\r\nNews about the DDOS attacks against Dyn.\r\nhttps://motherboard.vice.com/read/twitter-reddit-spotify-were-collateral-damage-in-major-internet-attack\r\nhttps://krebsonsecurity.com/2016/10/ddos-on-dyn-impacts-twitter-spotify-reddit/\r\nhttps://motherboard.vice.com/read/blame-the-internet-of-things-for-destroying-the-internet-today\r\n\r\nJosephine Wolff examines different Internet governance stakeholders and \r\nhow they frame security debates.\r\nhttps://policyreview.info/articles/analysis/what-we-talk-about-when-we-talk-about-cybersecurity-security-internet-governance\r\n\r\nThe UK is admitting \"offensive cyber\" operations against ISIS/Daesh. I \r\nthink this might be the first time it has been openly acknowledged.\r\nhttps://www.theguardian.com/politics/blog/live/2016/oct/20/philip-green-knighthood-commons-set-to-debate-stripping-philip-green-of-his-knighthood-politics-live\r\n\r\nIt's not hard to imagine the criminal possibilities of automation, \r\nautonomy, and artificial intelligence. But the imaginings are becoming \r\nmainstream -- and the future isn't too far off.\r\nhttp://www.nytimes.com/2016/10/24/technology/artificial-intelligence-evolves-with-its-criminal-potential.html\r\n\r\nAlong similar lines, computers are able to predict court verdicts. My \r\nguess is that the real use here isn't to predict actual court verdicts, \r\nbut for well-paid defense teams to test various defensive tactics.\r\nhttp://www.telegraph.co.uk/science/2016/10/23/artifically-intelligent-judge-developed-which-can-predict-court/\r\n\r\nGood long article on the 2015 attack against the US Office of Personnel \r\nManagement.\r\nhttps://www.wired.com/2016/10/inside-cyberattack-shocked-us-government/\r\n\r\nHow Powell's and Podesta's e-mail accounts were hacked. It was phishing.\r\nhttps://motherboard.vice.com/read/how-hackers-broke-into-john-podesta-and-colin-powells-gmail-accounts\r\n\r\nA year and a half ago, I wrote about hardware bit-flipping attacks, \r\nwhich were then largely theoretical. Now, they can be used to root \r\nAndroid phones.\r\nhttp://arstechnica.com/security/2016/10/using-rowhammer-bitflips-to-root-android-phones-is-now-a-thing/\r\nhttps://vvdveen.com/publications/drammer.pdf\r\nhttps://www.vusec.net/projects/drammer/\r\n\r\nEavesdropping on typing while connected over VoIP.\r\nhttps://arxiv.org/pdf/1609.09359.pdf\r\nhttps://news.uci.edu/research/typing-while-skyping-could-compromise-privacy/\r\n\r\nAn impressive Chinese device that automatically reads marked cards in \r\norder to cheat at poker and other card games.\r\nhttps://www.elie.net/blog/security/fuller-house-exposing-high-end-poker-cheating-devices\r\n\r\nA useful guide on how to avoid kidnapping children on Halloween.\r\nhttp://reductress.com/post/how-to-not-kidnap-any-kids-on-halloween-not-even-one/\r\n\r\nA card game based on the iterated prisoner's dilemma.\r\nhttps://opinionatedgamers.com/2016/10/26/h-m-s-dolores-game-review-by-chris-wray/\r\n\r\nThere's another leak of NSA hacking tools and data from the Shadow \r\nBrokers. This one includes a list of hacked sites. The data is old, but \r\nyou can see if you've been hacked.\r\nhttp://arstechnica.co.uk/security/2016/10/new-leak-may-show-if-you-were-hacked-by-the-nsa/\r\nHonestly, I am surprised by this release. I thought that the original \r\nShadow Brokers dump was everything. Now that we know they held things \r\nback, there could easily be more releases.\r\nhttp://www.networkworld.com/article/3137065/security/shadow-brokers-leak-list-of-nsa-targets-and-compromised-servers.html\r\nNote that the Hague-based Organization for the Prohibition of Chemical \r\nWeapons is on the list, hacked in 2000.\r\nhttps://boingboing.net/2016/11/06/in-2000-the-nsa-hacked-the-ha.html\r\n\r\nFree cybersecurity MOOC from F-Secure and the University of Finland.\r\nhttp://mooc.fi/courses/2016/cybersecurity/\r\n\r\nResearchers have trained a neural network to encrypt its communications. \r\nThis story is more about AI and neural networks than it is about \r\ncryptography. The algorithm isn't any good, but is a perfect example of \r\nwhat I've heard called \"Schneier's Law\": Anyone can design a cipher that \r\nthey themselves cannot break.\r\nhttps://www.newscientist.com/article/2110522-googles-neural-networks-invent-their-own-encryption/\r\nhttp://arstechnica.com/information-technology/2016/10/google-ai-neural-network-cryptography/\r\nhttps://www.engadget.com/2016/10/28/google-ai-created-its-own-form-of-encryption/\r\nhttps://arxiv.org/pdf/1610.06918v1.pdf\r\nSchneier's Law:\r\nhttps://www.schneier.com/blog/archives/2011/04/schneiers_law.html\r\n\r\nGoogle now links anonymous browser tracking with identifiable tracking. \r\nThe article also explains how to opt out.\r\nhttps://www.propublica.org/article/google-has-quietly-dropped-ban-on-personally-identifiable-web-tracking\r\n\r\nNew Atlas has a great three-part feature on the history of hacking as \r\nportrayed in films, including video clips. The 1980s. The 1990s. The \r\n2000s.\r\nhttp://newatlas.com/history-hollywood-hacking-1980s/45482/\r\nhttp://newatlas.com/hollywood-hacking-movies-1990s/45623/\r\nhttp://newatlas.com/hollywood-hacking-2000s/45965\r\n\r\nFor years, the DMCA has been used to stifle legitimate research into the \r\nsecurity of embedded systems. Finally, the research exemption to the \r\nDMCA is in effect (for two years, but we can hope it'll be extended \r\nforever).\r\nhttps://www.wired.com/2016/10/hacking-car-pacemaker-toaster-just-became-legal/\r\nhttps://www.eff.org/deeplinks/2016/10/why-did-we-have-wait-year-fix-our-cars\r\n\r\nFirefox is removing the battery status API, citing privacy concerns.\r\nhttps://www.fxsitecompat.com/en-CA/docs/2016/battery-status-api-has-been-removed/\r\nhttps://eprint.iacr.org/2015/616.pdf\r\nW3C is updating the spec.\r\nhttps://www.w3.org/TR/battery-status/#acknowledgements\r\nHere's a battery tracker found in the wild.\r\nhttp://randomwalker.info/publications/OpenWPM_1_million_site_tracking_measurement.pdf\r\n\r\nElection-day humor from 2004, but still relevent.\r\nhttp://www.ganssle.com/tem/tem316.html#article2\r\n\r\nA self-propagating smart light bulb worm.\r\nhttp://iotworm.eyalro.net/\r\nhttps://boingboing.net/2016/11/09/a-lightbulb-worm-could-take-ov.html\r\nhttps://tech.slashdot.org/story/16/11/09/0041201/researchers-hack-philips-hue-smart-bulbs-using-a-drone\r\nThis is exactly the sort of Internet-of-Things attack that has me \r\nworried.\r\n\r\nAd networks are surreptitiously using ultrasonic communications to jump \r\nfrom device to device. It should come as no surprise that this \r\ncommunications channel can be used to hack devices as well.\r\nhttps://www.newscientist.com/article/2110762-your-homes-online-gadgets-could-be-hacked-by-ultrasound/\r\nhttps://www.schneier.com/blog/archives/2015/11/ads_surreptitio.html\r\n\r\nThis is some interesting research. You can fool facial recognition \r\nsystems by wearing glasses printed with elements of other peoples' \r\nfaces.\r\nhttps://www.cs.cmu.edu/~sbhagava/papers/face-rec-ccs16.pdf\r\nhttp://qz.com/823820/carnegie-mellon-made-a-special-pair-of-glasses-that-lets-you-steal-a-digital-identity/\r\nhttps://boingboing.net/2016/11/02/researchers-trick-facial-recog.html\r\n\r\nInteresting research: \"Using Artificial Intelligence to Identify State \r\nSecrets,\" https://arxiv.org/abs/1611.00356\r\n\r\nThere's a Kickstarter for a sticker that you can stick on a glove and \r\nthen register with a biometric access system like an iPhone. It's an \r\ninteresting security trade-off: swapping something you are (the \r\nbiometric) with something you have (the glove).\r\nhttps://www.kickstarter.com/projects/nanotips/taps-touchscreen-sticker-w-touch-id-ships-before-x?token=5b586aa6\r\nhttps://gizmodo.com/these-fake-fingerprint-stickers-let-you-access-a-protec-1788710313\r\n\r\nJulian Oliver has designed and built a cellular eavesdropping device \r\nthat's disguised as an old HP printer. It's more of a conceptual art \r\npiece than an actual piece of eavesdropping equipment, but it still \r\nmakes the point.\r\nhttps://julianoliver.com/output/stealth-cell-tower\r\nhttps://www.wired.com/2016/11/evil-office-printer-hijacks-cellphone-connection/\r\nhttps://boingboing.net/2016/11/03/a-fake-hp-printer-thats-actu.html\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Lessons From the Dyn DDoS Attack\r\n\r\n\r\n\r\nA week ago Friday, someone took down numerous popular websites in a \r\nmassive distributed denial-of-service (DDoS) attack against the domain \r\nname provider Dyn. DDoS attacks are neither new nor sophisticated. The \r\nattacker sends a massive amount of traffic, causing the victim's system \r\nto slow to a crawl and eventually crash. There are more or less clever \r\nvariants, but basically, it's a datapipe-size battle between attacker \r\nand victim. If the defender has a larger capacity to receive and process \r\ndata, he or she will win. If the attacker can throw more data than the \r\nvictim can process, he or she will win.\r\n\r\nThe attacker can build a giant data cannon, but that's expensive. It is \r\nmuch smarter to recruit millions of innocent computers on the internet. \r\nThis is the \"distributed\" part of the DDoS attack, and pretty much how \r\nit's worked for decades. Cybercriminals infect innocent computers around \r\nthe internet and recruit them into a botnet. They then target that \r\nbotnet against a single victim.\r\n\r\nYou can imagine how it might work in the real world. If I can trick tens \r\nof thousands of others to order pizzas to be delivered to your house at \r\nthe same time, I can clog up your street and prevent any legitimate \r\ntraffic from getting through. If I can trick many millions, I might be \r\nable to crush your house from the weight. That's a DDoS attack -- it's \r\nsimple brute force.\r\n\r\nAs you'd expect, DDoSers have various motives. The attacks started out \r\nas a way to show off, then quickly transitioned to a method of \r\nintimidation -- or a way of just getting back at someone you didn't \r\nlike. More recently, they've become vehicles of protest. In 2013, the \r\nhacker group Anonymous petitioned the White House to recognize DDoS \r\nattacks as a legitimate form of protest. Criminals have used these \r\nattacks as a means of extortion, although one group found that just the \r\nfear of attack was enough. Military agencies are also thinking about \r\nDDoS as a tool in their cyberwar arsenals. A 2007 DDoS attack against \r\nEstonia was blamed on Russia and widely called an act of cyberwar.\r\n\r\nThe DDoS attack against Dyn two weeks ago was nothing new, but it \r\nillustrated several important trends in computer security.\r\n\r\nThese attack techniques are broadly available. Fully capable DDoS attack \r\ntools are available for free download. Criminal groups offer DDoS \r\nservices for hire. The particular attack technique used against Dyn was \r\nfirst used a month earlier. It's called Mirai, and since the source code \r\nwas released four weeks ago, over a dozen botnets have incorporated the \r\ncode.\r\n\r\nThe Dyn attacks were probably not originated by a government. The \r\nperpetrators were most likely hackers mad at Dyn for helping Brian Krebs \r\nidentify -- and the FBI arrest -- two Israeli hackers who were running a \r\nDDoS-for-hire ring. Recently I have written about probing DDoS attacks \r\nagainst internet infrastructure companies that appear to be perpetrated \r\nby a nation-state. But, honestly, we don't know for sure.\r\n\r\nThis is important. Software spreads capabilities. The smartest attacker \r\nneeds to figure out the attack and write the software. After that, \r\nanyone can use it. There's not even much of a difference between \r\ngovernment and criminal attacks. In December 2014, there was a \r\nlegitimate debate in the security community as to whether the massive \r\nattack against Sony had been perpetrated by a nation-state with a $20 \r\nbillion military budget or a couple of guys in a basement somewhere. The \r\ninternet is the only place where we can't tell the difference. Everyone \r\nuses the same tools, the same techniques and the same tactics.\r\n\r\nThese attacks are getting larger. The Dyn DDoS attack set a record at \r\n1.2 Tbps. The previous record holder was the attack against \r\ncybersecurity journalist Brian Krebs a month prior at 620 Gbps. This is \r\nmuch larger than required to knock the typical website offline. A year \r\nago, it was unheard of. Now it occurs regularly.\r\n\r\nThe botnets attacking Dyn and Brian Krebs consisted largely of unsecure \r\nInternet of Things (IoT) devices -- webcams, digital video recorders, \r\nrouters and so on. This isn't new, either. We've already seen \r\ninternet-enabled refrigerators and TVs used in DDoS botnets. But again, \r\nthe scale is bigger now. In 2014, the news was hundreds of thousands of \r\nIoT devices -- the Dyn attack used millions. Analysts expect the IoT to \r\nincrease the number of things on the internet by a factor of 10 or more. \r\nExpect these attacks to similarly increase.\r\n\r\nThe problem is that these IoT devices are unsecure and likely to remain \r\nthat way. The economics of internet security don't trickle down to the \r\nIoT. Commenting on the Krebs attack last month, I wrote:\r\n\r\n     The market can't fix this because neither the buyer nor the\r\n     seller cares. Think of all the CCTV cameras and DVRs used in\r\n     the attack against Brian Krebs. The owners of those devices\r\n     don't care. Their devices were cheap to buy, they still work,\r\n     and they don't even know Brian. The sellers of those devices\r\n     don't care: They're now selling newer and better models, and\r\n     the original buyers only cared about price and features. There\r\n     is no market solution because the insecurity is what economists\r\n     call an externality: It's an effect of the purchasing decision\r\n     that affects other people. Think of it kind of like invisible\r\n     pollution.\r\n\r\nTo be fair, one company that made some of the unsecure things used in \r\nthese attacks recalled its unsecure webcams. But this is more of a \r\npublicity stunt than anything else. I would be surprised if the company \r\ngot many devices back. We already know that the reputational damage from \r\nhaving your unsecure software made public isn't large and doesn't last. \r\nAt this point, the market still largely rewards sacrificing security in \r\nfavor of price and time-to-market.\r\n\r\nDDoS prevention works best deep in the network, where the pipes are the \r\nlargest and the capability to identify and block the attacks is the most \r\nevident. But the backbone providers have no incentive to do this. They \r\ndon't feel the pain when the attacks occur and they have no way of \r\nbilling for the service when they provide it. So they let the attacks \r\nthrough and force the victims to defend themselves. In many ways, this \r\nis similar to the spam problem. It, too, is best dealt with in the \r\nbackbone, but similar economics dump the problem onto the endpoints.\r\n\r\nWe're unlikely to get any regulation forcing backbone companies to clean \r\nup either DDoS attacks or spam, just as we are unlikely to get any \r\nregulations forcing IoT manufacturers to make their systems secure. This \r\nis me again:\r\n\r\n     What this all means is that the IoT will remain insecure unless\r\n     government steps in and fixes the problem. When we have market\r\n     failures, government is the only solution. The government could\r\n     impose security regulations on IoT manufacturers, forcing them\r\n     to make their devices secure even though their customers don't\r\n     care. They could impose liabilities on manufacturers, allowing\r\n     people like Brian Krebs to sue them. Any of these would raise\r\n     the cost of insecurity and give companies incentives to spend\r\n     money making their devices secure.\r\n\r\nThat leaves the victims to pay. This is where we are in much of computer \r\nsecurity. Because the hardware, software and networks we use are so \r\nunsecure, we have to pay an entire industry to provide after-the-fact \r\nsecurity.\r\n\r\nThere are solutions you can buy. Many companies offer DDoS protection, \r\nalthough they're generally calibrated to the older, smaller attacks. We \r\ncan safely assume that they'll up their offerings, although the cost \r\nmight be prohibitive for many users. Understand your risks. Buy \r\nmitigation if you need it, but understand its limitations. Know the \r\nattacks are possible and will succeed if large enough. And the attacks \r\nare getting larger all the time. Prepare for that.\r\n\r\nThis essay previously appeared on the SecurityIntelligence website.\r\nhttps://securityintelligence.com/lessons-from-the-dyn-ddos-attack/\r\n\r\nhttps://securityintelligence.com/news/multi-phased-ddos-attack-causes-hours-long-outages/\r\nhttp://arstechnica.com/information-technology/2016/10/inside-the-machine-uprising-how-cameras-dvrs-took-down-parts-of-the-internet/\r\nhttps://www.theguardian.com/technology/2016/oct/26/ddos-attack-dyn-mirai-botnet\r\nhttp://searchsecurity.techtarget.com/news/450401962/Details-emerging-on-Dyn-DNS-DDoS-attack-Mirai-IoT-botnet\r\nhttp://hub.dyn.com/static/hub.dyn.com/dyn-blog/dyn-statement-on-10-21-2016-ddos-attack.html\r\n\r\nDDoS petition:\r\nhttp://www.huffingtonpost.com/2013/01/12/anonymous-ddos-petition-white-house_n_2463009.html\r\n\r\nDDoS extortion:\r\nhttps://securityintelligence.com/ddos-extortion-easy-and-lucrative/\r\nhttp://www.computerworld.com/article/3061813/security/empty-ddos-threats-deliver-100k-to-extortion-group.html\r\n\r\nDDoS against Estonia:\r\nhttp://www.iar-gwu.org/node/65\r\n\r\nDDoS for hire:\r\nhttp://www.forbes.com/sites/thomasbrewster/2016/10/23/massive-ddos-iot-botnet-for-hire-twitter-dyn-amazon/#11f82518c915\r\n\r\nMirai:\r\nhttps://www.arbornetworks.com/blog/asert/mirai-iot-botnet-description-ddos-attack-mitigation/\r\nhttps://krebsonsecurity.com/2016/10/source-code-for-iot-botnet-mirai-released/\r\nhttps://threatpost.com/mirai-bots-more-than-double-since-source-code-release/121368/\r\n\r\nKrebs:\r\nhttp://krebsonsecurity.com/2016/09/israeli-online-attack-service-vdos-earned-600000-in-two-years/\r\nhttp://www.theverge.com/2016/9/11/12878692/vdos-israeli-teens-ddos-cyberattack-service-arrested\r\nhttps://krebsonsecurity.com/2016/09/krebsonsecurity-hit-with-record-ddos/\r\nhttp://www.businessinsider.com/akamai-brian-krebs-ddos-attack-2016-9\r\n\r\nNation-state DDoS Attacks:\r\nhttps://www.schneier.com/blog/archives/2016/09/someone_is_lear.html\r\n\r\nNorth Korea and Sony:\r\nhttps://www.theatlantic.com/international/archive/2014/12/did-north-korea-really-attack-sony/383973/\r\n\r\nInternet of Things (IoT) security:\r\nhttps://securityintelligence.com/will-internet-things-leveraged-ruin-companys-day-understanding-iot-security/\r\nhttps://thehackernews.com/2014/01/100000-refrigerators-and-other-home.html\r\n\r\nEver larger DDoS Attacks:\r\nhttp://www.ibtimes.co.uk/biggest-ever-terabit-scale-ddos-attack-yet-could-be-horizon-experts-warn-1588364\r\n\r\nMy previous essay on this:\r\nhttps://www.schneier.com/essays/archives/2016/10/we_need_to_save_the_.html\r\n\r\nrecalled:\r\nhttp://www.zdnet.com/article/chinese-tech-giant-recalls-webcams-used-in-dyn-cyberattack/\r\n\r\nidentify and block the attacks:\r\nhttp://www.ibm.com/security/threat-protection/\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Regulation of the Internet of Things\r\n\r\n\r\n\r\nLate last month, popular websites like Twitter, Pinterest, Reddit and \r\nPayPal went down for most of a day. The distributed denial-of-service \r\nattack that caused the outages, and the vulnerabilities that made the \r\nattack possible, was as much a failure of market and policy as it was of \r\ntechnology. If we want to secure our increasingly computerized and \r\nconnected world, we need more government involvement in the security of \r\nthe \"Internet of Things\" and increased regulation of what are now \r\ncritical and life-threatening technologies. It's no longer a question of \r\nif, it's a question of when.\r\n\r\nFirst, the facts. Those websites went down because their domain name \r\nprovider -- a company named Dyn -- was forced offline. We don't know who \r\nperpetrated that attack, but it could have easily been a lone hacker. \r\nWhoever it was launched a distributed denial-of-service attack against \r\nDyn by exploiting a vulnerability in large numbers -- possibly millions \r\n-- of Internet-of-Things devices like webcams and digital video \r\nrecorders, then recruiting them all into a single botnet. The botnet \r\nbombarded Dyn with traffic, so much that it went down. And when it went \r\ndown, so did dozens of websites.\r\n\r\nYour security on the Internet depends on the security of millions of \r\nInternet-enabled devices, designed and sold by companies you've never \r\nheard of to consumers who don't care about your security.\r\n\r\nThe technical reason these devices are insecure is complicated, but \r\nthere is a market failure at work. The Internet of Things is bringing \r\ncomputerization and connectivity to many tens of millions of devices \r\nworldwide. These devices will affect every aspect of our lives, because \r\nthey're things like cars, home appliances, thermostats, lightbulbs, \r\nfitness trackers, medical devices, smart streetlights and sidewalk \r\nsquares. Many of these devices are low-cost, designed and built \r\noffshore, then rebranded and resold. The teams building these devices \r\ndon't have the security expertise we've come to expect from the major \r\ncomputer and smartphone manufacturers, simply because the market won't \r\nstand for the additional costs that would require. These devices don't \r\nget security updates like our more expensive computers, and many don't \r\neven have a way to be patched. And, unlike our computers and phones, \r\nthey stay around for years and decades.\r\n\r\nAn additional market failure illustrated by the Dyn attack is that \r\nneither the seller nor the buyer of those devices cares about fixing the \r\nvulnerability. The owners of those devices don't care. They wanted a \r\nwebcam -- or thermostat, or refrigerator -- with nice features at a good \r\nprice. Even after they were recruited into this botnet, they still work \r\nfine -- you can't even tell they were used in the attack. The sellers of \r\nthose devices don't care: They've already moved on to selling newer and \r\nbetter models. There is no market solution because the insecurity \r\nprimarily affects other people. It's a form of invisible pollution.\r\n\r\nAnd, like pollution, the only solution is to regulate. The government \r\ncould impose minimum security standards on IoT manufacturers, forcing \r\nthem to make their devices secure even though their customers don't \r\ncare. They could impose liabilities on manufacturers, allowing companies \r\nlike Dyn to sue them if their devices are used in DDoS attacks. The \r\ndetails would need to be carefully scoped, but either of these options \r\nwould raise the cost of insecurity and give companies incentives to \r\nspend money making their devices secure.\r\n\r\nIt's true that this is a domestic solution to an international problem \r\nand that there's no U.S. regulation that will affect, say, an Asian-made \r\nproduct sold in South America, even though that product could still be \r\nused to take down U.S. websites. But the main costs in making software \r\ncome from development. If the United States and perhaps a few other \r\nmajor markets implement strong Internet-security regulations on IoT \r\ndevices, manufacturers will be forced to upgrade their security if they \r\nwant to sell to those markets. And any improvements they make in their \r\nsoftware will be available in their products wherever they are sold, \r\nsimply because it makes no sense to maintain two different versions of \r\nthe software. This is truly an area where the actions of a few countries \r\ncan drive worldwide change.\r\n\r\nRegardless of what you think about regulation vs. market solutions, I \r\nbelieve there is no choice. Governments will get involved in the IoT, \r\nbecause the risks are too great and the stakes are too high. Computers \r\nare now able to affect our world in a direct and physical manner.\r\n\r\nSecurity researchers have demonstrated the ability to remotely take \r\ncontrol of Internet-enabled cars. They've demonstrated ransomware \r\nagainst home thermostats and exposed vulnerabilities in implanted \r\nmedical devices. They've hacked voting machines and power plants. In one \r\nrecent paper, researchers showed how a vulnerability in smart lightbulbs \r\ncould be used to start a chain reaction, resulting in them *all* being \r\ncontrolled by the attackers -- that's every one in a city. Security \r\nflaws in these things could mean people dying and property being \r\ndestroyed.\r\n\r\nNothing motivates the U.S. government like fear. Remember 2001? A \r\nsmall-government Republican president created the Department of Homeland \r\nSecurity in the wake of the Sept. 11 terrorist attacks: a rushed and \r\nill-thought-out decision that we've been trying to fix for more than a \r\ndecade. A fatal IoT disaster will similarly spur our government into \r\naction, and it's unlikely to be well-considered and thoughtful action. \r\nOur choice isn't between government involvement and no government \r\ninvolvement. Our choice is between smarter government involvement and \r\nstupider government involvement. We have to start thinking about this \r\nnow. Regulations are necessary, important and complex -- and they're \r\ncoming. We can't afford to ignore these issues until it's too late.\r\n\r\nIn general, the software market demands that products be fast and cheap \r\nand that security be a secondary consideration. That was okay when \r\nsoftware didn't matter -- it was okay that your spreadsheet crashed once \r\nin a while. But a software bug that literally crashes your car is \r\nanother thing altogether. The security vulnerabilities in the Internet \r\nof Things are deep and pervasive, and they won't get fixed if the market \r\nis left to sort it out for itself. We need to proactively discuss good \r\nregulatory solutions; otherwise, a disaster will impose bad ones on us.\r\n\r\nThis essay previously appeared in the Washington Post.\r\nhttps://www.washingtonpost.com/posteverything/wp/2016/11/03/your-wifi-connected-thermostat-can-take-down-the-whole-internet-we-need-new-regulations/\r\n\r\n\r\nDDoS:\r\nhttps://www.washingtonpost.com/news/the-switch/wp/2016/10/21/someone-attacked-a-major-part-of-the-internets-infrastructure/\r\n\r\nIoT and DDoS:\r\nhttps://krebsonsecurity.com/2016/10/hacked-cameras-dvrs-powered-todays-massive-internet-outage/\r\n\r\nThe IoT market failure and regulation:\r\nhttps://www.schneier.com/essays/archives/2016/10/we_need_to_save_the_.html\r\nhttps://www.wired.com/2014/01/theres-no-good-way-to-patch-the-internet-of-things-and-thats-a-huge-problem/\r\nhttp://www.computerworld.com/article/3136650/security/after-ddos-attack-senator-seeks-industry-led-security-standards-for-iot-devices.html\r\n\r\nIoT ransomware:\r\nhttps://motherboard.vice.com/read/internet-of-things-ransomware-smart-thermostat\r\nmedical:\r\n\r\nHacking medical devices:\r\nhttp://motherboard.vice.com/read/hackers-killed-a-simulated-human-by-turning-off-its-pacemaker\r\nhttp://abcnews.go.com/US/vice-president-dick-cheney-feared-pacemaker-hacking/story?id=20621434\r\n\r\nHacking voting machines:\r\nhttp://www.politico.com/magazine/story/2016/08/2016-elections-russia-hack-how-to-hack-an-election-in-seven-minutes-214144\r\n\r\nHacking power plants:\r\nhttps://www.wired.com/2016/01/everything-we-know-about-ukraines-power-plant-hack/\r\n\r\nHacking light bulbs:\r\nhttp://iotworm.eyalro.net\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Schneier News\r\n\r\n\r\nI am speaking in Cambridge, MA on November 15 at the Harvard Big-Data \r\nClub.\r\nhttp://harvardbigdata.com/event/keynote-lecture-bruce-schneier\r\n\r\nI am speaking in Palm Springs, CA on November 30 at the TEDMED \r\nConference.\r\nhttp://www.tedmed.com/speakers/show?id=627300\r\n\r\nI am participating in the Resilient end-of-year webinar on December 8.\r\nhttp://info.resilientsystems.com/webinar-eoy-cybersecurity-2016-review-2017-predictions\r\n\r\nI am speaking on December 14 in Accra at the University of Ghana.\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Virtual Kidnapping\r\n\r\n\r\n\r\nThis is a harrowing story of a scam artist that convinced a mother that \r\nher daughter had been kidnapped. It's unclear if these virtual \r\nkidnappers use data about their victims, or just call people at random \r\nand hope to get lucky. Still, it's a new criminal use of smartphones and \r\nubiquitous information. Reminds me of the scammers who call low-wage \r\nworkers at retail establishments late at night and convince them to do \r\noutlandish and occasionally dangerous things.\r\nhttps://www.washingtonpost.com/local/we-have-your-daughter-a-virtual-kidnapping-and-a-mothers-five-hours-of-hell/2016/10/03/8f082690-8963-11e6-875e-2c1bfe943b66_story.html\r\nMore stories are here.\r\nhttp://www.nbcwashington.com/investigations/Several-Virtual-Kidnapping-Attempts-in-Maryland-Recently-375792991.html\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Intelligence Oversight and How It Can Fail\r\n\r\n\r\n\r\nFormer NSA attorneys John DeLong and Susan Hennessay have written a \r\nfascinating article describing a particular incident of oversight \r\nfailure inside the NSA. Technically, the story hinges on a definitional \r\ndifference between the NSA and the FISA court meaning of the word \r\n\"archived.\" (For the record, I would have defaulted to the NSA's \r\ninterpretation, which feels more accurate technically.) But while the \r\nstory is worth reading, what's especially interesting are the broader \r\nissues about how a nontechnical judiciary can provide oversight over a \r\nvery technical data collection-and-analysis organization -- especially \r\nif the oversight must largely be conducted in secret.\r\n\r\nIn many places I have separated different kinds of oversight: are we \r\ndoing things right versus are we doing the right things? This is very \r\nmuch about the first: is the NSA complying with the rules the courts \r\nimpose on them? I believe that the NSA tries very hard to follow the \r\nrules it's given, while at the same time being very aggressive about how \r\nit interprets any kind of ambiguities and using its nonadversarial \r\nrelationship with its overseers to its advantage.\r\n\r\nThe only possible solution I can see to all of this is more public \r\nscrutiny. Secrecy is toxic here.\r\n\r\nhttps://www.lawfareblog.com/understanding-footnote-14-nsa-lawyering-oversight-and-compliance\r\n\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n      Whistleblower Investigative Report on NSA Suite B Cryptography\r\n\r\n\r\n\r\nThe NSA has been abandoning secret and proprietary cryptographic \r\nalgorithms in favor of commercial public algorithms, generally known as \r\n\"Suite B.\" In 2010, an NSA employee filed some sort of whistleblower \r\ncomplaint, alleging that this move is both insecure and wasteful.  The \r\nUS DoD Inspector General investigated and wrote a report in 2011.\r\n\r\nThe report -- slightly redacted and declassified -- found that there was \r\nno wrongdoing. But the report is an interesting window into the NSA's \r\nsystem of algorithm selection and testing (pages 5 and 6), as well as \r\nhow they investigate whistleblower complaints.\r\n\r\nhttp://www.dodig.mil/FOIA/err/11-INTEL-06%20(Redacted).pdf\r\n\r\nSuite B Cryptography:\r\nhttp://csrc.nist.gov/groups/SMA/ispab/documents/minutes/2006-03/E_Barker-March2006-ISPAB.pdf\r\n\r\n\r\n** *** ***** ******* *********** *************\r\n\r\nSince 1998, CRYPTO-GRAM has been a free monthly newsletter providing \r\nsummaries, analyses, insights, and commentaries on security: computer \r\nand otherwise. You can subscribe, unsubscribe, or change your address on \r\nthe Web at . Back issues are \r\nalso available at that URL.\r\n\r\nPlease feel free to forward CRYPTO-GRAM, in whole or in part, to \r\ncolleagues and friends who will find it valuable. Permission is also \r\ngranted to reprint CRYPTO-GRAM, as long as it is reprinted in its \r\nentirety.\r\n\r\nCRYPTO-GRAM is written by Bruce Schneier. Bruce Schneier is an \r\ninternationally renowned security technologist, called a \"security guru\" \r\nby The Economist. He is the author of 13 books -- including his latest, \r\n\"Data and Goliath\" -- as well as hundreds of articles, essays, and \r\nacademic papers. His influential newsletter \"Crypto-Gram\" and his blog \r\n\"Schneier on Security\" are read by over 250,000 people. He has testified \r\nbefore Congress, is a frequent guest on television and radio, has served \r\non several government committees, and is regularly quoted in the press. \r\nSchneier is a fellow at the Berkman Center for Internet and Society at \r\nHarvard Law School, a program fellow at the New America Foundation's \r\nOpen Technology Institute, a board member of the Electronic Frontier \r\nFoundation, an Advisory Board Member of the Electronic Privacy \r\nInformation Center, and the Chief Technology Officer at Resilient, an \r\nIBM Company.  See .\r\n\r\nCrypto-Gram is a personal newsletter. Opinions expressed are not \r\nnecessarily those of Resilient, an IBM Company.\r\n\r\nCopyright (c) 2016 by Bruce Schneier.\r\n\r\n** *** ***** ******* *********** *************\r\n\r\n\r\n\r\n\r\n\r\nTo unsubscribe from Crypto-Gram, click this link:\r\n\r\nhttps://lists.schneier.com/cgi-bin/mailman/options/crypto-gram/christine%40spang.cc?login-unsub=Unsubscribe\r\n\r\nYou will be e-mailed a confirmation message.  Follow the instructions in that message to confirm your removal from the list.\r\n
","snippet":" CRYPTO-GRAM November 15, 2016 by Bruce Schneier CTO, Resilient, an IBM Company schneier@schneier.com","unread":false,"starred":false,"date":"2016-11-15T07:50:26.000Z","folderImapUID":345982,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc","crypto-gram@lists.schneier.com"],"received":["by 10.31.185.141 with SMTP id j135csp15122vkf; Mon, 14 Nov 2016 23:50:26 -0800 (PST)","from schneier.modwest.com (schneier.modwest.com. [204.11.247.92]) by mx.google.com with ESMTPS id i126si6507480ybb.7.2016.11.14.23.50.26 for  (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 14 Nov 2016 23:50:26 -0800 (PST)","from schneier.modwest.com (localhost [127.0.0.1]) by schneier.modwest.com (Postfix) with ESMTP id A57D33A66E for ; Tue, 15 Nov 2016 00:48:53 -0700 (MST)","from webmail.schneier.com (localhost [127.0.0.1]) by schneier.modwest.com (Postfix) with ESMTPA id 735B038F18; Tue, 15 Nov 2016 00:27:10 -0700 (MST)"],"x-received":["by 10.37.220.66 with SMTP id y63mr6697075ybe.190.1479196226438; Mon, 14 Nov 2016 23:50:26 -0800 (PST)"],"return-path":[""],"received-spf":["pass (google.com: domain of crypto-gram-bounces@lists.schneier.com designates 204.11.247.92 as permitted sender) client-ip=204.11.247.92;"],"authentication-results":["mx.google.com; spf=pass (google.com: domain of crypto-gram-bounces@lists.schneier.com designates 204.11.247.92 as permitted sender) smtp.mailfrom=crypto-gram-bounces@lists.schneier.com"],"x-original-to":["crypto-gram@lists.schneier.com"],"mime-version":["1.0"],"date":["Tue, 15 Nov 2016 01:27:10 -0600"],"from":["Bruce Schneier "],"subject":["CRYPTO-GRAM, November 15, 2016"],"message-id":["<76bcad7045e1f498eb00e27fc969ee53@schneier.com>"],"x-sender":["schneier@schneier.com"],"user-agent":["Roundcube Webmail/0.9.5"],"x-mailman-approved-at":["Tue, 15 Nov 2016 00:45:13 -0700"],"x-beenthere":["crypto-gram@lists.schneier.com"],"x-mailman-version":["2.1.15"],"precedence":["list"],"cc":["Crypto-Gram Mailing List "],"list-id":["Crypto-Gram Mailing List "],"list-unsubscribe":[", "],"list-post":[""],"list-help":[""],"list-subscribe":[", "],"content-transfer-encoding":["7bit"],"content-type":["text/plain; charset=\"us-ascii\"; Format=\"flowed\""],"to":["christine@spang.cc"],"errors-to":["crypto-gram-bounces@lists.schneier.com"],"sender":["\"Crypto-Gram\" "],"x-gm-thrid":"1551049662245032910","x-gm-msgid":"1551049662245032910","x-gm-labels":["\\Inbox"]},"headerMessageId":"<76bcad7045e1f498eb00e27fc969ee53@schneier.com>","subject":"CRYPTO-GRAM, November 15, 2016","folderImapXGMLabels":"[\"\\\\Inbox\"]"}}
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/eff-plaintext-no-mime.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/eff-plaintext-no-mime.json
new file mode 100644
index 000000000..2658d492a
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/eff-plaintext-no-mime.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"partID":"1","type":"text","subtype":"plain","params":null,"id":null,"description":null,"encoding":"7BIT","size":3050,"lines":67,"md5":null,"disposition":null,"language":null}],"date":"2016-12-01T01:34:44.000Z","flags":["\\Seen"],"uid":348040,"modseq":"8228548","x-gm-labels":["\\Inbox"],"x-gm-msgid":"1552475576878158784","x-gm-thrid":"1552475576878158784"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.140.100.181 with SMTP id s50csp492441qge; Wed, 30 Nov 2016\r\n 17:34:44 -0800 (PST)\r\nX-Received: by 10.129.120.215 with SMTP id t206mr41825274ywc.39.1480556084610;\r\n Wed, 30 Nov 2016 17:34:44 -0800 (PST)\r\nReturn-Path: \r\nReceived: from mail2.eff.org (mail2.eff.org. [173.239.79.204]) by\r\n mx.google.com with ESMTPS id c198si18454640ywb.136.2016.11.30.17.34.44 for\r\n  (version=TLS1_2 cipher=AES128-SHA bits=128/128); Wed, 30\r\n Nov 2016 17:34:44 -0800 (PST)\r\nReceived-SPF: pass (google.com: best guess record for domain of\r\n www-data@web5.eff.org designates 173.239.79.204 as permitted sender)\r\n client-ip=173.239.79.204;\r\nAuthentication-Results: mx.google.com; dkim=pass header.i=@eff.org; spf=pass\r\n (google.com: best guess record for domain of www-data@web5.eff.org designates\r\n 173.239.79.204 as permitted sender) smtp.mailfrom=www-data@web5.eff.org;\r\n dmarc=pass (p=NONE dis=NONE) header.from=eff.org\r\nDKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=eff.org;\r\n s=mail2; h=Date:Message-Id:Sender:Reply-To:From:Subject:To;\r\n bh=85sXKrmP+EL3X9i986SKLxXpb0v60xG0c09b/uRaP10=;\r\n b=IByfHFGQGSbxCZfsWU5gd3ek92bd4yhEReZ8qDGPo/CWDCeUO3QnB6yY3aMpuJdD9TUUUKM6rcmfpz4zTmCtwkakMW/uIay2CBXUWuAsowRwUtofpIDmn4aDOhkMUHvyMZe9cZhpgWr7EC1JqEI+3J/kvhR/HTgi7r0dVnz7FBk=;\r\nReceived: from static-69.50.232.52.nephosdns.com ([69.50.232.52]:57896\r\n helo=web5.eff.org) by mail2.eff.org with esmtp (Exim 4.80) (envelope-from\r\n ) id 1cCGGo-000366-QC for christine@spang.cc; Wed, 30\r\n Nov 2016 17:34:42 -0800\r\nReceived: by web5.eff.org (Postfix, from userid 33) id BFBE132AD8C; Wed, 30\r\n Nov 2016 17:34:42 -0800 (PST)\r\nTo: christine@spang.cc\r\nSubject: EFF Membership Benefits\r\nFrom: Member Services \r\nReply-To: membership@eff.org\r\nSender: membership@eff.org\r\nMessage-Id: <20161201013442.BFBE132AD8C@web5.eff.org>\r\nDate: Wed, 30 Nov 2016 17:34:42 -0800 (PST)\r\nReceived-SPF: skipped for local relay\r\n\r\n","parts":{"1":"Thank you for being a member of the Electronic Frontier Foundation! Your\r\nEFF membership lasts for 12 months and you have the opportunity to\r\nselect a free gift every time you renew! If you are a monthly donor,\r\nyour membership lasts as long as you like, and you are eligible for a\r\nmember gift every 12 months. Contributions are tax deductible as allowed\r\nby law. EFF is a U.S. 501(c)(3) organization, and our federal tax ID\r\nnumber is 04-3091431.\r\n\r\nYour contribution makes a significant difference in our ability to\r\ndefend your rights in the digital world whether that means standing up\r\nfor users in the courts, educating the public about dangerous laws, or\r\ncreating privacy-enhancing technologies. We are pleased to offer you a\r\nfew modest perks as a token of our thanks. This is a one-time email\r\nnotice and EFF will never share your information with partnering or\r\nsupporting organizations. Contact us at membership@eff.org if you have\r\nany questions!\r\n\r\nEFF MEMBER BENEFITS:\r\n\r\n-EFF Online Rights bumper sticker (via print mail).\r\n\r\n-EFF Member Card (via print mail).\r\n\r\n-Digital Badges: Show your support on your blog, website, or in an HTML\r\nemail signature.\r\nhttps://www.eff.org/files/2016/11/01/2017mb.png\r\n\r\n-Discounts on Events: Receive discounts on General Admission to EFF\r\nevents! Choose the member price when purchasing tickets or show your EFF\r\nMember card at the door. Watch for specific details in event invitations.\r\n\r\n-Speakeasy Invitations: Get notified about informal members-only meetups\r\nand drink ups with EFF lawyers, activists, and technologists in your area!\r\n\r\n-EFF Online Shop Discount: Get 10% off purchases (Gift Memberships\r\nexcluded) at EFF's online store by using the discount code ALIQUID16 at\r\nhttps://www.eff.org/shop.\r\n\r\nPARTNER DISCOUNTS FOR EFF MEMBERS:\r\n\r\n-LeanPub: Get a copy of Uncensored, an ever expanding collection of\r\nessays on Internet freedom from many of its original innovators. Follow\r\nthis promo url: http://leanpub.com/uncensored/c/effdonor.\r\n\r\n-Borderlands Books and Cafe: Borderlands is San Francisco's home for\r\nscience fiction, fantasy, and horror books. Present your EFF Member Card\r\nand get a 10% discount on in-store purchases. Borderlands Cafe also\r\noffers occasional EFF Member specials so be sure to stop by for some\r\nreading time and a treat! http://www.borderlands-books.com\r\n\r\n-Magnatune: Get an $8 gift card from Magnatune, the music download\r\nservice that isn't evil. All the music is DRM free and licensed under\r\nCreative Commons. Musicians get 50%, you get to choose the price you\r\npay, and you're encouraged to share your purchased music with 3 friends.\r\nGo to http://magnatune.com and use the gift code: 111801248707\r\n\r\n-No Starch Press: Get a 30% discount on geektastic books that entertain\r\nand/or instruct by using the code EFFMEMBER at http://www.nostarch.com.\r\n\r\n-Take Control Ebooks: Get a 30% off by following this EFF member portal.\r\nhttp://www.takecontrolbooks.com/?pt=EFF&cp=CPN80131EFF\r\n\r\n-- \r\nAaron Jue\r\nEFF Senior Membership Advocate\r\nmembership@eff.org\r\n\r\n"}},"desiredParts":[{"id":"1","encoding":"7BIT","mimetype":"text/plain"}],"result":{"id":"addb4e92bde89a7898ea31b9bbc866b35eb1b5e150bc69a47ed239b1679a7c1a","to":[{"name":"","email":"christine@spang.cc"}],"cc":[],"bcc":[],"from":[{"name":"Member Services","email":"membership@eff.org"}],"replyTo":[{"name":"","email":"membership@eff.org"}],"accountId":"test-account-id","body":"
Thank you for being a member of the Electronic Frontier Foundation! Your\r\nEFF membership lasts for 12 months and you have the opportunity to\r\nselect a free gift every time you renew! If you are a monthly donor,\r\nyour membership lasts as long as you like, and you are eligible for a\r\nmember gift every 12 months. Contributions are tax deductible as allowed\r\nby law. EFF is a U.S. 501(c)(3) organization, and our federal tax ID\r\nnumber is 04-3091431.\r\n\r\nYour contribution makes a significant difference in our ability to\r\ndefend your rights in the digital world whether that means standing up\r\nfor users in the courts, educating the public about dangerous laws, or\r\ncreating privacy-enhancing technologies. We are pleased to offer you a\r\nfew modest perks as a token of our thanks. This is a one-time email\r\nnotice and EFF will never share your information with partnering or\r\nsupporting organizations. Contact us at membership@eff.org if you have\r\nany questions!\r\n\r\nEFF MEMBER BENEFITS:\r\n\r\n-EFF Online Rights bumper sticker (via print mail).\r\n\r\n-EFF Member Card (via print mail).\r\n\r\n-Digital Badges: Show your support on your blog, website, or in an HTML\r\nemail signature.\r\nhttps://www.eff.org/files/2016/11/01/2017mb.png\r\n\r\n-Discounts on Events: Receive discounts on General Admission to EFF\r\nevents! Choose the member price when purchasing tickets or show your EFF\r\nMember card at the door. Watch for specific details in event invitations.\r\n\r\n-Speakeasy Invitations: Get notified about informal members-only meetups\r\nand drink ups with EFF lawyers, activists, and technologists in your area!\r\n\r\n-EFF Online Shop Discount: Get 10% off purchases (Gift Memberships\r\nexcluded) at EFF's online store by using the discount code ALIQUID16 at\r\nhttps://www.eff.org/shop.\r\n\r\nPARTNER DISCOUNTS FOR EFF MEMBERS:\r\n\r\n-LeanPub: Get a copy of Uncensored, an ever expanding collection of\r\nessays on Internet freedom from many of its original innovators. Follow\r\nthis promo url: http://leanpub.com/uncensored/c/effdonor.\r\n\r\n-Borderlands Books and Cafe: Borderlands is San Francisco's home for\r\nscience fiction, fantasy, and horror books. Present your EFF Member Card\r\nand get a 10% discount on in-store purchases. Borderlands Cafe also\r\noffers occasional EFF Member specials so be sure to stop by for some\r\nreading time and a treat! http://www.borderlands-books.com\r\n\r\n-Magnatune: Get an $8 gift card from Magnatune, the music download\r\nservice that isn't evil. All the music is DRM free and licensed under\r\nCreative Commons. Musicians get 50%, you get to choose the price you\r\npay, and you're encouraged to share your purchased music with 3 friends.\r\nGo to http://magnatune.com and use the gift code: 111801248707\r\n\r\n-No Starch Press: Get a 30% discount on geektastic books that entertain\r\nand/or instruct by using the code EFFMEMBER at http://www.nostarch.com.\r\n\r\n-Take Control Ebooks: Get a 30% off by following this EFF member portal.\r\nhttp://www.takecontrolbooks.com/?pt=EFF爀- \r\nAaron Jue\r\nEFF Senior Membership Advocate\r\nmembership@eff.org\r\n\r\n
","snippet":"Thank you for being a member of the Electronic Frontier Foundation! Your EFF membership lasts for 12","unread":false,"starred":false,"date":"2016-12-01T01:34:44.000Z","folderImapUID":348040,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc"],"received":["by 10.140.100.181 with SMTP id s50csp492441qge; Wed, 30 Nov 2016 17:34:44 -0800 (PST)","from mail2.eff.org (mail2.eff.org. [173.239.79.204]) by mx.google.com with ESMTPS id c198si18454640ywb.136.2016.11.30.17.34.44 for  (version=TLS1_2 cipher=AES128-SHA bits=128/128); Wed, 30 Nov 2016 17:34:44 -0800 (PST)","from static-69.50.232.52.nephosdns.com ([69.50.232.52]:57896 helo=web5.eff.org) by mail2.eff.org with esmtp (Exim 4.80) (envelope-from ) id 1cCGGo-000366-QC for christine@spang.cc; Wed, 30 Nov 2016 17:34:42 -0800","by web5.eff.org (Postfix, from userid 33) id BFBE132AD8C; Wed, 30 Nov 2016 17:34:42 -0800 (PST)"],"x-received":["by 10.129.120.215 with SMTP id t206mr41825274ywc.39.1480556084610; Wed, 30 Nov 2016 17:34:44 -0800 (PST)"],"return-path":[""],"received-spf":["pass (google.com: best guess record for domain of www-data@web5.eff.org designates 173.239.79.204 as permitted sender) client-ip=173.239.79.204;","skipped for local relay"],"authentication-results":["mx.google.com; dkim=pass header.i=@eff.org; spf=pass (google.com: best guess record for domain of www-data@web5.eff.org designates 173.239.79.204 as permitted sender) smtp.mailfrom=www-data@web5.eff.org; dmarc=pass (p=NONE dis=NONE) header.from=eff.org"],"dkim-signature":["v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=eff.org; s=mail2; h=Date:Message-Id:Sender:Reply-To:From:Subject:To; bh=85sXKrmP+EL3X9i986SKLxXpb0v60xG0c09b/uRaP10=; b=IByfHFGQGSbxCZfsWU5gd3ek92bd4yhEReZ8qDGPo/CWDCeUO3QnB6yY3aMpuJdD9TUUUKM6rcmfpz4zTmCtwkakMW/uIay2CBXUWuAsowRwUtofpIDmn4aDOhkMUHvyMZe9cZhpgWr7EC1JqEI+3J/kvhR/HTgi7r0dVnz7FBk=;"],"to":["christine@spang.cc"],"subject":["EFF Membership Benefits"],"from":["Member Services "],"reply-to":["membership@eff.org"],"sender":["membership@eff.org"],"message-id":["<20161201013442.BFBE132AD8C@web5.eff.org>"],"date":["Wed, 30 Nov 2016 17:34:42 -0800 (PST)"],"x-gm-thrid":"1552475576878158784","x-gm-msgid":"1552475576878158784","x-gm-labels":["\\Inbox"]},"headerMessageId":"<20161201013442.BFBE132AD8C@web5.eff.org>","subject":"EFF Membership Benefits","folderImapXGMLabels":"[\"\\\\Inbox\"]"}}
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/hacker-newsletter-multipart-alternative.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/hacker-newsletter-multipart-alternative.json
new file mode 100644
index 000000000..2eec23199
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/hacker-newsletter-multipart-alternative.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"type":"alternative","params":{"boundary":"_----------=_MCPart_776050397"},"disposition":null,"language":null},[{"partID":"1","type":"text","subtype":"plain","params":{"charset":"utf-8","format":"fixed"},"id":null,"description":null,"encoding":"QUOTED-PRINTABLE","size":17854,"lines":362,"md5":null,"disposition":null,"language":null}],[{"partID":"2","type":"text","subtype":"html","params":{"charset":"utf-8"},"id":null,"description":null,"encoding":"QUOTED-PRINTABLE","size":66229,"lines":1172,"md5":null,"disposition":null,"language":null}]],"date":"2016-11-04T13:33:12.000Z","flags":[],"uid":344200,"modseq":"8118301","x-gm-labels":["\\Important"],"x-gm-msgid":"1550074660376052354","x-gm-thrid":"1550074660376052354"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.31.236.3 with SMTP id k3csp257576vkh; Fri, 4 Nov 2016 06:33:12\r\n -0700 (PDT)\r\nX-Received: by 10.55.207.210 with SMTP id v79mr12608992qkl.199.1478266392152;\r\n Fri, 04 Nov 2016 06:33:12 -0700 (PDT)\r\nReturn-Path: \r\nReceived: from mail250.atl61.mcsv.net (mail250.atl61.mcsv.net.\r\n [205.201.135.250]) by mx.google.com with ESMTP id\r\n f11si7594937qte.84.2016.11.04.06.33.10 for ; Fri, 04 Nov\r\n 2016 06:33:12 -0700 (PDT)\r\nReceived-SPF: pass (google.com: domain of\r\n bounce-mc.us1_1832689.2559073-christine=spang.cc@mail250.atl61.mcsv.net\r\n designates 205.201.135.250 as permitted sender) client-ip=205.201.135.250;\r\nAuthentication-Results: mx.google.com; dkim=pass\r\n header.i=@hackernewsletter.com; spf=pass (google.com: domain of\r\n bounce-mc.us1_1832689.2559073-christine=spang.cc@mail250.atl61.mcsv.net\r\n designates 205.201.135.250 as permitted sender)\r\n smtp.mailfrom=bounce-mc.us1_1832689.2559073-Christine=spang.cc@mail250.atl61.mcsv.net\r\nDKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; s=k1;\r\n d=hackernewsletter.com;\r\n h=Subject:From:Reply-To:To:Date:Message-ID:List-ID:List-Unsubscribe:Content-Type:MIME-Version;\r\n i=kale@hackernewsletter.com; bh=vZB3PnpQdFA+LVdXRQIdK2lMkvI=;\r\n b=RuL/qDIhe22oioO4rRO81v3j4nGp4VXHbJILx7VRSkXQ6yFLcyEPISC1+l3WaAEyBeyBU5lTNvvr\r\n 6KdJ0UzAYt854dXg0fefITQUDJmKEZS+wGWsJtvSK380WU3M5V6rQXWodFYBmvZmJXJH0oqh5TMo\r\n Iwel9pTPpzKCVGtG3L4=\r\nReceived: from (127.0.0.1) by mail250.atl61.mcsv.net id h3i71e174acc for\r\n ; Fri, 4 Nov 2016 13:33:01 +0000 (envelope-from\r\n )\r\nSubject: =?utf-8?Q?Hacker=20Newsletter=20#325?=\r\nFrom: =?utf-8?Q?Hacker=20Newsletter?= \r\nReply-To: =?utf-8?Q?Hacker=20Newsletter?= \r\nTo: \r\nDate: Fri, 4 Nov 2016 13:33:01 +0000\r\nMessage-ID: \r\nX-Mailer: MailChimp Mailer - **CIDa018682c80765272fcdd**\r\nX-Campaign: mailchimpfaa8eb4ef3a111cef92c4f3d4.a018682c80\r\nX-campaignid: mailchimpfaa8eb4ef3a111cef92c4f3d4.a018682c80\r\nX-Report-Abuse: Please report abuse for this campaign here:\r\n http://www.mailchimp.com/abuse/abuse.phtml?u=faa8eb4ef3a111cef92c4f3d4&id=a018682c80&e=765272fcdd\r\nX-MC-User: faa8eb4ef3a111cef92c4f3d4\r\nX-Feedback-ID: 1832689:1832689.2559073:us1:mc\r\nList-ID: faa8eb4ef3a111cef92c4f3d4mc list\r\n \r\nX-Accounttype: pr\r\nList-Unsubscribe: ,\r\n \r\nx-mcda: FALSE\r\nContent-Type: multipart/alternative; boundary=\"_----------=_MCPart_776050397\"\r\nMIME-Version: 1.0\r\n\r\n","parts":{"1":"The only thing more expensive than writing software is writing bad softwar=\r\ne. //Alan Cooper\r\n\r\n\r\n** hackernewsletter (http://hackernewsletter.us1.list-manage2.com/track/cl=\r\nick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Db9331dfc08&e=3D765272fcdd)\r\n------------------------------------------------------------\r\nIssue #325 // November 04=2C 2016 // View in your browser (http://us1.camp=\r\naign-archive2.com/?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Da018682c80&e=3D=\r\n765272fcdd)\r\n\r\n\r\n** #Sponsor\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nhttp://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a11=\r\n1cef92c4f3d4&id=3D49346392ee&e=3D765272fcdd Hired - The End of Job H=\r\nunting As You Know It (http://hackernewsletter.us1.list-manage.com/track/c=\r\nlick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dc2bf580f8d&e=3D765272fcdd)\r\nHired brings job offers to you=2C so you can cut to the chase and focus on=\r\n finding the right fit=2C (not applying). Get multiple job offers and upfr=\r\nont compensation information=2C with just one application. Get Hired (http=\r\n://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef9=\r\n2c4f3d4&id=3Dec31429e96&e=3D765272fcdd)\r\n\r\n\r\n** #Favorites\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nEve: Programming designed for humans (http://hackernewsletter.us1.list-man=\r\nage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3De08dd4413c&e=3D=\r\n765272fcdd)\r\n//witheve comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com/t=\r\nrack/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd9b62e9185&e=3D=\r\n765272fcdd)\r\n\r\nTotal Nightmare: USB-C and Thunderbolt 3 (http://hackernewsletter.us1.list=\r\n-manage2.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D0e51dee03d&e=\r\n=3D765272fcdd)\r\n//fosketts comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com/=\r\ntrack/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D64456f4c47&e=3D=\r\n765272fcdd)\r\n\r\nSigns that a startup is focused on stuff that doesn=E2=80=99t matter (http=\r\n://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef9=\r\n2c4f3d4&id=3Daae701c750&e=3D765272fcdd)\r\n//groovehq comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com/=\r\ntrack/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D98188f8267&e=3D=\r\n765272fcdd)\r\n\r\nNew MacBook Pro Is Not a Laptop for Developers Anymore (http://hackernewsl=\r\netter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D9=\r\n79a461b49&e=3D765272fcdd)\r\n//blog comments=E2=86=92 (http://hackernewsletter.us1.list-manage2.com/tra=\r\nck/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd0e2535fa5&e=3D=\r\n765272fcdd)\r\n\r\nWeb fonts=2C boy=2C I don't know (http://hackernewsletter.us1.list-manage.=\r\ncom/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D4c37a6c08d&e=3D=\r\n765272fcdd)\r\n//meowni comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com/tr=\r\nack/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D86f87b9395&e=3D=\r\n765272fcdd)\r\n\r\nNo One Saw Tesla=E2=80=99s Solar Roof Coming (http://hackernewsletter.us1.=\r\nlist-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd506272321=\r\n&e=3D765272fcdd)\r\n//bloomberg comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com=\r\n/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D46898a2195&e=3D=\r\n765272fcdd)\r\n\r\nWays Data Projects Fail (http://hackernewsletter.us1.list-manage.com/track=\r\n/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D155328bf94&e=3D765272fcdd)\r\n//martingoodson comments=E2=86=92 (http://hackernewsletter.us1.list-manage=\r\n=2Ecom/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dcf6b7316d0&e=3D=\r\n765272fcdd)\r\n\r\nA very valuable vulnerability (http://hackernewsletter.us1.list-manage.com=\r\n/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D741d7004e5&e=3D=\r\n765272fcdd)\r\n//daemonology comments=E2=86=92 (http://hackernewsletter.us1.list-manage1.=\r\ncom/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D87d1dd6ee7&e=3D=\r\n765272fcdd)\r\n\r\nMaking what people want isn=E2=80=99t enough=2C you have to share it (http=\r\n://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef=\r\n92c4f3d4&id=3D1b614a6e79&e=3D765272fcdd)\r\n//oldgeekjobs comments=E2=86=92 (http://hackernewsletter.us1.list-manage.c=\r\nom/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D38d624ea5f&e=3D=\r\n765272fcdd)\r\n\r\nHow ReadMe Went from SaaS to On-Premises in Less Than One Week (http://hac=\r\nkernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d=\r\n4&id=3Df4c054b8a4&e=3D765272fcdd)\r\n//stackshare comments=E2=86=92 (http://hackernewsletter.us1.list-manage1.c=\r\nom/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Da04633cd33&e=3D=\r\n765272fcdd)\r\n\r\n\r\n** #Ask HN\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nAny other blind devs interested in working on dev tools for the blind? (ht=\r\ntp://hackernewsletter.us1.list-manage2.com/track/click?u=3Dfaa8eb4ef3a111c=\r\nef92c4f3d4&id=3Da4a9a08427&e=3D765272fcdd)\r\n\r\nHow to make a career working remotely? (http://hackernewsletter.us1.list-m=\r\nanage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D517f7a0c8b&e=3D=\r\n765272fcdd)\r\n\r\n\r\n** #Show HN\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nMusic for Programming (http://hackernewsletter.us1.list-manage.com/track/c=\r\nlick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Df72d025a71&e=3D765272fcdd) /=\r\n/musicforprogramming comments=E2=86=92 (http://hackernewsletter.us1.list-m=\r\nanage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D793628fd74&e=3D=\r\n765272fcdd)\r\n\r\nPowerwall 2 and Integrated Solar (http://hackernewsletter.us1.list-manage.=\r\ncom/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dba1ef6be0e&e=3D=\r\n765272fcdd) //tesla comments=E2=86=92 (http://hackernewsletter.us1.list-mana=\r\nge1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D242de3029e&e=3D=\r\n765272fcdd)\r\n\r\nSonder E-Ink Keyboard (http://hackernewsletter.us1.list-manage2.com/track/=\r\nclick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dca8a503ec9&e=3D765272fcdd)=\r\n //sonderdesign comments=E2=86=92 (http://hackernewsletter.us1.list-manage=\r\n2.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D88ec65b546&e=3D=\r\n765272fcdd)\r\n\r\nMicrosoft Teams=2C the new chat-based workspace in Office 365 (http://hack=\r\nernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4=\r\n&id=3D2a1be32eea&e=3D765272fcdd) //office comments=E2=86=92 (http://=\r\nhackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4=\r\nf3d4&id=3D0414a41b94&e=3D765272fcdd)\r\n\r\nFlow =E2=80=93 Create automated workflows between your favorite apps (http=\r\n://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef9=\r\n2c4f3d4&id=3D93b5aafe0d&e=3D765272fcdd) //microsoft comments=E2=86=\r\n=92 (http://hackernewsletter.us1.list-manage2.com/track/click?u=3Dfaa8eb4e=\r\nf3a111cef92c4f3d4&id=3D9a0e4a6a60&e=3D765272fcdd)\r\n\r\nPortier =E2=80=93 An email-based=2C passwordless authentication service (h=\r\nttp://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a111=\r\ncef92c4f3d4&id=3D1359a45752&e=3D765272fcdd) //github comments=E2=86=\r\n=92 (http://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4e=\r\nf3a111cef92c4f3d4&id=3D29564a0bdd&e=3D765272fcdd)\r\n\r\nNewly Redesigned Boston.gov Just Went Open Source (http://hackernewsletter=\r\n=2Eus1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd8f82=\r\n851ff&e=3D765272fcdd) //routefifty comments=E2=86=92 (http://hackern=\r\newsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&i=\r\nd=3D815b94b7ed&e=3D765272fcdd)\r\n\r\nSodaphonic =E2=80=93 record and edit audio in the browser (http://hackerne=\r\nwsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=\r\n=3Dbab7541aac&e=3D765272fcdd) //sodaphonic comments=E2=86=92 (http:/=\r\n/hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c=\r\n4f3d4&id=3Ded91ce0da4&e=3D765272fcdd)\r\n\r\n\r\n** #Featured\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nDear Matt Mullenweg: An Open Letter from Wix.com=E2=80=99s CEO Avishai Abr=\r\nahami (http://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb=\r\n4ef3a111cef92c4f3d4&id=3D6585f01a53&e=3D765272fcdd) //wix comments=\r\n=E2=86=92 (http://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa=\r\n8eb4ef3a111cef92c4f3d4&id=3D604030a8b7&e=3D765272fcdd)\r\n\r\nOpen Letter to Tim Cook (http://hackernewsletter.us1.list-manage2.com/trac=\r\nk/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D34d547c50d&e=3D765272fcdd=\r\n) //petersphilo comments=E2=86=92 (http://hackernewsletter.us1.list-manage=\r\n1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D08b7caf574&e=3D=\r\n765272fcdd)\r\n\r\nDear Microsoft (http://hackernewsletter.us1.list-manage.com/track/click?u=\r\n=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dec227e8c1f&e=3D765272fcdd) //slack=\r\nhq comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com/track/cl=\r\nick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D9b0a2aaca5&e=3D765272fcdd)\r\n\r\n=E2=9D=97=EF=B8=8FSlack may regret its letter to Microsoft (http://hackern=\r\newsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=\r\n=3D9ddcd67f11&e=3D765272fcdd) //theverge comments=E2=86=92 (http://h=\r\nackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f=\r\n3d4&id=3D338bd7f466&e=3D765272fcdd)\r\n\r\n\r\n** #Code\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nDarling =E2=80=93 MacOS translation layer for Linux (http://hackernewslett=\r\ner.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D094c=\r\n969b13&e=3D765272fcdd) //darlinghq comments=E2=86=92 (http://hackern=\r\newsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&i=\r\nd=3D56990dcc67&e=3D765272fcdd)\r\n\r\nI don't understand Python's Asyncio (http://hackernewsletter.us1.list-mana=\r\nge.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D4a3a08a849&e=3D=\r\n765272fcdd) //pocoo comments=E2=86=92 (http://hackernewsletter.us1.list-m=\r\nanage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D3419a482f1&e=3D=\r\n765272fcdd)\r\n\r\nStep-by-step tutorial to build a modern JavaScript stack from scratch (htt=\r\np://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef=\r\n92c4f3d4&id=3Dea6c97f36d&e=3D765272fcdd) //github comments=E2=86=92=\r\n (http://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a1=\r\n11cef92c4f3d4&id=3D45059d8ac3&e=3D765272fcdd)\r\n\r\nA fork of sudo with Touch ID support (http://hackernewsletter.us1.list-man=\r\nage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dc52784337f&e=3D=\r\n765272fcdd) //github comments=E2=86=92 (http://hackernewsletter.us1.lis=\r\nt-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D00e5c49e67&e=\r\n=3D765272fcdd)\r\n\r\nWriting more legible SQL (http://hackernewsletter.us1.list-manage1.com/tra=\r\nck/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Def47d7c8d3&e=3D=\r\n765272fcdd) //craigkerstiens comments=E2=86=92 (http://hackernewsletter.us1.list-ma=\r\nnage2.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D505c66dcf1&e=3D=\r\n765272fcdd)\r\n\r\nLighthouse =E2=80=93 Auditing and Performance Metrics for Progressive Web=\r\n Apps (http://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb=\r\n4ef3a111cef92c4f3d4&id=3D3705b0e290&e=3D765272fcdd) //github comment=\r\ns=E2=86=92 (http://hackernewsletter.us1.list-manage.com/track/click?u=3Dfa=\r\na8eb4ef3a111cef92c4f3d4&id=3D563534ac47&e=3D765272fcdd)\r\n\r\nBashcached =E2=80=93 memcached built on bash and ncat (http://hackernewsle=\r\ntter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd3=\r\n8c37782d&e=3D765272fcdd) //github comments=E2=86=92 (http://hackerne=\r\nwsletter.us1.list-manage2.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=\r\n=3D759a4124b3&e=3D765272fcdd)\r\n\r\n\r\n** #Design\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nDon=E2=80=99t go to art school (2013) (http://hackernewsletter.us1.list-ma=\r\nnage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D19aaf7ce5b&e=3D=\r\n765272fcdd) //medium comments=E2=86=92 (http://hackernewsletter.us1.lis=\r\nt-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd211d367da&e=\r\n=3D765272fcdd)\r\n\r\nUX Myths (2014) (http://hackernewsletter.us1.list-manage1.com/track/click?=\r\nu=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D6ffa88b963&e=3D765272fcdd) //uxmy=\r\nths comments=E2=86=92 (http://hackernewsletter.us1.list-manage1.com/track/=\r\nclick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D00e93d7214&e=3D765272fcdd)\r\n\r\n\r\n** #Watching\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nThomas Piketty=E2=80=99s Capital in the 21st Century=2C in 20 minutes (htt=\r\np://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef=\r\n92c4f3d4&id=3Dab43dfa0d6&e=3D765272fcdd) //boingboing comments=E2=86=\r\n=92 (http://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef=\r\n3a111cef92c4f3d4&id=3D28fd789f6f&e=3D765272fcdd)\r\n\r\nKeep Ruby Weird Again (http://hackernewsletter.us1.list-manage.com/track/c=\r\nlick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D98946ca1d4&e=3D765272fcdd) /=\r\n/testdouble comments=E2=86=92 (http://hackernewsletter.us1.list-manage.com=\r\n/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D86d3c6925c&e=3D=\r\n765272fcdd)\r\n\r\nJapan=E2=80=99s Disposable Workers: Dumping Ground (http://hackernewslette=\r\nr.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D97347=\r\n9829c&e=3D765272fcdd) //vimeo comments=E2=86=92 (http://hackernewsle=\r\ntter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D3d=\r\nc2036d48&e=3D765272fcdd)\r\n\r\n\r\n** #Working\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\n=E2=9D=97=EF=B8=8FAsk HN: Who is firing? (http://hackernewsletter.us1.list=\r\n-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dd2c73aae27&e=3D=\r\n765272fcdd) //ycombinator\r\n\r\nGuide to Remote Work (http://hackernewsletter.us1.list-manage1.com/track/c=\r\nlick?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D18afc5989a&e=3D765272fcdd) /=\r\n/zapier comments=E2=86=92 (http://hackernewsletter.us1.list-manage1.com/tr=\r\nack/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D0167b4f792&e=3D=\r\n765272fcdd)\r\n\r\nAsk HN: Who is hiring? (http://hackernewsletter.us1.list-manage1.com/track=\r\n/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dc604af7c93&e=3D765272fcdd)=\r\n //ycombinator\r\n\r\nAsk HN: Who wants to be hired? (http://hackernewsletter.us1.list-manage.co=\r\nm/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3De46ca7ced5&e=3D=\r\n765272fcdd) //ycombinator\r\n\r\nAsk HN: Freelancer? Seeking freelancer? (http://hackernewsletter.us1.list-=\r\nmanage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Ddbcca91439&e=3D=\r\n765272fcdd) //ycombinator\r\n\r\n\r\n** #Fun\r\n------------------------------------------------------------\r\n------------------------------------------------------------\r\n\r\nStealth Cell Tower Disguised as Printer (http://hackernewsletter.us1.list-=\r\nmanage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dae17e93e7f&e=3D=\r\n765272fcdd) //julianoliver comments=E2=86=92 (http://hackernewsletter=\r\n=2Eus1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Ddd0291=\r\n23dd&e=3D765272fcdd)\r\n\r\nMuseu de la T=C3=A8cnica (http://hackernewsletter.us1.list-manage2.com/tra=\r\nck/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D093d9205aa&e=3D=\r\n765272fcdd) //twitter comments=E2=86=92 (http://hackernewsletter.us1.list-manage.co=\r\nm/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D39549aa682&e=3D=\r\n765272fcdd)\r\n\r\nBenjamin Button Reviews the New MacBook Pro (http://hackernewsletter.us1.l=\r\nist-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dce263de0f0&=\r\ne=3D765272fcdd) //pinboard comments=E2=86=92 (http://hackernewslette=\r\nr.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Ddb15b=\r\n6b38b&e=3D765272fcdd)\r\n\r\nA Gamer Spent 200 Hours Building an Incredibly Detailed Digital San Franci=\r\nsco (http://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfaa8eb4e=\r\nf3a111cef92c4f3d4&id=3Df6289432f1&e=3D765272fcdd) //citylab comments=\r\n=E2=86=92 (http://hackernewsletter.us1.list-manage1.com/track/click?u=3Dfa=\r\na8eb4ef3a111cef92c4f3d4&id=3D5deab9201e&e=3D765272fcdd)\r\n\r\nWhy html thinks 'chucknorris' is a color (http://hackernewsletter.us1.list=\r\n-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Dba55670176&e=\r\n=3D765272fcdd) //stackoverflow comments=E2=86=92 (http://hackernewsl=\r\netter.us1.list-manage1.com/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3D=\r\ne27a284d73&e=3D765272fcdd)\r\n__END__\r\n\r\nYou're among 38=2C633 others who received this email because you wanted a=\r\n weekly recap of the best articles from Hacker News. Published by Curpress=\r\n (http://hackernewsletter.us1.list-manage.com/track/click?u=3Dfaa8eb4ef3a1=\r\n11cef92c4f3d4&id=3D2620a45995&e=3D765272fcdd) from a smallish metal=\r\n box at PO BOX 2621 Decatur=2C Georgia 30031. Hacker Newsletter is not aff=\r\niliated with Y Combinator in any way.\r\n\r\nYou can update your email (http://hackernewsletter.us1.list-manage.com/pro=\r\nfile?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3De505c88a2e&e=3D765272fcdd) or unsubs=\r\ncribe (http://hackernewsletter.us1.list-manage.com/unsubscribe?u=3Dfaa8eb4=\r\nef3a111cef92c4f3d4&id=3De505c88a2e&e=3D765272fcdd&c=3Da018682c80) .\r\n\r\nNot a subscriber? Subscribe at http://hackernewsletter.us1.list-manage.com=\r\n/track/click?u=3Dfaa8eb4ef3a111cef92c4f3d4&id=3Daba6892cb9&e=3D=\r\n765272fcdd.\r\n\r\nEmail Marketing Powered by MailChimp\r\nhttp://www.mailchimp.com/monkey-rewards/?utm_source=3Dfreemium_newsletter&=\r\nutm_medium=3Demail&utm_campaign=3Dmonkey_rewards&aid=3Dfaa8eb4ef3a111cef92=\r\nc4f3d4&afl=3D1","2":"\r\n\r\n=\r\n\r\n  Hacker Newsletter #325\r\n  \r\n  \r\n  \r\n  \r\n\r\n
\r\n  
\r\n\r\n    \r\n      \r\n        \r\n      \r\n    
\r\n          
\r\n            The only thing more expensive than writing software is writing=\r\n bad software. //Alan Cooper\r\n          
\r\n        
\r\n\r\n    \r\n\r\n      \r\n        \r\n      \r\n\r\n      \r\n        \r\n      \r\n\r\n      \r\n        \r\n      \r\n    
\r\n\r\n          
\r\n            hackernewsletter\r\n          
\r\n        
\r\n          Issue #325 // Nove=\r\nmber 04=2C 2016 // View in your browser\r\n        
\r\n          
#Spons=\r\nor
\r\n\r\n
\r\n          
\r\n            \r\n            Hired=\r\n - The End of Job Hunting As You Know It
Hired brings job offers to=\r\n you=2C so you can cut to the chase and focus on finding the right fit=2C=\r\n (not applying). Get multiple job offers and upfront compensation informat=\r\nion=2C with just one application. Get Hired\r\n          
\r\n          
\r\n\r\n\r\n=09\r\n            
#Fa=\r\nvorites
\r\n\r\n
\r\n\r\n\r\n                
Eve: Programming designed for hum=\r\nans
//witheve comments→
\r\n\r\n\r\n\r\n                
Total Nightmare: USB-C and Thun=\r\nderbolt 3
//fosketts comments→
\r\n\r\n\r\n\r\n                
Signs that a startup is focused=\r\n on stuff that doesn=E2=80=99t matter
//groovehq comments→
\r\n\r\n\r\n\r\n                
New MacBook Pro Is Not a Lapto=\r\np for Developers Anymore
//blog comments→
\r\n\r\n\r\n\r\n                
Web fonts=2C boy=2C I don't kno=\r\nw
//meowni comments→
\r\n\r\n\r\n\r\n                
No One Saw Tesla=E2=80=99s Solar R=\r\noof Coming
//bloomberg comments→
\r\n\r\n\r\n\r\n                
Ways Data Projects Fail
//martingoodson co=\r\nmments→
\r\n\r\n\r\n\r\n                
A very valuable vulnerability//daemonology comments→
\r\n\r\n\r\n\r\n                
Making what people want isn=E2=80=\r\n=99t enough=2C you have to share it
//o=\r\nldgeekjobs comments→
\r\n\r\n\r\n\r\n                
How ReadMe Went from SaaS to On=\r\n-Premises in Less Than One Week
//stack=\r\nshare comments→
\r\n\r\n\r\n\r\n=09\r\n            
#As=\r\nk HN
\r\n\r\n
\r\n\r\n\r\n                
Any other blind devs interested=\r\n in working on dev tools for the blind?
\r\n\r\n\r\n\r\n                
How to make a career working remot=\r\nely?
\r\n\r\n\r\n\r\n=09\r\n            
#Sh=\r\now HN
\r\n\r\n
\r\n\r\n\r\n                
Music for Programming //musicforprogramming comments→
\r\n\r\n\r\n\r\n                
Powerwall 2 and Integrated Solar //tesla comments→
\r\n\r\n\r\n\r\n                
Sonder E-Ink Keyboard //sonderdesign comments→
\r\n\r\n\r\n\r\n                
Microsoft Teams=2C the new chat-based workspace in Of=\r\nfice 365 //office comments→
\r\n\r\n\r\n\r\n                
Flow =E2=80=93 Create automated workflows between yo=\r\nur favorite apps //microsoft comments→
\r\n\r\n\r\n\r\n                
Portier =E2=80=93 An email-based=2C passwordless aut=\r\nhentication service //github comments→
\r\n\r\n\r\n\r\n                
Newly Redesigned Boston.gov Just Went Open Source=\r\n //routefifty c=\r\nomments→
\r\n\r\n\r\n\r\n                
Sodaphonic =E2=80=93 record and edit audio in the brow=\r\nser //sodaphonic comments→
\r\n\r\n\r\n\r\n\r\n\r\n\r\n=09\r\n            
#Fe=\r\natured
\r\n\r\n
\r\n\r\n\r\n                
Dear Matt Mullenweg: An Open Letter from Wix.com=E2=\r\n=80=99s CEO Avishai Abrahami //wix comments→
\r\n\r\n\r\n\r\n                
Open Letter to Tim Cook //<=\r\n/span>petersphilo comments→
\r\n\r\n\r\n\r\n                
Dear Microsoft //sla=\r\nckhq comments→
\r\n\r\n                 
=E2=9D=97=EF=B8=8FSlack may regret its letter to M=\r\nicrosoft //theverge comments→<=\r\n/p>\r\n\r\n\r\n\r\n            
#Co=\r\nde
\r\n\r\n
\r\n\r\n\r\n                
Darling =E2=80=93 MacOS translation layer for Linux //darlinghq =\r\ncomments→
\r\n\r\n\r\n\r\n                
I don't understand Python's Asyncio //pocoo comments→
\r\n\r\n\r\n\r\n                
Step-by-step tutorial to build a modern JavaScript st=\r\nack from scratch //github comments→
\r\n\r\n\r\n\r\n                
A fork of sudo with Touch ID support //github comments→
\r\n\r\n\r\n\r\n                
Writing more legible SQL //<=\r\n/span>craigkerstiens comments→
\r\n\r\n\r\n\r\n                
Lighthouse =E2=80=93 Auditing and Performance Metrics=\r\n for Progressive Web Apps //github comments→
\r\n\r\n\r\n\r\n                
Bashcached =E2=80=93 memcached built on bash and ncat //github co=\r\nmments→
\r\n\r\n\r\n\r\n=09\r\n            
#De=\r\nsign
\r\n\r\n
\r\n\r\n\r\n                
Don=E2=80=99t go to art school (2013) //medium comments→
\r\n\r\n\r\n\r\n                
UX Myths (2014) //uxm=\r\nyths comments→
\r\n\r\n\r\n\r\n=09\r\n            
#Wa=\r\ntching
\r\n\r\n
\r\n\r\n\r\n                
Thomas Piketty=E2=80=99s Capital in the 21st Century=\r\n=2C in 20 minutes //boingboing comments→<=\r\n/a>
\r\n\r\n\r\n\r\n                
Keep Ruby Weird Again //testdouble comments→
\r\n\r\n\r\n\r\n                
Japan=E2=80=99s Disposable Workers: Dumping Ground=\r\n //vimeo commen=\r\nts→
\r\n\r\n\r\n\r\n=09\r\n            
#Wo=\r\nrking
\r\n\r\n
\r\n\r\n                
=E2=9D=97=EF=B8=8FAsk HN: Who is firing? //ycombinator
\r\n\r\n\r\n\r\n                
Guide to Remote Work //zapier comments→
\r\n\r\n\r\n\r\n                
Ask HN: Who is hiring? //ycombinator
\r\n\r\n\r\n\r\n                
Ask HN: Who wants to be hired? //ycombinator
\r\n\r\n\r\n\r\n                
Ask HN: Freelancer? Seeking freelancer? //ycombinator
\r\n\r\n\r\n=09\r\n            
#Fu=\r\nn
\r\n\r\n
\r\n\r\n\r\n                
Stealth Cell Tower Disguised as Printer //julianoliver comments=\r\n→
\r\n\r\n\r\n\r\n                
Museu de la T=C3=A8cnica //<=\r\n/span>twitter comments→
\r\n\r\n\r\n\r\n                
Benjamin Button Reviews the New MacBook Pro //pinboard comments<=\r\nspan style=3D\"color:#ff3300;\">→
\r\n\r\n\r\n\r\n                
A Gamer Spent 200 Hours Building an Incredibly Detail=\r\ned Digital San Francisco =\r\n//citylab comments→
\r\n\r\n\r\n\r\n                
Why html thinks 'chucknorris' is a color //stackoverflow comments<=\r\nspan style=3D\"color:#ff3300;\">→
\r\n\r\n\r\n\r\n=09\r\n\r\n=09\r\n=09
\r\n\r\n    \r\n      \r\n        \r\n      \r\n    
\r\n          
__END__
\r\n\r\n          
\r\n            You're among 38=2C633 others who received this email because y=\r\nou wanted a weekly recap of the best articles from Hacker News. Published=\r\n by Curpress from a smallish meta=\r\nl box at PO BOX 2621 Decatur=2C Georgia 30031. Hacker Newsletter is not af=\r\nfiliated with Y Combinator in any way.\r\n            

\r\n            You can update your email or unsubscribe.\r\n            

\r\n            Not a subscriber? Subscribe at http://hackernewsletter.com.\r\n            

\r\n            3D\"Email=\r\n\r\n          
\r\n        
\r\n  
\r\n\r\n"}},"desiredParts":[{"id":"1","encoding":"QUOTED-PRINTABLE","mimetype":"text/plain"},{"id":"2","encoding":"QUOTED-PRINTABLE","mimetype":"text/html"}],"result":{"id":"646293acc173eb589cb579b5665a97489583726e0f35be93ab3163c62c698051","to":[{"name":"","email":"Christine@spang.cc"}],"cc":[],"bcc":[],"from":[{"name":"Hacker Newsletter","email":"kale@hackernewsletter.com"}],"replyTo":[{"name":"Hacker Newsletter","email":"kale@hackernewsletter.com"}],"accountId":"test-account-id","body":"\r\n\r\n\r\n  Hacker Newsletter #325\r\n  \r\n  \r\n  \r\n  \r\n\r\n
\r\n  
\r\n\r\n    \r\n      \r\n        \r\n      \r\n    
\r\n          
\r\n            The only thing more expensive than writing software is writing bad software. //Alan Cooper\r\n          
\r\n        
\r\n\r\n    \r\n\r\n      \r\n        \r\n      \r\n\r\n      \r\n        \r\n      \r\n\r\n      \r\n        \r\n      \r\n    
\r\n\r\n          
\r\n            hackernewsletter\r\n          
\r\n        
\r\n          Issue #325 // November 04, 2016 // View in your browser\r\n        
\r\n          
#Sponsor
\r\n\r\n
\r\n          
\r\n            \r\n            Hired - The End of Job Hunting As You Know It
Hired brings job offers to you, so you can cut to the chase and focus on finding the right fit, (not applying). Get multiple job offers and upfront compensation information, with just one application. Get Hired\r\n          
\r\n          
\r\n\r\n\r\n\t\r\n            
#Favorites
\r\n\r\n
\r\n\r\n\r\n                
Eve: Programming designed for humans
//witheve comments→
\r\n\r\n\r\n\r\n                
Total Nightmare: USB-C and Thunderbolt 3
//fosketts comments→
\r\n\r\n\r\n\r\n                
Signs that a startup is focused on stuff that doesn’t matter
//groovehq comments→
\r\n\r\n\r\n\r\n                
New MacBook Pro Is Not a Laptop for Developers Anymore
//blog comments→
\r\n\r\n\r\n\r\n                
Web fonts, boy, I don't know
//meowni comments→
\r\n\r\n\r\n\r\n                
No One Saw Tesla’s Solar Roof Coming
//bloomberg comments→
\r\n\r\n\r\n\r\n                
Ways Data Projects Fail
//martingoodson comments→
\r\n\r\n\r\n\r\n                
A very valuable vulnerability
//daemonology comments→
\r\n\r\n\r\n\r\n                
Making what people want isn’t enough, you have to share it
//oldgeekjobs comments→
\r\n\r\n\r\n\r\n                
How ReadMe Went from SaaS to On-Premises in Less Than One Week
//stackshare comments→
\r\n\r\n\r\n\r\n\t\r\n            
#Ask HN
\r\n\r\n
\r\n\r\n\r\n                
Any other blind devs interested in working on dev tools for the blind?
\r\n\r\n\r\n\r\n                
How to make a career working remotely?
\r\n\r\n\r\n\r\n\t\r\n            
#Show HN
\r\n\r\n
\r\n\r\n\r\n                
Music for Programming //musicforprogramming comments→
\r\n\r\n\r\n\r\n                
Powerwall 2 and Integrated Solar //tesla comments→
\r\n\r\n\r\n\r\n                
Sonder E-Ink Keyboard //sonderdesign comments→
\r\n\r\n\r\n\r\n                
Microsoft Teams, the new chat-based workspace in Office 365 //office comments→
\r\n\r\n\r\n\r\n                
Flow – Create automated workflows between your favorite apps //microsoft comments→
\r\n\r\n\r\n\r\n                
Portier – An email-based, passwordless authentication service //github comments→
\r\n\r\n\r\n\r\n                
Newly Redesigned Boston.gov Just Went Open Source //routefifty comments→
\r\n\r\n\r\n\r\n                
Sodaphonic – record and edit audio in the browser //sodaphonic comments→
\r\n\r\n\r\n\r\n\r\n\r\n\r\n\t\r\n            
#Featured
\r\n\r\n
\r\n\r\n\r\n                
Dear Matt Mullenweg: An Open Letter from Wix.com’s CEO Avishai Abrahami //wix comments→
\r\n\r\n\r\n\r\n                
Open Letter to Tim Cook //petersphilo comments→
\r\n\r\n\r\n\r\n                
Dear Microsoft //slackhq comments→
\r\n\r\n                 
❗️Slack may regret its letter to Microsoft //theverge comments→
\r\n\r\n\r\n\r\n            
#Code
\r\n\r\n
\r\n\r\n\r\n                
Darling – MacOS translation layer for Linux //darlinghq comments→
\r\n\r\n\r\n\r\n                
I don't understand Python's Asyncio //pocoo comments→
\r\n\r\n\r\n\r\n                
Step-by-step tutorial to build a modern JavaScript stack from scratch //github comments→
\r\n\r\n\r\n\r\n                
A fork of sudo with Touch ID support //github comments→
\r\n\r\n\r\n\r\n                
Writing more legible SQL //craigkerstiens comments→
\r\n\r\n\r\n\r\n                
Lighthouse – Auditing and Performance Metrics for Progressive Web Apps //github comments→
\r\n\r\n\r\n\r\n                
Bashcached – memcached built on bash and ncat //github comments→
\r\n\r\n\r\n\r\n\t\r\n            
#Design
\r\n\r\n
\r\n\r\n\r\n                
Don’t go to art school (2013) //medium comments→
\r\n\r\n\r\n\r\n                
UX Myths (2014) //uxmyths comments→
\r\n\r\n\r\n\r\n\t\r\n            
#Watching
\r\n\r\n
\r\n\r\n\r\n                
Thomas Piketty’s Capital in the 21st Century, in 20 minutes //boingboing comments→
\r\n\r\n\r\n\r\n                
Keep Ruby Weird Again //testdouble comments→
\r\n\r\n\r\n\r\n                
Japan’s Disposable Workers: Dumping Ground //vimeo comments→
\r\n\r\n\r\n\r\n\t\r\n            
#Working
\r\n\r\n
\r\n\r\n                
❗️Ask HN: Who is firing? //ycombinator
\r\n\r\n\r\n\r\n                
Guide to Remote Work //zapier comments→
\r\n\r\n\r\n\r\n                
Ask HN: Who is hiring? //ycombinator
\r\n\r\n\r\n\r\n                
Ask HN: Who wants to be hired? //ycombinator
\r\n\r\n\r\n\r\n                
Ask HN: Freelancer? Seeking freelancer? //ycombinator
\r\n\r\n\r\n\t\r\n            
#Fun
\r\n\r\n
\r\n\r\n\r\n                
Stealth Cell Tower Disguised as Printer //julianoliver comments→
\r\n\r\n\r\n\r\n                
Museu de la Tècnica //twitter comments→
\r\n\r\n\r\n\r\n                
Benjamin Button Reviews the New MacBook Pro //pinboard comments→
\r\n\r\n\r\n\r\n                
A Gamer Spent 200 Hours Building an Incredibly Detailed Digital San Francisco //citylab comments→
\r\n\r\n\r\n\r\n                
Why html thinks 'chucknorris' is a color //stackoverflow comments→
\r\n\r\n\r\n\r\n\t\r\n\r\n\t\r\n\t
\r\n\r\n    \r\n      \r\n        \r\n      \r\n    
\r\n          
__END__
\r\n\r\n          
\r\n            You're among 38,633 others who received this email because you wanted a weekly recap of the best articles from Hacker News. Published by Curpress from a smallish metal box at PO BOX 2621 Decatur, Georgia 30031. Hacker Newsletter is not affiliated with Y Combinator in any way.\r\n            

\r\n            You can update your email or unsubscribe.\r\n            

\r\n            Not a subscriber? Subscribe at http://hackernewsletter.com.\r\n            

\r\n            \"Email\r\n          
\r\n        
\r\n  
\r\n\r\n","snippet":"The only thing more expensive than writing software is writing bad software. //Alan Cooper ** hackernewsletter","unread":true,"starred":false,"date":"2016-11-04T13:33:12.000Z","folderImapUID":344200,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc"],"received":["by 10.31.236.3 with SMTP id k3csp257576vkh; Fri, 4 Nov 2016 06:33:12 -0700 (PDT)","from mail250.atl61.mcsv.net (mail250.atl61.mcsv.net. [205.201.135.250]) by mx.google.com with ESMTP id f11si7594937qte.84.2016.11.04.06.33.10 for ; Fri, 04 Nov 2016 06:33:12 -0700 (PDT)","from (127.0.0.1) by mail250.atl61.mcsv.net id h3i71e174acc for ; Fri, 4 Nov 2016 13:33:01 +0000 (envelope-from )"],"x-received":["by 10.55.207.210 with SMTP id v79mr12608992qkl.199.1478266392152; Fri, 04 Nov 2016 06:33:12 -0700 (PDT)"],"return-path":[""],"received-spf":["pass (google.com: domain of bounce-mc.us1_1832689.2559073-christine=spang.cc@mail250.atl61.mcsv.net designates 205.201.135.250 as permitted sender) client-ip=205.201.135.250;"],"authentication-results":["mx.google.com; dkim=pass header.i=@hackernewsletter.com; spf=pass (google.com: domain of bounce-mc.us1_1832689.2559073-christine=spang.cc@mail250.atl61.mcsv.net designates 205.201.135.250 as permitted sender) smtp.mailfrom=bounce-mc.us1_1832689.2559073-Christine=spang.cc@mail250.atl61.mcsv.net"],"dkim-signature":["v=1; a=rsa-sha1; c=relaxed/relaxed; s=k1; d=hackernewsletter.com; h=Subject:From:Reply-To:To:Date:Message-ID:List-ID:List-Unsubscribe:Content-Type:MIME-Version; i=kale@hackernewsletter.com; bh=vZB3PnpQdFA+LVdXRQIdK2lMkvI=; b=RuL/qDIhe22oioO4rRO81v3j4nGp4VXHbJILx7VRSkXQ6yFLcyEPISC1+l3WaAEyBeyBU5lTNvvr 6KdJ0UzAYt854dXg0fefITQUDJmKEZS+wGWsJtvSK380WU3M5V6rQXWodFYBmvZmJXJH0oqh5TMo Iwel9pTPpzKCVGtG3L4="],"subject":["Hacker Newsletter #325"],"from":["Hacker Newsletter "],"reply-to":["Hacker Newsletter "],"to":[""],"date":["Fri, 4 Nov 2016 13:33:01 +0000"],"message-id":[""],"x-mailer":["MailChimp Mailer - **CIDa018682c80765272fcdd**"],"x-campaign":["mailchimpfaa8eb4ef3a111cef92c4f3d4.a018682c80"],"x-campaignid":["mailchimpfaa8eb4ef3a111cef92c4f3d4.a018682c80"],"x-report-abuse":["Please report abuse for this campaign here: http://www.mailchimp.com/abuse/abuse.phtml?u=faa8eb4ef3a111cef92c4f3d4&id=a018682c80&e=765272fcdd"],"x-mc-user":["faa8eb4ef3a111cef92c4f3d4"],"x-feedback-id":["1832689:1832689.2559073:us1:mc"],"list-id":["faa8eb4ef3a111cef92c4f3d4mc list "],"x-accounttype":["pr"],"list-unsubscribe":[", "],"x-mcda":["FALSE"],"content-type":["multipart/alternative; boundary=\"_----------=_MCPart_776050397\""],"mime-version":["1.0"],"x-gm-thrid":"1550074660376052354","x-gm-msgid":"1550074660376052354","x-gm-labels":["\\Important"]},"headerMessageId":"","subject":"Hacker Newsletter #325","folderImapXGMLabels":"[\"\\\\Important\"]"}}
\ No newline at end of file
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/mileageplus-mime-html-only.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/mileageplus-mime-html-only.json
new file mode 100644
index 000000000..c10c08391
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/mileageplus-mime-html-only.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"partID":"1","type":"text","subtype":"html","params":{"charset":"UTF-8"},"id":null,"description":null,"encoding":"QUOTED-PRINTABLE","size":63607,"lines":1555,"md5":null,"disposition":null,"language":null}],"date":"2016-11-16T20:22:55.000Z","flags":[],"uid":346225,"modseq":"8197177","x-gm-labels":[],"x-gm-msgid":"1551187601250126809","x-gm-thrid":"1551187601250126809"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.31.185.141 with SMTP id j135csp32042vkf; Wed, 16 Nov 2016\r\n 12:22:55 -0800 (PST)\r\nX-Received: by 10.129.160.81 with SMTP id x78mr4860400ywg.273.1479327775749;\r\n Wed, 16 Nov 2016 12:22:55 -0800 (PST)\r\nReturn-Path: \r\nReceived: from omp.news.united.com (omp.news.united.com. [12.130.136.195]) by\r\n mx.google.com with ESMTP id e80si8263532ywa.331.2016.11.16.12.22.55 for\r\n ; Wed, 16 Nov 2016 12:22:55 -0800 (PST)\r\nReceived-SPF: pass (google.com: domain of united.5765@envfrm.rsys2.com\r\n designates 12.130.136.195 as permitted sender) client-ip=12.130.136.195;\r\nAuthentication-Results: mx.google.com; dkim=pass header.i=@news.united.com;\r\n spf=pass (google.com: domain of united.5765@envfrm.rsys2.com designates\r\n 12.130.136.195 as permitted sender)\r\n smtp.mailfrom=united.5765@envfrm.rsys2.com; dmarc=pass (p=REJECT dis=NONE)\r\n header.from=news.united.com\r\nDKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; s=united;\r\n d=news.united.com;\r\n h=MIME-Version:Content-Type:Content-Transfer-Encoding:Date:To:From:Reply-To:Subject:List-Unsubscribe:Message-ID;\r\n i=MileagePlus_Partner@news.united.com; bh=oiP9wNJbkuGtDmX9JXmAjTpQYe4=;\r\n b=lWhKDlwoeUSLppBUyzjcmkSvlgQys/kL+1R6BJEllHgaawrH/c2sBjY0NAAsZ4GPPUB/rF4h58NO\r\n FHBElr/V0H/k4rkQmSrzudpLfIElGb0WN2etlGZeO0qhMmtNvwmbhw7QO5uZu+x6sKMVutOFxmpa\r\n 6oPStO1uVojaiyQhVTA=\r\nDomainKey-Signature: a=rsa-sha1; c=nofws; q=dns; s=united; d=news.united.com;\r\n b=ZkKrC8ZdJvofP8CEVdEeIkv3UmDibivko/0dxilZkSYfk8sPe/o2YR+zo0VqA9kr1o07ORe3dcxv\r\n Nz0E0TUCcv4YapXs9qxlN8Bm/Zz8PY8D572GBMV0T34PZ6+5v3ai57LtfUBpPy93fjcRTgNHJqQl\r\n HmQTBlZ3wUHv1TBGOqI=;\r\nReceived: by omp.news.united.com id h5j01k161o48 for ;\r\n Wed, 16 Nov 2016 12:22:49 -0800 (envelope-from\r\n )\r\nX-CSA-Complaints: whitelist-complaints@eco.de\r\nReceived: by omp.news.united.com id h5j01i161o4e for ;\r\n Wed, 16 Nov 2016 12:22:48 -0800 (envelope-from\r\n )\r\nX-CSA-Complaints: whitelist-complaints@eco.de\r\nMIME-Version: 1.0\r\nContent-Type: text/html; charset=\"UTF-8\"\r\nContent-Transfer-Encoding: quoted-printable\r\nDate: Wed, 16 Nov 2016 12:22:53 -0800\r\nTo: christine@spang.cc\r\nFrom: \"MileagePlus Explorer Card\" \r\nReply-To: \"MileagePlus Explorer Card\" \r\nSubject: Free checked bag for you and one companion when you use your Card\r\nFeedback-ID: 5765:1192222:oraclersys\r\nList-Unsubscribe: ,\r\n \r\nX-sgxh1: JojpklpgLxkiHgnQJJ\r\nX-rext: 5.interact2.EonqtlMeFYuUCgf5wg26BVPnIWv1eCI0ufmo8-nj-fqdCJCz5Wz8hM\r\nX-cid: united.3098382\r\nMessage-ID: <0.0.AC.822.1D2404732E8AF0C.0@omp.news.united.com>\r\n\r\n","parts":{"1":"\r\n\r\n\r\n\r\nUnited Airlines - United MileagePlus\r\n\r\n\r\n<=21-- LANGUAGE GLOBAL VARIABLE=20\r\nDefault is EN\r\nTo pull language code from field in Unica file use setglobalvars(LANGUAGE,l=\r\nookup(GENERIC05))\r\nTo pull language code from CELL_CODE use setglobalvars(LANGUAGE,cond(eq(loo=\r\nkup(CELL_CODE),ET01),CH,cond(eq(lookup(CELL_CODE),ET02),DE,cond(eq(lookup(C=\r\nELL_CODE),ET03),EN,cond(eq(lookup(CELL_CODE),ET04),ES,cond(eq(lookup(CELL_C=\r\nODE),ET05),FR,cond(eq(lookup(CELL_CODE),ET06),JA,cond(eq(lookup(CELL_CODE),=\r\nET07),PT,cond(eq(lookup(CELL_CODE),ET08),KO,cond(eq(lookup(CELL_CODE),ET09)=\r\n,TCH,EN))))))))),EN)\r\nTo pull language code from Master Contacts List use setglobalvars(LANGUAGE,=\r\nlookup(LANG_CD))\r\n-->\r\n\r\n<=21-- OPT GLOBAL VARIABLE=20\r\nDefault is OPT_TYPE from Unica file\r\nFor MileagePlus ADMN/NONM campaigns use setglobalvars(OPT,MPPR)\r\nFor United ADMN/NONM campaigns use setglobalvars(OPT,UADL)\r\nYou will need to change the From and Reply To fields on the Campaign Dashbo=\r\nard\r\n-->\r\n\r\n<=21-- COUNTRY GLOBAL VARIABLE=20\r\nDefault is COUNTRY_ from the Master Contacts List\r\nFor ADMN campaigns use setglobalvars(COUNTRY,lookup(ISO_CNTRY_CD))\r\nFor NONM campaigns use setglobalvars(COUNTRY,US)\r\nor whichever country is appropriate\r\n-->\r\n\r\n<=21-- MEMBER_LEVEL GLOBAL VARIABLE=20\r\nDefault is ELITE_LEVEL from the Master Contacts List\r\nFor ADMN campaigns use setglobalvars(MEMBER_LEVEL,lookup(MP_PGM_MBRSP_LVL_C=\r\nD))\r\nFor NONM campaigns use setglobalvars(MEMBER_LEVEL,0)\r\nor whichever member level is appropriate\r\n-->\r\n\r\n<=21-- POS GLOBAL VARIABLE=20\r\nThis variable is passed to the POS=3D attribute in the link table\r\n-->\r\n\r\n<=21-- SECOND LANGUAGE GLOBAL VARIABLE=20\r\nUsed by Language Toggle\r\nDefault is EN\r\nTo set second language by CELL_CODE use\r\nsetglobalvars(SCND_LANG,cond(lookup(LANG_TOGGLE_INDICATOR),select(lookup(CE=\r\nLL_CODE),ET01,CH,ET02,DE,ET03,EN,ET04,ES,ET05,FR,ET06,JA,ET07,PT,ET08,KO,ET=\r\n09,TCH),nothing()))\r\n-->\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n  \r\n    \r\n  \r\n
<=21-- Main Container -->\r\n     =20\r\n      \r\n        \r\n          \r\n        \r\n        \r\n          \r\n          \r\n          \r\n          \r\n          \r\n        \r\n        \r\n          \r\n        \r\n      
3D=22=22
=\r\n3D=22=223D=22=22<=21-- margin -->\r\n            \r\n              \r\n                \r\n              \r\n            
3D=\r\n=22=22
\r\n            <=21--/margin -->\r\n           =20\r\n            \r\n              \r\n                \r\n              \r\n            
<=21-- PREHEADER -->\r\n                 =20\r\n                  \r\n                    \r\n                      \r\n                      \r\n                    \r\n                  
<=21-- margin -->\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
3D=22=22
\r\n                        <=21--/margin -->\r\n                       =20\r\n                        
 You and one com=\r\npanion get a free checked bag when you purchase United tickets with your Mi=\r\nleagePlus Explorer Card=2E 
\r\n                       =20\r\n                        <=21-- margin -->\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
3D=22=22
\r\n                        <=21--/margin -->\r\n                       =20\r\n                        
To ensure delivery to your inbox, please add=\r\n MileagePlus_Partner=40news=2Eunited=2Ecom to your address book=\r\n=2E
View in Web browser <=\r\n/div>\r\n                       =20\r\n                        <=21-- margin -->\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
3D=22=22
\r\n                        <=21--/margin -->
\r\n                 =20\r\n                  <=21--/PREHEADER -->
\r\n           =20\r\n            <=21-- UNITED MILEAGEPLUS HEADER -->=20\r\n           =20\r\n            <=21--/UNITED MILEAGEPLUS HEADER -->=20\r\n            <=21-- MODULES GO HERE -->=20\r\n            <=21-- margin -->\r\n            \r\n              \r\n                \r\n              \r\n            
3D=\r\n=22=22
\r\n           =20\r\n            <=21--/margin -->=20\r\n            <=21-- Hero Module -->=20\r\n           =20\r\n\r\n        \r\n          \r\n        \r\n        \r\n          \r\n        \r\n        \r\n          \r\n        \r\n      
<=21-- Begin modules -->\r\n            
 
\r\n\r\n              \r\n                \r\n                \r\n                \r\n              \r\n            
 \r\n                    \r\n                      \r\n                    \r\n                  
 <=\r\n/div>\r\n                        Free checked bag for you and one companion\r\n                      
=\r\n 
 
\r\n            \r\n              \r\n                \r\n                \r\n                \r\n              \r\n            
 \r\n                    \r\n                      \r\n                      \r\n                      \r\n                    \r\n                  
 \r\n                          \r\n                            \r\n                          \r\n                        
&=23160;
3D=22=22=\r\n
\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
&=23160;
\r\n                             =20\r\n                              <=21-- Body Copy=2E -->Your United MileagePlu=\r\ns® Explorer Card provides a free first standard checked bag* for you (t=\r\nhe primary Cardmember) and one companion on United®-operated flights &m=\r\ndash; a savings of up to =24100 per roundtrip=2E\r\n                              
 
\r\n                              As a reminder, to receive this benefit you ne=\r\ned to:\r\n                              
 
\r\n                             =20\r\n                              <=21-- Begin Bulleted List -->\r\n                             =20\r\n                              \r\n                                \r\n                                  \r\n                                  \r\n                                  \r\n                                \r\n                                \r\n                                  \r\n                                  \r\n                                \r\n                              
•Include your MileagePlus number in your reservation\r\n                                    
 
•Purchase your ticket(s) with your Explorer Card
\r\n                             =20\r\n                              <=21-- Ended Bulleted List -->\r\n                             =20\r\n                              
 
\r\n                              \r\n                                \r\n                                  \r\n                                  \r\n                                \r\n                              
\r\n                                      \r\n                                        \r\n                                    
See more details\r\n                                      
=\r\n3D=22=22
 
 
<=21-- End modules -->
 
\r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            
\r\n                    \r\n                      \r\n                      \r\n                      \r\n                      \r\n                    \r\n                  
 
 
\r\n                        3D=22Like=22		 
   
\r\n                        MileagePlus Cards by Chase
\r\n                  \r\n                    \r\n                      \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                
 
   
\r\n                        3D=22MileagePlus\r\n                      		 
3D=22=22
=\r\n3D=22=22
 
\r\n                <=21-- Begin Social Media -->\r\n               =20\r\n\r\n \r\n  \r\n    \r\n    \r\n  \r\n
Stay connected to United\r\n        \r\n          \r\n          \r\n          \r\n          \r\n          \r\n          \r\n          \r\n        \r\n      
 3D=22Mobile=22 =\r\n 3D=22Hub=22 \r\n           3D=22Facebook=22  3D=22Twitter=22 =\r\n 3D=22Instagram=22  3D=22YouTube=22 =\r\n 3D=22=22
\r\n\r\n                <=21-- End Social Media -->\r\n            
 
\r\n            
&n=\r\nbsp;
=20\r\n            <=21-- /Hero Module -->=20\r\n            <=21-- margin -->\r\n           =20\r\n            \r\n              \r\n                \r\n              \r\n            
3D=\r\n=22=22
\r\n           =20\r\n            <=21--/margin -->=20\r\n            <=21-- margin -->\r\n           =20\r\n            \r\n              \r\n                \r\n              \r\n            
3D=\r\n=22=22
\r\n           =20\r\n            <=21--/margin -->=20\r\n            <=21--/MODULES GO HERE -->=20\r\n            <=21-- FOOTER -->=20\r\n            <=21-- margin -->\r\n            \r\n              \r\n                \r\n              \r\n            
3D=\r\n=22=22
\r\n            <=21--/margin -->=20\r\n            <=21-- global links -->\r\n           =20\r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            
<=21-- margin -->\r\n                  \r\n                    \r\n                      \r\n                    \r\n                  
\r\n                  <=21--/margin -->
 united=2Ecom   Partner offers   =\r\nChange email address   Update email preferences   Privacy policy 
<=21-- margin -->\r\n                  \r\n                    \r\n                      \r\n                    \r\n                  
\r\n                  <=21--/margin -->
\r\n           =20\r\n            <=21--/global links -->\r\n           =20\r\n            \r\n              \r\n                \r\n              \r\n            
\r\n                  \r\n                    \r\n                      \r\n                      \r\n                    \r\n                  
3D=22=22\r\n                      <=21-- legal -->=20\r\n                       =20\r\n
Terms and=\r\n conditions:
\r\n*FREE CHECKED BAG: The primary Cardmember and one traveling companion on th=\r\ne same reservation are each eligible to receive their first standard checke=\r\nd bag free; authorized users are only eligible if they are on the same rese=\r\nrvation as the primary Cardmember=2E To receive first standard checked bag =\r\nfree, the primary Cardmember must include their MileagePlus® number in =\r\ntheir reservation and use their MileagePlus Explorer Card to purchase their=\r\n ticket(s)=2E First standard checked bag free is only available on United&r=\r\neg;- and United Express®-operated flights; codeshare partner-operated f=\r\nlights are not eligible=2E Service charges for oversized, overweight and ex=\r\ntra baggage may apply=2E Cardmembers who are already exempt from other chec=\r\nked baggage service charges will not receive an additional free standard ch=\r\necked bag=2E Chase is not responsible for the provision of, or failure to p=\r\nrovide, the stated benefits=2E Please visit united=2Ecom/chasebag for details=2E
\r\n
\r\nMileagePlus terms and conditions:
\r\nMiles accrued, awards, and benefits issued are subject to change and are su=\r\nbject to the rules of the United MileagePlus program, including without lim=\r\nitation the Premier® program (the =22MileagePlus Program=22), which are=\r\n expressly incorporated herein=2E Please allow 6–8 weeks after comple=\r\nted qualifying activity for miles to post to your account=2E United may cha=\r\nnge the MileagePlus Program including, but not limited to, rules, regulatio=\r\nns, travel awards and special offers or terminate the MileagePlus Program a=\r\nt any time and without notice=2E United and its subsidiaries, affiliates an=\r\nd agents are not responsible for any products or services of other particip=\r\nating companies and partners=2E Taxes and fees related to award travel are =\r\nthe responsibility of the member=2E Bonus award miles, award miles and any =\r\nother miles earned through non-flight activity do not count toward qualific=\r\nation for Premier status unless expressly stated otherwise=2E The accumulat=\r\nion of mileage or Premier status or any other status does not entitle membe=\r\nrs to any vested rights with respect to the MileagePlus Program=2E All calc=\r\nulations made in connection with the MileagePlus Program, including without=\r\n limitation with respect to the accumulation of mileage and the satisfactio=\r\nn of the qualification requirements for Premier status, will be made by Uni=\r\nted Airlines and MileagePlus in their discretion and such calculations will=\r\n be considered final=2E Information in this communication that relates to t=\r\nhe MileagePlus Program does not purport to be complete or comprehensive and=\r\n may not include all of the information that a member may believe is import=\r\nant, and is qualified in its entirety by reference to all of the informatio=\r\nn on the united=2Ecom we=\r\nbsite and the MileagePlus Program rules=2E United and MileagePlus are regis=\r\ntered service marks=2E For complete details about the MileagePlus Program, =\r\ngo to united=2Ecom=2E\r\n
\r\nSee=\r\n additional MileagePlus terms and conditions
\r\n
\r\nC000013567 14342 ET06
\r\n
=20\r\n                        <=21--/legal -->=20\r\n                        <=21-- margin -->\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
3D=22=22
\r\n                        <=21--/margin -->=20\r\n                        <=21-- footer -->=20\r\n                       =20\r\n    <=21-- footer -->\r\n
\r\nDon't miss the latest emails; add =40news=2Eunited=2Ecom 
\r\nYou have received this email because it includes important information rega=\r\nrding your MileagePlus Credit Card benefits=2E Your privacy and email prefe=\r\nrences are very important to us=2E If you previously opted not to receive p=\r\nromotional emails from MileagePlus, we will continue to respect your prefer=\r\nences=2E However, we will continue to send you non-promotional, service ema=\r\nils concerning your account, the MileagePlus program or United=2E\r\n
 
\r\nThis email was sent to christine=40spang=2Ecc\r\n
 
\r\nPlease do not reply to this email=2E We cannot accept electronic replies to=\r\n this email address=2E
\r\nEmail mileageplus=40united=2Ecom with any =\r\nquestions about your MileagePlus account or the MileagePlus program=2E 
\r\n
\r\n\r\n                  \r\n                    \r\n                    \r\n  
\r\n                      \r\n                        \r\n                        \r\n                      
 To contact the sender, write t=\r\no: 
\r\n                          United MileagePlus 
\r\n                          900 Grand Plaza Dr=2E
\r\nHouston, TX 77067
\r\n                          
\r\n                          © 2016 United Airlines, Inc=2E All rights re=\r\nserved=2E
\r\n                      \r\n                        \r\n                          \r\n                          \r\n                        
3D=22A
\r\n    <=21--/footer -->=20\r\n                        <=21--/footer -->=20\r\n                        <=21-- margin -->\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
3D=22=22
\r\n                        <=21--/margin -->		3D=22=22
\r\n           =20\r\n            <=21-- margin -->\r\n            \r\n              \r\n                \r\n              \r\n            
3D=\r\n=22=22
\r\n            <=21--/margin -->=20\r\n            <=21--/FOOTER -->		3D=22=22=\r\n3D=22=22
3D=22=22
\r\n     =20\r\n      <=21-- /Main Container -->=20\r\n      <=21-- Retargeting/tracking pixels -->=20\r\n      =20\r\n      <=21-- /Retargeting/tracking pixels -->

\r\n\r\n\r\n\r\n\r\n
\r\n
\r\n
<=21--Start BK pixel--><=21--End BK pixel-->\r\n
\r\n
\r\n\r\n\r\n\r\n\r\n"}},"desiredParts":[{"id":"1","encoding":"QUOTED-PRINTABLE","mimetype":"text/html"}],"result":{"id":"9e0b4aef2d6c4d4510652819bbd700469273653b3a01f70c35677e2c549d01c3","to":[{"name":"","email":"christine@spang.cc"}],"cc":[],"bcc":[],"from":[{"name":"MileagePlus Explorer Card","email":"MileagePlus_Partner@news.united.com"}],"replyTo":[{"name":"MileagePlus Explorer Card","email":"MileagePlus_NoReply@united.com"}],"accountId":"test-account-id","body":"\r\n\r\n\r\n\r\nUnited Airlines - United MileagePlus\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n  \r\n    \r\n  \r\n
\r\n      \r\n      \r\n        \r\n          \r\n        \r\n        \r\n          \r\n          \r\n          \r\n          \r\n          \r\n        \r\n        \r\n          \r\n        \r\n      
\"\"
\"\"\"\"\r\n            \r\n              \r\n                \r\n              \r\n            
\"\"
\r\n            \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\r\n                  \r\n                  \r\n                    \r\n                      \r\n                      \r\n                    \r\n                  
\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
\"\"
\r\n                        \r\n                        \r\n                        
 You and one companion get a free checked bag when you purchase United tickets with your MileagePlus Explorer Card. 
\r\n                        \r\n                        \r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
\"\"
\r\n                        \r\n                        \r\n                        
To ensure delivery to your inbox, please add MileagePlus_Partner@news.united.com to your address book.
\r\n                        \r\n                        \r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
\"\"
\r\n                        
\r\n                  \r\n                  
\r\n            \r\n             \r\n            \r\n             \r\n             \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\"\"
\r\n            \r\n             \r\n             \r\n            \r\n\r\n        \r\n          \r\n        \r\n        \r\n          \r\n        \r\n        \r\n          \r\n        \r\n      
\r\n            
 
\r\n\r\n              \r\n                \r\n                \r\n                \r\n              \r\n            
 \r\n                    \r\n                      \r\n                    \r\n                  
 
\r\n                        Free checked bag for you and one companion\r\n                      
 
 
\r\n            \r\n              \r\n                \r\n                \r\n                \r\n              \r\n            
 \r\n                    \r\n                      \r\n                      \r\n                      \r\n                    \r\n                  
 \r\n                          \r\n                            \r\n                          \r\n                        
 
\"\"
\r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
 
\r\n                              \r\n                              Your United MileagePlus® Explorer Card provides a free first standard checked bag* for you (the primary Cardmember) and one companion on United®-operated flights — a savings of up to $100 per roundtrip.\r\n                              
 
\r\n                              As a reminder, to receive this benefit you need to:\r\n                              
 
\r\n                              \r\n                              \r\n                              \r\n                              \r\n                                \r\n                                  \r\n                                  \r\n                                  \r\n                                \r\n                                \r\n                                  \r\n                                  \r\n                                \r\n                              
\"\"•Include your MileagePlus number in your reservation\r\n                                    
 
•Purchase your ticket(s) with your Explorer Card
\r\n                              \r\n                              \r\n                              \r\n                              
 
\r\n                              \r\n                                \r\n                                  \r\n                                  \r\n                                \r\n                              
\r\n                                      \r\n                                        \r\n                                      \r\n                                    
See more details
\"\"
 
 
 
\r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            
\r\n                    \r\n                      \r\n                      \r\n                      \r\n                      \r\n                    \r\n                  
 
 
\r\n                        \"Like\"		 
   
\r\n                        MileagePlus Cards by Chase
\r\n                  \r\n                    \r\n                      \r\n                      \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                
 
   
\r\n                        \"MileagePlus		 
\"\"
\"\"
\"\"
 
\r\n                \r\n                \r\n\r\n \r\n  \r\n    \r\n    \r\n  \r\n
Stay connected to United\r\n        \r\n          \r\n          \r\n          \r\n          \r\n          \r\n          \r\n          \r\n          \r\n        \r\n      
 \"Mobile\"  \"Hub\"  \"Facebook\"  \"Twitter\"  \"Instagram\"  \"YouTube\"  \"LinkedIn\"\"\"
\r\n\r\n                \r\n            
 
\r\n            
 
 \r\n             \r\n            \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\"\"
\r\n            \r\n             \r\n            \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\"\"
\r\n            \r\n             \r\n             \r\n             \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\"\"
\r\n             \r\n            \r\n            \r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            
\r\n                  \r\n                    \r\n                      \r\n                    \r\n                  
\"\"
\r\n                  
 united.com   Partner offers   Change email address   Update email preferences   Privacy policy 
\r\n                  \r\n                    \r\n                      \r\n                    \r\n                  
\"\"
\r\n                  
\r\n            \r\n            \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\r\n                  \r\n                    \r\n                      \r\n                      \r\n                      \r\n                    \r\n                  
\"\" \r\n                        \r\n
Terms and conditions:
\r\n*FREE CHECKED BAG: The primary Cardmember and one traveling companion on the same reservation are each eligible to receive their first standard checked bag free; authorized users are only eligible if they are on the same reservation as the primary Cardmember. To receive first standard checked bag free, the primary Cardmember must include their MileagePlus® number in their reservation and use their MileagePlus Explorer Card to purchase their ticket(s). First standard checked bag free is only available on United®- and United Express®-operated flights; codeshare partner-operated flights are not eligible. Service charges for oversized, overweight and extra baggage may apply. Cardmembers who are already exempt from other checked baggage service charges will not receive an additional free standard checked bag. Chase is not responsible for the provision of, or failure to provide, the stated benefits. Please visit united.com/chasebag for details.
\r\n
\r\nMileagePlus terms and conditions:
\r\nMiles accrued, awards, and benefits issued are subject to change and are subject to the rules of the United MileagePlus program, including without limitation the Premier® program (the \"MileagePlus Program\"), which are expressly incorporated herein. Please allow 6–8 weeks after completed qualifying activity for miles to post to your account. United may change the MileagePlus Program including, but not limited to, rules, regulations, travel awards and special offers or terminate the MileagePlus Program at any time and without notice. United and its subsidiaries, affiliates and agents are not responsible for any products or services of other participating companies and partners. Taxes and fees related to award travel are the responsibility of the member. Bonus award miles, award miles and any other miles earned through non-flight activity do not count toward qualification for Premier status unless expressly stated otherwise. The accumulation of mileage or Premier status or any other status does not entitle members to any vested rights with respect to the MileagePlus Program. All calculations made in connection with the MileagePlus Program, including without limitation with respect to the accumulation of mileage and the satisfaction of the qualification requirements for Premier status, will be made by United Airlines and MileagePlus in their discretion and such calculations will be considered final. Information in this communication that relates to the MileagePlus Program does not purport to be complete or comprehensive and may not include all of the information that a member may believe is important, and is qualified in its entirety by reference to all of the information on the united.com website and the MileagePlus Program rules. United and MileagePlus are registered service marks. For complete details about the MileagePlus Program, go to united.com.
\r\n
\r\nSee additional MileagePlus terms and conditions
\r\n
\r\nC000013567 14342 ET06
\r\n
 \r\n                         \r\n                        \r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
\"\"
\r\n                         \r\n                         \r\n                        \r\n    \r\n
\r\nDon't miss the latest emails; add @news.united.com 
\r\nYou have received this email because it includes important information regarding your MileagePlus Credit Card benefits. Your privacy and email preferences are very important to us. If you previously opted not to receive promotional emails from MileagePlus, we will continue to respect your preferences. However, we will continue to send you non-promotional, service emails concerning your account, the MileagePlus program or United.\r\n
 
\r\nThis email was sent to christine@spang.cc\r\n
 
\r\nPlease do not reply to this email. We cannot accept electronic replies to this email address.
\r\nEmail mileageplus@united.com with any questions about your MileagePlus account or the MileagePlus program. 
\r\n
\r\n\r\n                  \r\n                    \r\n                    \r\n  
\r\n                      \r\n                        \r\n                        \r\n                      
 To contact the sender, write to: 
\r\n                          United MileagePlus 
\r\n                          900 Grand Plaza Dr.
\r\nHouston, TX 77067
\r\n                          
\r\n                          © 2016 United Airlines, Inc. All rights reserved.
\r\n                      \r\n                        \r\n                          \r\n                          \r\n                        
\"A
\r\n     \r\n                         \r\n                        \r\n                        \r\n                          \r\n                            \r\n                          \r\n                        
\"\"
\r\n                        		\"\"
\r\n            \r\n            \r\n            \r\n              \r\n                \r\n              \r\n            
\"\"
\r\n             \r\n            		\"\"\"\"
\"\"
\r\n      \r\n       \r\n       \r\n       \r\n      

\r\n\r\n\r\n\r\n\r\n
\r\n
\r\n
\r\n
\r\n
\r\n\r\n\r\n\r\n\r\n","snippet":"United Airlines - United MileagePlus\r\n\r\nhtml {\r\n\t-webkit-text-size-adjust: none;\r\n}\r\n.ReadMsgBody {\r","unread":true,"starred":false,"date":"2016-11-16T20:22:55.000Z","folderImapUID":346225,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc"],"received":["by 10.31.185.141 with SMTP id j135csp32042vkf; Wed, 16 Nov 2016 12:22:55 -0800 (PST)","from omp.news.united.com (omp.news.united.com. [12.130.136.195]) by mx.google.com with ESMTP id e80si8263532ywa.331.2016.11.16.12.22.55 for ; Wed, 16 Nov 2016 12:22:55 -0800 (PST)","by omp.news.united.com id h5j01k161o48 for ; Wed, 16 Nov 2016 12:22:49 -0800 (envelope-from )","by omp.news.united.com id h5j01i161o4e for ; Wed, 16 Nov 2016 12:22:48 -0800 (envelope-from )"],"x-received":["by 10.129.160.81 with SMTP id x78mr4860400ywg.273.1479327775749; Wed, 16 Nov 2016 12:22:55 -0800 (PST)"],"return-path":[""],"received-spf":["pass (google.com: domain of united.5765@envfrm.rsys2.com designates 12.130.136.195 as permitted sender) client-ip=12.130.136.195;"],"authentication-results":["mx.google.com; dkim=pass header.i=@news.united.com; spf=pass (google.com: domain of united.5765@envfrm.rsys2.com designates 12.130.136.195 as permitted sender) smtp.mailfrom=united.5765@envfrm.rsys2.com; dmarc=pass (p=REJECT dis=NONE) header.from=news.united.com"],"dkim-signature":["v=1; a=rsa-sha1; c=relaxed/relaxed; s=united; d=news.united.com; h=MIME-Version:Content-Type:Content-Transfer-Encoding:Date:To:From:Reply-To:Subject:List-Unsubscribe:Message-ID; i=MileagePlus_Partner@news.united.com; bh=oiP9wNJbkuGtDmX9JXmAjTpQYe4=; b=lWhKDlwoeUSLppBUyzjcmkSvlgQys/kL+1R6BJEllHgaawrH/c2sBjY0NAAsZ4GPPUB/rF4h58NO FHBElr/V0H/k4rkQmSrzudpLfIElGb0WN2etlGZeO0qhMmtNvwmbhw7QO5uZu+x6sKMVutOFxmpa 6oPStO1uVojaiyQhVTA="],"domainkey-signature":["a=rsa-sha1; c=nofws; q=dns; s=united; d=news.united.com; b=ZkKrC8ZdJvofP8CEVdEeIkv3UmDibivko/0dxilZkSYfk8sPe/o2YR+zo0VqA9kr1o07ORe3dcxv Nz0E0TUCcv4YapXs9qxlN8Bm/Zz8PY8D572GBMV0T34PZ6+5v3ai57LtfUBpPy93fjcRTgNHJqQl HmQTBlZ3wUHv1TBGOqI=;"],"x-csa-complaints":["whitelist-complaints@eco.de","whitelist-complaints@eco.de"],"mime-version":["1.0"],"content-type":["text/html; charset=\"UTF-8\""],"content-transfer-encoding":["quoted-printable"],"date":["Wed, 16 Nov 2016 12:22:53 -0800"],"to":["christine@spang.cc"],"from":["\"MileagePlus Explorer Card\" "],"reply-to":["\"MileagePlus Explorer Card\" "],"subject":["Free checked bag for you and one companion when you use your Card"],"feedback-id":["5765:1192222:oraclersys"],"list-unsubscribe":[", "],"x-sgxh1":["JojpklpgLxkiHgnQJJ"],"x-rext":["5.interact2.EonqtlMeFYuUCgf5wg26BVPnIWv1eCI0ufmo8-nj-fqdCJCz5Wz8hM"],"x-cid":["united.3098382"],"message-id":["<0.0.AC.822.1D2404732E8AF0C.0@omp.news.united.com>"],"x-gm-thrid":"1551187601250126809","x-gm-msgid":"1551187601250126809","x-gm-labels":[]},"headerMessageId":"<0.0.AC.822.1D2404732E8AF0C.0@omp.news.united.com>","subject":"Free checked bag for you and one companion when you use your Card","folderImapXGMLabels":"[]"}}
\ No newline at end of file
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/node-streamtest-windows-1252.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/node-streamtest-windows-1252.json
new file mode 100644
index 000000000..057d9d39b
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/node-streamtest-windows-1252.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"partID":"1","type":"text","subtype":"plain","params":{"charset":"windows-1252"},"id":null,"description":null,"encoding":"7BIT","size":467,"lines":14,"md5":null,"disposition":null,"language":null}],"date":"2016-12-05T18:18:35.000Z","flags":["\\Seen"],"uid":348641,"modseq":"8252381","x-gm-labels":["debiandevel"],"x-gm-msgid":"1552901121651491080","x-gm-thrid":"1552901121651491080"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.140.100.181 with SMTP id s50csp1618416qge; Mon, 5 Dec 2016\r\n 10:18:35 -0800 (PST)\r\nX-Received: by 10.194.138.111 with SMTP id qp15mr33288574wjb.3.1480961915687;\r\n Mon, 05 Dec 2016 10:18:35 -0800 (PST)\r\nReturn-Path: \r\nReceived: from dmz-mailsec-scanner-7.mit.edu (dmz-mailsec-scanner-7.mit.edu.\r\n [18.7.68.36]) by mx.google.com with ESMTPS id\r\n d81si1055491wmc.164.2016.12.05.10.18.35 for \r\n (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 05 Dec\r\n 2016 10:18:35 -0800 (PST)\r\nReceived-SPF: neutral (google.com: 18.7.68.36 is neither permitted nor denied\r\n by manual fallback record for domain of\r\n bounce-debian-devel=spang=mit.edu@lists.debian.org) client-ip=18.7.68.36;\r\nAuthentication-Results: mx.google.com; dkim=fail header.i=@disroot.org;\r\n dkim=fail header.i=@disroot.org; spf=neutral (google.com: 18.7.68.36 is\r\n neither permitted nor denied by manual fallback record for domain of\r\n bounce-debian-devel=spang=mit.edu@lists.debian.org)\r\n smtp.mailfrom=bounce-debian-devel=spang=mit.edu@lists.debian.org\r\nReceived: from mailhub-dmz-2.mit.edu ( [18.7.62.37]) (using TLS with cipher\r\n DHE-RSA-AES256-SHA (256/256 bits)) (Client did not present a certificate) by \r\n (Symantec Messaging Gateway) with SMTP id 4D.F7.26209.97FA5485; Mon,  5 Dec\r\n 2016 13:18:33 -0500 (EST)\r\nReceived: from dmz-mailsec-scanner-5.mit.edu (dmz-mailsec-scanner-5.mit.edu\r\n [18.7.68.34]) by mailhub-dmz-2.mit.edu (8.13.8/8.9.2) with ESMTP id\r\n uB5IHhdj021246 for ; Mon, 5 Dec 2016 13:18:33 -0500\r\nX-AuditID: 12074424-9b7ff70000006661-5a-5845af79d415\r\nReceived: from bendel.debian.org (bendel.debian.org [82.195.75.100]) (using\r\n TLS with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not\r\n present a certificate) by  (Symantec Messaging Gateway) with SMTP id\r\n D2.B7.11606.87FA5485; Mon,  5 Dec 2016 13:18:32 -0500 (EST)\r\nReceived: from localhost (localhost [127.0.0.1]) by bendel.debian.org\r\n (Postfix) with QMQP id 75C2B17F; Mon,  5 Dec 2016 18:18:21 +0000 (UTC)\r\nX-Mailbox-Line: From debian-devel-request@lists.debian.org  Mon Dec  5\r\n 18:18:21 2016\r\nOld-Return-Path: \r\nX-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on bendel.debian.org\r\nX-Spam-Level: \r\nX-Spam-Status: No, score=-8.0 required=4.0 tests=DKIM_SIGNED,\r\n HEADER_FROM_DIFFERENT_DOMAINS,LDO_WHITELIST,RP_MATCHES_RCVD,T_DKIM_INVALID\r\n autolearn=unavailable autolearn_force=no version=3.4.0\r\nX-Original-To: lists-debian-devel@bendel.debian.org\r\nDelivered-To: lists-debian-devel@bendel.debian.org\r\nReceived: from localhost (localhost [127.0.0.1]) by bendel.debian.org\r\n (Postfix) with ESMTP id 9C7A7151 for ;\r\n Mon,  5 Dec 2016 18:18:12 +0000 (UTC)\r\nX-Virus-Scanned: at lists.debian.org with policy bank en-ht\r\nX-Amavis-Spam-Status: No, score=-7.2 tagged_above=-10000 required=5.3\r\n tests=[BAYES_00=-2, DKIM_SIGNED=0.1, HEADER_FROM_DIFFERENT_DOMAINS=0.001,\r\n LDO_WHITELIST=-5, RP_MATCHES_RCVD=-0.311, T_DKIM_INVALID=0.01] autolearn=ham\r\n autolearn_force=no\r\nReceived: from bendel.debian.org ([127.0.0.1]) by localhost (lists.debian.org\r\n [127.0.0.1]) (amavisd-new, port 2525) with ESMTP id XjRiqZkpBpLJ for\r\n ; Mon,  5 Dec 2016 18:18:07 +0000 (UTC)\r\nReceived: from buxtehude.debian.org (buxtehude.debian.org\r\n [IPv6:2607:f8f0:614:1::1274:39]) (using TLSv1.2 with cipher\r\n ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (Client CN\r\n \"buxtehude.debian.org\", Issuer \"Debian SMTP CA\" (not verified)) by\r\n bendel.debian.org (Postfix) with ESMTPS id 82D7D170; Mon,  5 Dec 2016\r\n 18:18:07 +0000 (UTC)\r\nReceived: from debbugs by buxtehude.debian.org with local (Exim 4.84_2)\r\n (envelope-from ) id 1cDxq0-0004I6-E6; Mon, 05\r\n Dec 2016 18:18:04 +0000\r\nX-Loop: owner@bugs.debian.org\r\nSubject: Bug#847116: ITP: node-streamtest -- set of utils to test your stream\r\n based modules accross various, stream implementations of NodeJS\r\nReply-To: Sruthi Chandran , 847116@bugs.debian.org\r\nResent-From: Sruthi Chandran \r\nResent-To: debian-bugs-dist@lists.debian.org\r\nResent-CC: debian-devel@lists.debian.org, wnpp@debian.org\r\nX-Loop: owner@bugs.debian.org\r\nResent-Date: Mon, 05 Dec 2016 18:18:02 +0000\r\nResent-Message-ID: \r\nX-Debian-PR-Message: report 847116\r\nX-Debian-PR-Package: wnpp\r\nX-Debian-PR-Keywords: \r\nReceived: via spool by submit@bugs.debian.org id=B.148096168015430 (code B);\r\n Mon, 05 Dec 2016 18:18:02 +0000\r\nReceived: (at submit) by bugs.debian.org; 5 Dec 2016 18:14:40 +0000\r\nReceived: from bs-one.disroot.org ([178.21.23.139] helo=disroot.org) by\r\n buxtehude.debian.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256)\r\n (Exim 4.84_2) (envelope-from ) id 1cDxmi-00040c-Gx for\r\n submit@bugs.debian.org; Mon, 05 Dec 2016 18:14:40 +0000\r\nReceived: from localhost (localhost [127.0.0.1]) by disroot.org (Postfix) with\r\n ESMTP id 53193239F6 for ; Mon,  5 Dec 2016 19:14:37\r\n +0100 (CET)\r\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=disroot.org; s=mail;\r\n t=1480961677; bh=D83w+mcR33/VQkITQI3T+hYFSLlGEhXR0gNcNOKma1M=;\r\n h=From:Subject:To:Date:From;\r\n b=RGfjrzldXfrn98cwJECU13CLtE6NurUcFar/ymwcIOWcI/iurS1yB77ufr6lvSnIv\r\n l7GcLyXyiXlFHJI0gacV3jsigJG3NWGyxOT670K+8/00FSUPPI0oT7GgSQQKrZ0RnX\r\n 1ydoD4ZCpm148HmxuCk972HdCYIRlA1YO4ggm35U=\r\nReceived: from disroot.org ([127.0.0.1]) by localhost (mail01.disroot.lan\r\n [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id z-Tz-PcP0cLK for\r\n ; Mon,  5 Dec 2016 19:14:37 +0100 (CET)\r\nFrom: Sruthi Chandran \r\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=disroot.org; s=mail;\r\n t=1480961677; bh=D83w+mcR33/VQkITQI3T+hYFSLlGEhXR0gNcNOKma1M=;\r\n h=From:Subject:To:Date:From;\r\n b=RGfjrzldXfrn98cwJECU13CLtE6NurUcFar/ymwcIOWcI/iurS1yB77ufr6lvSnIv\r\n l7GcLyXyiXlFHJI0gacV3jsigJG3NWGyxOT670K+8/00FSUPPI0oT7GgSQQKrZ0RnX\r\n 1ydoD4ZCpm148HmxuCk972HdCYIRlA1YO4ggm35U=\r\nTo: submit@bugs.debian.org\r\nMessage-ID: <2bee7571-9098-d8d5-b4a4-59fc369c579b@disroot.org>\r\nDate: Mon, 5 Dec 2016 23:44:30 +0530\r\nUser-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101\r\n Icedove/45.5.0\r\nMIME-Version: 1.0\r\nContent-Type: text/plain; charset=windows-1252\r\nContent-Transfer-Encoding: 7bit\r\nDelivered-To: submit@bugs.debian.org\r\nX-Rc-Virus: 2007-09-13_01\r\nX-Rc-Spam: 2008-11-04_01\r\nX-Mailing-List:  archive/latest/323341\r\nX-Loop: debian-devel@lists.debian.org\r\nList-Id: \r\nList-URL: \r\nList-Post: \r\nList-Help: \r\nList-Subscribe: \r\nList-Unsubscribe: \r\nPrecedence: list\r\nResent-Sender: debian-devel-request@lists.debian.org\r\nList-Archive: https://lists.debian.org/msgid-search/2bee7571-9098-d8d5-b4a4-59fc369c579b@disroot.org\r\nAuthentication-Results: symauth.service.identifier\r\nX-Brightmail-Tracker: H4sIAAAAAAAAA1VTa0gUYRT1mxl1fEyNY7VXyx5DEURqpVBmSJD2wH5kBUkEObmTu7S72sz6\r\n WElaf2zFqmBhWWqskYWKqAmmUUm6lI8eUJm9LItURFFXsswMacZvTfp37jnnnnvv8A1Nck1e\r\n wbTeZBYlk2DgvXwpzjtmXailLi5x043La7aNl/zx3on2zkzcIQ6go747tKJBnyFK4TFJvrp7\r\n 43eptNdk1k9nEbKiAcKOfGhgI6G85jFSMcfaCLBdSrAjXwX3IDjf30vZET1nsk2RmC8jYMTR\r\n TeGiEMG7bocXTjoMb10vSBUvZsOhuc2JML8f7jfOeGO8CF4O/SLUZmCdCIodTyksRECl4/tc\r\n A8Vugca+fveEBwhanffcE6KhobcS4e4pBC03O91LDSKwP5xEuHhFQI/tk1upQzDYOeuJzwiB\r\n Z++zVT6QLUCQ13OfwHfHwpPuSVL1cMrmfVc5TG8Ex/QYwnQU9BTk/JeCHTWfbpEYx8Hg6xH3\r\n ojxUFz5wf+AgeDVR4z5/GTRZq7ywPxNy2y9ShSimRCkZNgA6r/VTKibZMGh++9sT41XQNFpG\r\n liOyGoVojdmhRkFvkMXkUDlZMJlEKXRrmFFvDhO16Q1o7jXErm1GefnxbYilEe/PuKxxiZyn\r\n kCFbjG0oiCb4pUxRkkItOpGqtegEWXdcSjeIchsCmuSXMEuVh8QxWsGSLUqp89JymuI1jMee\r\n H0c4NkUwi6dEMU2U5tUVNM0D86FWaQyQxBQx66TeYF6QCdpHDfdXwqtUDyOnCUZZn4L1LhRB\r\n f2ivGCDoR+OVAwRHmVJNYrCGKVWtrGrVpZv+pc2/92GkUY4LZGJUl7/yN/zLG1ZGEcqojvZd\r\n 6iizsCAFWxEzuy/rVKV5JnB17MkE576RrMHTZ2s/5yTH11svFOVmTDVm+rWPUd8Otvh/PRNn\r\n 01QMdD+iXYbtLYeKg/3q9Xm9rvXr5I/OjQ3TUXe4/D6f/mTNqpWJqWu3PB86FvLmXPSViTF+\r\n d+vN8N4jAV9Hb0sdI9c7n33pivSwR0SW0pZalx9PyTph8wZSkoW/az3I+MoDAAA=\r\nX-Brightmail-Tracker: H4sIAAAAAAAAA1VSa0wUVxTeMzMsw3YHh0HhuHWjmRATbRcfaNJXWhLEmrZGQzTapo2O7shO\r\n 3AeZWZ4NLaWxJRjCWm2sBLIkikF8JSiyVMS6GwWU1ogEfCGJAkGxaKMitYF0hrvY+ufmO+f7\r\n zvm+e3NZWrhntrFygV9WvZJbNFuYrMinTkfByczNS89Xr3ynu6vJnA5rSrsu0uvhC8sHTtmt\r\n 5Mnqkg+3Wlwtj88wOdfpgvHIPiiBIaocWBb5FbjrBV0OFlbgqykcDfYwpAgA3ugJmsshThdt\r\n wL4nf9AGnsUvwVA4AqT/GZ5t+ieW4Hi8NjJBGcPIRwD3B68whEjD+uDT6QGGX45NA4NRh1bA\r\n C5GWqMP72HinHsj0C8C2g53RUMOA5eeeASm6Kezd1R9lTgIOd07FkGvYsetmkdFP5CsAd/ee\r\n pYy9Ar8KL/U8ow2NoCcf+EUg7bcx+PcYkPa72FtR/NoWojjWX0cTnInD10ejQUVsCLRSBM/F\r\n 7r+ORa+fhM0lR8xEn4/fte9hCM7Aq1XPzSRxDYWVdUMxpGgAfNBcCQF4r0oXcnwCdh4YZAxM\r\n 86kY6nsZQ/B8bP6zmq4FugHsTk+RwyMpbk3e7tC2S16vrDpWpnoUf6rszG0E/TsIsavEEOyZ\r\n +iQMPAuilVvzfeZmIUbK0wo9YZjLUuIcbt9WvRW/zecsdEmaa4ua65a1MCBLi7O5shM6xzml\r\n wiJZ9c1Qb7KMmMyZPn6+SeCzJb+8U5ZzZHWGpdjYMMxjWRG5HcZ0gipnywU7FLf//5o447AY\r\n Nlbd5si0jZYjeTQlm4guQxp7q/3QEMX+9rh+iBIYr88r25I54bgu5Q2pK9f7auXM1+8Guy2R\r\n A5PJJFj1TPpTvM4/hGT9GRK5OsPQqnj9r/we6lEoPUpHe4YRxS/9R9lKIAhfd5R+ef8W90a7\r\n qXEi0Ky0Tkobf1ydNjWeFb/QvjsUt25ydR73luP3jq9+yN07ssCXMj/UdGq8yvIrn3Hupu1l\r\n 2e3P28y+/fTdbw4v7zmaZLX1/bTWUdOyyC4qabXpvR+F8vd+m5AVmJfy5HRxyoD56Ymstk0d\r\n ig/VnyfPLB57NCYymktatphWNelf2asph/UDAAA=\r\n\r\n","parts":{"1":"Package: wnpp\r\nSeverity: wishlist\r\nOwner: Sruthi Chandran \r\nX-Debbugs-CC: debian-devel@lists.debian.org\r\n\r\n* Package name    : node-streamtest\r\n  Version         : 1.2.2\r\n  Upstream Author : Nicolas Froidure\r\n* URL             : https://github.com/nfroidure/streamtest\r\n* License         : Expat\r\n  Programming Lang: JavaScript\r\n  Description     : set of utils to test your stream based modules\r\naccross various stream implementations of NodeJS\r\n\r\n"}},"desiredParts":[{"id":"1","encoding":"7BIT","mimetype":"text/plain"}],"result":{"id":"6ba205ff710273ddc0bdeeb32daf2246a609df4bc7a6f898ecc4473dce85404c","to":[{"name":"","email":"submit@bugs.debian.org"}],"cc":[],"bcc":[],"from":[{"name":"Sruthi Chandran","email":"srud@disroot.org"}],"replyTo":[{"name":"","email":"847116@bugs.debian.org"}],"accountId":"test-account-id","body":"
Package: wnpp\r\nSeverity: wishlist\r\nOwner: Sruthi Chandran \r\nX-Debbugs-CC: debian-devel@lists.debian.org\r\n\r\n* Package name    : node-streamtest\r\n  Version         : 1.2.2\r\n  Upstream Author : Nicolas Froidure\r\n* URL             : https://github.com/nfroidure/streamtest\r\n* License         : Expat\r\n  Programming Lang: JavaScript\r\n  Description     : set of utils to test your stream based modules\r\naccross various stream implementations of NodeJS\r\n\r\n
","snippet":"Package: wnpp Severity: wishlist Owner: Sruthi Chandran  X-Debbugs-CC: debian-devel@lists.debian.org","unread":false,"starred":false,"date":"2016-12-05T18:18:35.000Z","folderImapUID":348641,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc","lists-debian-devel@bendel.debian.org","submit@bugs.debian.org"],"received":["by 10.140.100.181 with SMTP id s50csp1618416qge; Mon, 5 Dec 2016 10:18:35 -0800 (PST)","from dmz-mailsec-scanner-7.mit.edu (dmz-mailsec-scanner-7.mit.edu. [18.7.68.36]) by mx.google.com with ESMTPS id d81si1055491wmc.164.2016.12.05.10.18.35 for  (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 05 Dec 2016 10:18:35 -0800 (PST)","from mailhub-dmz-2.mit.edu ( [18.7.62.37]) (using TLS with cipher DHE-RSA-AES256-SHA (256/256 bits)) (Client did not present a certificate) by  (Symantec Messaging Gateway) with SMTP id 4D.F7.26209.97FA5485; Mon,  5 Dec 2016 13:18:33 -0500 (EST)","from dmz-mailsec-scanner-5.mit.edu (dmz-mailsec-scanner-5.mit.edu [18.7.68.34]) by mailhub-dmz-2.mit.edu (8.13.8/8.9.2) with ESMTP id uB5IHhdj021246 for ; Mon, 5 Dec 2016 13:18:33 -0500","from bendel.debian.org (bendel.debian.org [82.195.75.100]) (using TLS with cipher DHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by  (Symantec Messaging Gateway) with SMTP id D2.B7.11606.87FA5485; Mon,  5 Dec 2016 13:18:32 -0500 (EST)","from localhost (localhost [127.0.0.1]) by bendel.debian.org (Postfix) with QMQP id 75C2B17F; Mon,  5 Dec 2016 18:18:21 +0000 (UTC)","from localhost (localhost [127.0.0.1]) by bendel.debian.org (Postfix) with ESMTP id 9C7A7151 for ; Mon,  5 Dec 2016 18:18:12 +0000 (UTC)","from bendel.debian.org ([127.0.0.1]) by localhost (lists.debian.org [127.0.0.1]) (amavisd-new, port 2525) with ESMTP id XjRiqZkpBpLJ for ; Mon,  5 Dec 2016 18:18:07 +0000 (UTC)","from buxtehude.debian.org (buxtehude.debian.org [IPv6:2607:f8f0:614:1::1274:39]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (Client CN \"buxtehude.debian.org\", Issuer \"Debian SMTP CA\" (not verified)) by bendel.debian.org (Postfix) with ESMTPS id 82D7D170; Mon,  5 Dec 2016 18:18:07 +0000 (UTC)","from debbugs by buxtehude.debian.org with local (Exim 4.84_2) (envelope-from ) id 1cDxq0-0004I6-E6; Mon, 05 Dec 2016 18:18:04 +0000","via spool by submit@bugs.debian.org id=B.148096168015430 (code B); Mon, 05 Dec 2016 18:18:02 +0000","(at submit) by bugs.debian.org; 5 Dec 2016 18:14:40 +0000","from bs-one.disroot.org ([178.21.23.139] helo=disroot.org) by buxtehude.debian.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.84_2) (envelope-from ) id 1cDxmi-00040c-Gx for submit@bugs.debian.org; Mon, 05 Dec 2016 18:14:40 +0000","from localhost (localhost [127.0.0.1]) by disroot.org (Postfix) with ESMTP id 53193239F6 for ; Mon,  5 Dec 2016 19:14:37 +0100 (CET)","from disroot.org ([127.0.0.1]) by localhost (mail01.disroot.lan [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id z-Tz-PcP0cLK for ; Mon,  5 Dec 2016 19:14:37 +0100 (CET)"],"x-received":["by 10.194.138.111 with SMTP id qp15mr33288574wjb.3.1480961915687; Mon, 05 Dec 2016 10:18:35 -0800 (PST)"],"return-path":[""],"received-spf":["neutral (google.com: 18.7.68.36 is neither permitted nor denied by manual fallback record for domain of bounce-debian-devel=spang=mit.edu@lists.debian.org) client-ip=18.7.68.36;"],"authentication-results":["mx.google.com; dkim=fail header.i=@disroot.org; dkim=fail header.i=@disroot.org; spf=neutral (google.com: 18.7.68.36 is neither permitted nor denied by manual fallback record for domain of bounce-debian-devel=spang=mit.edu@lists.debian.org) smtp.mailfrom=bounce-debian-devel=spang=mit.edu@lists.debian.org","symauth.service.identifier"],"x-auditid":["12074424-9b7ff70000006661-5a-5845af79d415"],"x-mailbox-line":["From debian-devel-request@lists.debian.org  Mon Dec  5 18:18:21 2016"],"old-return-path":[""],"x-spam-checker-version":["SpamAssassin 3.4.0 (2014-02-07) on bendel.debian.org"],"x-spam-level":[""],"x-spam-status":["No, score=-8.0 required=4.0 tests=DKIM_SIGNED, HEADER_FROM_DIFFERENT_DOMAINS,LDO_WHITELIST,RP_MATCHES_RCVD,T_DKIM_INVALID autolearn=unavailable autolearn_force=no version=3.4.0"],"x-original-to":["lists-debian-devel@bendel.debian.org"],"x-virus-scanned":["at lists.debian.org with policy bank en-ht"],"x-amavis-spam-status":["No, score=-7.2 tagged_above=-10000 required=5.3 tests=[BAYES_00=-2, DKIM_SIGNED=0.1, HEADER_FROM_DIFFERENT_DOMAINS=0.001, LDO_WHITELIST=-5, RP_MATCHES_RCVD=-0.311, T_DKIM_INVALID=0.01] autolearn=ham autolearn_force=no"],"x-loop":["owner@bugs.debian.org","owner@bugs.debian.org","debian-devel@lists.debian.org"],"subject":["Bug#847116: ITP: node-streamtest -- set of utils to test your stream based modules accross various, stream implementations of NodeJS"],"reply-to":["Sruthi Chandran , 847116@bugs.debian.org"],"resent-from":["Sruthi Chandran "],"resent-to":["debian-bugs-dist@lists.debian.org"],"resent-cc":["debian-devel@lists.debian.org, wnpp@debian.org"],"resent-date":["Mon, 05 Dec 2016 18:18:02 +0000"],"resent-message-id":[""],"x-debian-pr-message":["report 847116"],"x-debian-pr-package":["wnpp"],"x-debian-pr-keywords":[""],"dkim-signature":["v=1; a=rsa-sha256; c=relaxed/simple; d=disroot.org; s=mail; t=1480961677; bh=D83w+mcR33/VQkITQI3T+hYFSLlGEhXR0gNcNOKma1M=; h=From:Subject:To:Date:From; b=RGfjrzldXfrn98cwJECU13CLtE6NurUcFar/ymwcIOWcI/iurS1yB77ufr6lvSnIv l7GcLyXyiXlFHJI0gacV3jsigJG3NWGyxOT670K+8/00FSUPPI0oT7GgSQQKrZ0RnX 1ydoD4ZCpm148HmxuCk972HdCYIRlA1YO4ggm35U=","v=1; a=rsa-sha256; c=relaxed/simple; d=disroot.org; s=mail; t=1480961677; bh=D83w+mcR33/VQkITQI3T+hYFSLlGEhXR0gNcNOKma1M=; h=From:Subject:To:Date:From; b=RGfjrzldXfrn98cwJECU13CLtE6NurUcFar/ymwcIOWcI/iurS1yB77ufr6lvSnIv l7GcLyXyiXlFHJI0gacV3jsigJG3NWGyxOT670K+8/00FSUPPI0oT7GgSQQKrZ0RnX 1ydoD4ZCpm148HmxuCk972HdCYIRlA1YO4ggm35U="],"from":["Sruthi Chandran "],"to":["submit@bugs.debian.org"],"message-id":["<2bee7571-9098-d8d5-b4a4-59fc369c579b@disroot.org>"],"date":["Mon, 5 Dec 2016 23:44:30 +0530"],"user-agent":["Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Icedove/45.5.0"],"mime-version":["1.0"],"content-type":["text/plain; charset=windows-1252"],"content-transfer-encoding":["7bit"],"x-rc-virus":["2007-09-13_01"],"x-rc-spam":["2008-11-04_01"],"x-mailing-list":[" archive/latest/323341"],"list-id":[""],"list-url":[""],"list-post":[""],"list-help":[""],"list-subscribe":[""],"list-unsubscribe":[""],"precedence":["list"],"resent-sender":["debian-devel-request@lists.debian.org"],"list-archive":["https://lists.debian.org/msgid-search/2bee7571-9098-d8d5-b4a4-59fc369c579b@disroot.org"],"x-brightmail-tracker":["H4sIAAAAAAAAA1VTa0gUYRT1mxl1fEyNY7VXyx5DEURqpVBmSJD2wH5kBUkEObmTu7S72sz6 WElaf2zFqmBhWWqskYWKqAmmUUm6lI8eUJm9LItURFFXsswMacZvTfp37jnnnnvv8A1Nck1e wbTeZBYlk2DgvXwpzjtmXailLi5x043La7aNl/zx3on2zkzcIQ6go747tKJBnyFK4TFJvrp7 43eptNdk1k9nEbKiAcKOfGhgI6G85jFSMcfaCLBdSrAjXwX3IDjf30vZET1nsk2RmC8jYMTR TeGiEMG7bocXTjoMb10vSBUvZsOhuc2JML8f7jfOeGO8CF4O/SLUZmCdCIodTyksRECl4/tc A8Vugca+fveEBwhanffcE6KhobcS4e4pBC03O91LDSKwP5xEuHhFQI/tk1upQzDYOeuJzwiB Z++zVT6QLUCQ13OfwHfHwpPuSVL1cMrmfVc5TG8Ex/QYwnQU9BTk/JeCHTWfbpEYx8Hg6xH3 ojxUFz5wf+AgeDVR4z5/GTRZq7ywPxNy2y9ShSimRCkZNgA6r/VTKibZMGh++9sT41XQNFpG liOyGoVojdmhRkFvkMXkUDlZMJlEKXRrmFFvDhO16Q1o7jXErm1GefnxbYilEe/PuKxxiZyn kCFbjG0oiCb4pUxRkkItOpGqtegEWXdcSjeIchsCmuSXMEuVh8QxWsGSLUqp89JymuI1jMee H0c4NkUwi6dEMU2U5tUVNM0D86FWaQyQxBQx66TeYF6QCdpHDfdXwqtUDyOnCUZZn4L1LhRB f2ivGCDoR+OVAwRHmVJNYrCGKVWtrGrVpZv+pc2/92GkUY4LZGJUl7/yN/zLG1ZGEcqojvZd 6iizsCAFWxEzuy/rVKV5JnB17MkE576RrMHTZ2s/5yTH11svFOVmTDVm+rWPUd8Otvh/PRNn 01QMdD+iXYbtLYeKg/3q9Xm9rvXr5I/OjQ3TUXe4/D6f/mTNqpWJqWu3PB86FvLmXPSViTF+ d+vN8N4jAV9Hb0sdI9c7n33pivSwR0SW0pZalx9PyTph8wZSkoW/az3I+MoDAAA=","H4sIAAAAAAAAA1VSa0wUVxTeMzMsw3YHh0HhuHWjmRATbRcfaNJXWhLEmrZGQzTapo2O7shO 3AeZWZ4NLaWxJRjCWm2sBLIkikF8JSiyVMS6GwWU1ogEfCGJAkGxaKMitYF0hrvY+ufmO+f7 zvm+e3NZWrhntrFygV9WvZJbNFuYrMinTkfByczNS89Xr3ynu6vJnA5rSrsu0uvhC8sHTtmt 5Mnqkg+3Wlwtj88wOdfpgvHIPiiBIaocWBb5FbjrBV0OFlbgqykcDfYwpAgA3ugJmsshThdt wL4nf9AGnsUvwVA4AqT/GZ5t+ieW4Hi8NjJBGcPIRwD3B68whEjD+uDT6QGGX45NA4NRh1bA C5GWqMP72HinHsj0C8C2g53RUMOA5eeeASm6Kezd1R9lTgIOd07FkGvYsetmkdFP5CsAd/ee pYy9Ar8KL/U8ow2NoCcf+EUg7bcx+PcYkPa72FtR/NoWojjWX0cTnInD10ejQUVsCLRSBM/F 7r+ORa+fhM0lR8xEn4/fte9hCM7Aq1XPzSRxDYWVdUMxpGgAfNBcCQF4r0oXcnwCdh4YZAxM 86kY6nsZQ/B8bP6zmq4FugHsTk+RwyMpbk3e7tC2S16vrDpWpnoUf6rszG0E/TsIsavEEOyZ +iQMPAuilVvzfeZmIUbK0wo9YZjLUuIcbt9WvRW/zecsdEmaa4ua65a1MCBLi7O5shM6xzml wiJZ9c1Qb7KMmMyZPn6+SeCzJb+8U5ZzZHWGpdjYMMxjWRG5HcZ0gipnywU7FLf//5o447AY Nlbd5si0jZYjeTQlm4guQxp7q/3QEMX+9rh+iBIYr88r25I54bgu5Q2pK9f7auXM1+8Guy2R A5PJJFj1TPpTvM4/hGT9GRK5OsPQqnj9r/we6lEoPUpHe4YRxS/9R9lKIAhfd5R+ef8W90a7 qXEi0Ky0Tkobf1ydNjWeFb/QvjsUt25ydR73luP3jq9+yN07ssCXMj/UdGq8yvIrn3Hupu1l 2e3P28y+/fTdbw4v7zmaZLX1/bTWUdOyyC4qabXpvR+F8vd+m5AVmJfy5HRxyoD56Ymstk0d ig/VnyfPLB57NCYymktatphWNelf2asph/UDAAA="],"x-gm-thrid":"1552901121651491080","x-gm-msgid":"1552901121651491080","x-gm-labels":["debiandevel"]},"headerMessageId":"<2bee7571-9098-d8d5-b4a4-59fc369c579b@disroot.org>","subject":"Bug#847116: ITP: node-streamtest -- set of utils to test your stream based modules accross various, stream implementations of NodeJS","folderImapXGMLabels":"[\"debiandevel\"]"}}
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/spam-mime-html-base64-encoded.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/spam-mime-html-base64-encoded.json
new file mode 100644
index 000000000..4fff83e47
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/spam-mime-html-base64-encoded.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"partID":"1","type":"text","subtype":"html","params":{"charset":"utf-8"},"id":null,"description":null,"encoding":"BASE64","size":1318,"lines":19,"md5":null,"disposition":null,"language":null}],"date":"2016-12-05T22:09:12.000Z","flags":[],"uid":6466,"modseq":"8251529","x-gm-labels":[],"x-gm-msgid":"1552915630214378399","x-gm-thrid":"1552915630214378399"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.140.100.181 with SMTP id s50csp1710867qge; Mon, 5 Dec 2016\r\n 14:09:12 -0800 (PST)\r\nX-Received: by 10.129.97.134 with SMTP id v128mr54944350ywb.338.1480975752135;\r\n Mon, 05 Dec 2016 14:09:12 -0800 (PST)\r\nReturn-Path: \r\nReceived: from muffat.debian.org (muffat.debian.org.\r\n [2607:f8f0:614:1::1274:33]) by mx.google.com with ESMTPS id\r\n w195si4900468ywd.160.2016.12.05.14.09.11 for \r\n (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 05 Dec\r\n 2016 14:09:12 -0800 (PST)\r\nReceived-SPF: neutral (google.com: 2607:f8f0:614:1::1274:33 is neither\r\n permitted nor denied by best guess record for domain of cfvqtub@rapab.com)\r\n client-ip=2607:f8f0:614:1::1274:33;\r\nAuthentication-Results: mx.google.com; spf=neutral (google.com:\r\n 2607:f8f0:614:1::1274:33 is neither permitted nor denied by best guess record\r\n for domain of cfvqtub@rapab.com) smtp.mailfrom=cfvqtub@rapab.com\r\nMessage-Id: <5845e588.ccd40d0a.ea741.b529SMTPIN_ADDED_MISSING@mx.google.com>\r\nReceived: from [180.127.164.182] (helo=rapab.com) by muffat.debian.org with\r\n esmtp (Exim 4.84_2) (envelope-from ) id 1cE1Re-0002e1-KR\r\n for christine@spang.cc; Mon, 05 Dec 2016 22:09:11 +0000\r\nReceived: from vps5754 ([127.0.0.1]) by localhost via TCP with ESMTPA; Tue, 06\r\n Dec 2016 06:08:54 +0800\r\nMIME-Version: 1.0\r\nFrom: Peggy \r\nSender: Peggy \r\nTo: christine@debian.org\r\nReply-To: Peggy \r\nDate: 6 Dec 2016 06:08:54 +0800\r\nSubject: =?utf-8?B?VGhlIEJlc3QgT2ZmZXIgb2YgbGFueWFyZHM=?=\r\nContent-Type: text/html; charset=utf-8\r\nContent-Transfer-Encoding: base64\r\n\r\n","parts":{"1":"PGh0bWw+PGJvZHk+PFA+RGVhciBmcmllbmQsPC9QPg0KPFA+VGhhbmsgeW91IGZvciB5b3Vy\r\nIGF0dGVudGlvbi4gPC9QPg0KPFA+T3VyIGNvbXBhbnkgaXMgYSBvbmUtc3RvcCBsYW55YXJk\r\ncyBmYWN0b3J5ICwgcHJvdmlkaW5nIHByaW50aW5nIGFuZCBkaXN0cmlidXRpbmcgc2Vydmlj\r\nZSB0byBjdXN0b21lcnMgYXJvdW5kIHRoZSB3b3JsZC48L1A+DQo8UD5NYWluIHByb2R1Y3Rz\r\nIGFyZSBzaWxrIHByaW50IC9oZWF0IHRyYW5zZmVycmVkIC8gc2F0aW4gbGFjZSBvbiByaWJi\r\nb24vIHdvdmVuIGxhbnlhcmRzLCZuYnNwOyBsaWdodCB1cCBsYW55YXJkcywgYm90dGxlIGhv\r\nbGRlcnMsIFVTQiB3cmlzdGJhbmRzLCBsYW55YXJkIHdpdGggd2F0ZXJwcm9vZiByYWluIGhh\r\ndCwgYm90dGxlIG9wZW5lcnMsa2V5IGNoYWlucywgY2FyYWJpbmVycywgc2hvZXNsYWNlcywg\r\nbHVnZ2FnZSBiZWx0cywgSUQgY2FyZCBob2xkZXJzLG1vYmlsZSBwaG9uZSBzdHJhcHMsIG5l\r\nY2tzdHJhcHMuZXRjLjwvUD4NCjxQPkFsbCBwcm9kdWN0cyBhcmUgY3VzdG9taXplZC4gTm8g\r\ncHJpY2UgbGlzdC4gSWYgeW91IG5lZWQgcXVvdGF0aW9uLCBwbGVhc2UgbGV0IG1lIGtub3cg\r\ndGhlIHNwZWNpZmljYXRpb25zLjwvUD4NCjxQPldlIGNhbiBtYWtlIGFzIHBlciB5b3VyIGRl\r\nc2lnbiBhbmQgeW91ciBsb2dvLjwvUD4NCjxQPk5vIE1PUSBzZXJ2aWNlPC9QPg0KPFA+RGVz\r\naWduZXIgU2VydmljZTwvUD4NCjxQPlBNUyBleHByZXNzIHNlcnZpY2U8L1A+DQo8UD5JdCB3\r\naWxsIGJlIGdyZWF0IGlmIHlvdSBjb3VsZCBzZW5kIG1lIGZpbGUgb3IgcGljdHVyZSBmb3Ig\r\ncmVmZXJlbmNlLjwvUD4NCjxQPkJlc3QgUmVnYXJkcyw8L1A+DQo8UD5QZWdneTxCUj5Nb2I6\r\nODYtMTg4IDI1NTQgNTg0NjxCUj5BREQ6Q2hhbmcgQW4gVG93bixEb25nZ3VhbiBDaXR5IEd1\r\nYW5kb25nIFByb3ZpbmNlLCBDaGluYTwvUD48L2JvZHk+PC9odG1sPg==\r\n\r\n"}},"desiredParts":[{"id":"1","encoding":"BASE64","mimetype":"text/html"}],"result":{"id":"9c81f13eb472b193dcd8ac571e77c5881d9d31c8d13e710eb7f394e735ef0f64","to":[{"name":"","email":"christine@debian.org"}],"cc":[],"bcc":[],"from":[{"name":"Peggy","email":"yanmusinei55@163.com"}],"replyTo":[{"name":"Peggy","email":"yanmusinei55@163.com"}],"accountId":"test-account-id","body":"
Dear friend,
\r\n
Thank you for your attention. 
\r\n
Our company is a one-stop lanyards factory , providing printing and distributing service to customers around the world.
\r\n
Main products are silk print /heat transferred / satin lace on ribbon/ woven lanyards,  light up lanyards, bottle holders, USB wristbands, lanyard with waterproof rain hat, bottle openers,key chains, carabiners, shoeslaces, luggage belts, ID card holders,mobile phone straps, neckstraps.etc.
\r\n
All products are customized. No price list. If you need quotation, please let me know the specifications.
\r\n
We can make as per your design and your logo.
\r\n
No MOQ service
\r\n
Designer Service
\r\n
PMS express service
\r\n
It will be great if you could send me file or picture for reference.
\r\n
Best Regards,
\r\n
Peggy
Mob:86-188 2554 5846
ADD:Chang An Town,Dongguan City Guandong Province, China
","snippet":"Dear friend,\r\nThank you for your attention. \r\nOur company is a one-stop lanyards factory , providing","unread":true,"starred":false,"date":"2016-12-05T22:09:12.000Z","folderImapUID":6466,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc"],"received":["by 10.140.100.181 with SMTP id s50csp1710867qge; Mon, 5 Dec 2016 14:09:12 -0800 (PST)","from muffat.debian.org (muffat.debian.org. [2607:f8f0:614:1::1274:33]) by mx.google.com with ESMTPS id w195si4900468ywd.160.2016.12.05.14.09.11 for  (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Mon, 05 Dec 2016 14:09:12 -0800 (PST)","from [180.127.164.182] (helo=rapab.com) by muffat.debian.org with esmtp (Exim 4.84_2) (envelope-from ) id 1cE1Re-0002e1-KR for christine@spang.cc; Mon, 05 Dec 2016 22:09:11 +0000","from vps5754 ([127.0.0.1]) by localhost via TCP with ESMTPA; Tue, 06 Dec 2016 06:08:54 +0800"],"x-received":["by 10.129.97.134 with SMTP id v128mr54944350ywb.338.1480975752135; Mon, 05 Dec 2016 14:09:12 -0800 (PST)"],"return-path":[""],"received-spf":["neutral (google.com: 2607:f8f0:614:1::1274:33 is neither permitted nor denied by best guess record for domain of cfvqtub@rapab.com) client-ip=2607:f8f0:614:1::1274:33;"],"authentication-results":["mx.google.com; spf=neutral (google.com: 2607:f8f0:614:1::1274:33 is neither permitted nor denied by best guess record for domain of cfvqtub@rapab.com) smtp.mailfrom=cfvqtub@rapab.com"],"message-id":["<5845e588.ccd40d0a.ea741.b529SMTPIN_ADDED_MISSING@mx.google.com>"],"mime-version":["1.0"],"from":["Peggy "],"sender":["Peggy "],"to":["christine@debian.org"],"reply-to":["Peggy "],"date":["6 Dec 2016 06:08:54 +0800"],"subject":["The Best Offer of lanyards"],"content-type":["text/html; charset=utf-8"],"content-transfer-encoding":["base64"],"x-gm-thrid":"1552915630214378399","x-gm-msgid":"1552915630214378399","x-gm-labels":[]},"headerMessageId":"<5845e588.ccd40d0a.ea741.b529SMTPIN_ADDED_MISSING@mx.google.com>","subject":"The Best Offer of lanyards","folderImapXGMLabels":"[]"}}
\ No newline at end of file
diff --git a/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/theskimm-multipart-alternative-quoted-printable.json b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/theskimm-multipart-alternative-quoted-printable.json
new file mode 100644
index 000000000..ee3c81ad8
--- /dev/null
+++ b/packages/local-sync/spec/fixtures/MessageFactory/parseFromImap/theskimm-multipart-alternative-quoted-printable.json
@@ -0,0 +1 @@
+{"imapMessage":{"attributes":{"struct":[{"type":"alternative","params":{"boundary":"Aw0OfBZoDhFa=_?:"},"disposition":null,"language":null},[{"partID":"1","type":"text","subtype":"plain","params":{"charset":"us-ascii"},"id":null,"description":null,"encoding":"QUOTED-PRINTABLE","size":16310,"lines":365,"md5":null,"disposition":null,"language":null}],[{"partID":"2","type":"text","subtype":"html","params":{"charset":"us-ascii"},"id":null,"description":null,"encoding":"QUOTED-PRINTABLE","size":107910,"lines":2599,"md5":null,"disposition":null,"language":null}]],"date":"2016-11-02T10:21:52.000Z","flags":["\\Seen"],"uid":343848,"modseq":"8022913","x-gm-labels":["\\Important"],"x-gm-msgid":"1549881429115204149","x-gm-thrid":"1549881429115204149"},"headers":"Delivered-To: christine@spang.cc\r\nReceived: by 10.31.236.3 with SMTP id k3csp698913vkh; Wed, 2 Nov 2016 03:21:52\r\n -0700 (PDT)\r\nX-Received: by 10.55.47.193 with SMTP id v184mr2167122qkh.259.1478082112295;\r\n Wed, 02 Nov 2016 03:21:52 -0700 (PDT)\r\nReturn-Path: \r\nReceived: from mta.morning7.theskimm.com (mta.morning7.theskimm.com.\r\n [136.147.177.13]) by mx.google.com with ESMTPS id\r\n j59si882394qtb.16.2016.11.02.03.21.50 for \r\n (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Wed, 02 Nov\r\n 2016 03:21:52 -0700 (PDT)\r\nReceived-SPF: pass (google.com: domain of\r\n bounce-20_html-215009-5489-7208679-430@bounce.morning7.theskimm.com\r\n designates 136.147.177.13 as permitted sender) client-ip=136.147.177.13;\r\nAuthentication-Results: mx.google.com; dkim=pass\r\n header.i=@morning7.theskimm.com; spf=pass (google.com: domain of\r\n bounce-20_html-215009-5489-7208679-430@bounce.morning7.theskimm.com\r\n designates 136.147.177.13 as permitted sender)\r\n smtp.mailfrom=bounce-20_HTML-215009-5489-7208679-430@bounce.morning7.theskimm.com;\r\n dmarc=pass (p=NONE dis=NONE) header.from=theskimm.com\r\nDKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; s=200608;\r\n d=morning7.theskimm.com;\r\n h=From:To:Subject:Date:List-Unsubscribe:MIME-Version:Reply-To:List-ID:Message-ID:Content-Type;\r\n i=dailyskimm@morning7.theskimm.com; bh=TfZNUfri1El8i4AUulbXxJgLUCA=;\r\n b=H6ZRxb2e2tmobb63CIcQ1ecsfv+X+Ky/G0EW0Kct6kvNmLP/yASdttr9bcgRbUMO45Su7bytynrE\r\n U+kbevaHw5nbatKKWQFP7JxGMjzCtQGM6LzOjGV8qCSu7XwdQ0QgUCJo0V4eo7Iu6bafR9h2WJST\r\n ct12e4elnzHFZLnF7GM=\r\nReceived: by mta.morning7.theskimm.com id h36v3s163hst for\r\n ; Wed, 2 Nov 2016 10:21:47 +0000 (envelope-from\r\n )\r\nFrom: \"theSkimm\" \r\nTo: \r\nSubject: Daily Skimm: Hey batter batter\r\nDate: Wed, 02 Nov 2016 04:21:46 -0600\r\nList-Unsubscribe: \r\nMIME-Version: 1.0\r\nReply-To: \"theSkimm\"\r\n \r\nList-ID: <7208679_5489.xt.local>\r\nX-CSA-Complaints: whitelistcomplaints@eco.de\r\nx-job: 7208679_5489\r\nMessage-ID: \r\nContent-Type: multipart/alternative; boundary=\"Aw0OfBZoDhFa=_?:\"\r\n\r\n","parts":{"1":"\r\n\r\n\r\n\r\n\r\nIs this email not displaying correctly?\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda378a2f7dccb8e2=\r\n6b5b302c761b48c753ae37dd56d06342bd7761205726e794f19c \r\nView it in your browser=2E \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda37898ee592de28=\r\nce7d2b78cb72be88185744d89b55cde5d34a4ad6dba714d942a6 \r\nSHARE THIS\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda3721d18ab2f8a5=\r\na53c473e99b28bc622950017c4958d9d3d47623dfaa963b2fe2f \r\nSHARE THIS\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda370f716da4e7a0=\r\nd2fc45cc1fb72979495e447d7a98e5f9d8e8385598d3a36bf093 \r\n\r\n\r\nSkimm for November 2nd\r\n\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda3737d8d9ff2a65=\r\n6be9041e65e9ad55a4c1c75a705f32854b662fd04c5b65251cda \r\n\r\n\r\n\r\n\r\n\r\nSkimm'd while getting up on what's at stake in 2016=2E \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518e4ff0b477556=\r\nee2011e623a9141618b501e48d7ada8a3ed516d0b3c0f3f9fa1b \r\nReady to vote? \r\n\r\n\r\n\r\nQUOTE OF THE DAY\r\n\r\n\r\n\";Moisture harvested from the clouds\"; - \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015187600c0dbcd6d=\r\n17ec7331f3cfc3b5f32608d5bedc4d80db9f776591ce730db6c1 \r\nA description  of how Sky PA, a new Scottish beer, is made=2E Cloudy with a=\r\n chance of hipster=2E\r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015180b3ee957efc7=\r\n43770fa87d771b9e197126982da785ce1918153b77f693c3ce8b \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518f2d49f9d3484=\r\n562b6c859a38a9b756ddc039572f4aa0f9c5037e50c2d0bae33c \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518e7517a647921=\r\n45e64a9ed2022fa40ac9e1b913bf4cbf414945e77b598a60d31d \r\n\r\nGAS PAINS\r\n\r\nTHE STORY\r\nEarlier this week, a major oil pipeline that runs through Alabama \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015180ed06f7e37bd=\r\n36a28c9824956812295ae4df7646f389759b34dfd2fda72017a9 \r\nexploded =2E\r\nWAIT…BACK UP=2E\r\nThe Colonial Pipeline is a system of pipes thousands of miles long that car=\r\nries millions of barrels of gas, diesel, and jet fuel aday from Texas to Ne=\r\nw Jersey=2E It supplies about a third of the East Coast's gas=2E So it's a =\r\nBF gassy D=2E But the pipeline's run into some problems lately=2E Earlier t=\r\nhis fall, part of it was \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518813c5513accd=\r\n483e8db374227aa0e48b9f59b7e9d29b9b6525b209d347dd27b5 \r\nshut down  for over a week after a leak in Alabama spilled hundreds of thou=\r\nsands of gallons of gas=2E Cue a \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015185f16c50280b9=\r\nde78bd24094b4015e8630991b8156c7bcd4cb37b39824eefd5b3 \r\ngas shortage  and prices in southern states going up, up, up=2E\r\nSO WHAT'S THE LATEST?\r\nOn Monday, there was an explosion along the pipeline in Alabama that caused=\r\n a major fire=2E One person was killed and several others were injured=2E Y=\r\nesterday, Alabama's governor declared a state of emergencyto help make sure=\r\n gas is delivered throughout the state=2EPart of the pipeline's been shut d=\r\nown and now the company that operates it says it \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518f1a7a6aa74e5=\r\n4eaee3f22c427c587f46e59947aedc70e2ddbba3242f6bfb8b69 \r\ncould take days  to get the whole thing up and running again=2E Sofornow it=\r\nlooks like gas prices could be back on the up and up=2E\r\ntheSKIMM\r\nA major pipeline that millions of people depend on for gas every day can't =\r\nseem to get off the struggle bus=2E And now this latest incident has fuel c=\r\nompanies scrambling to stock up on supply=2E Ifit starts costing more to fi=\r\nll up the tank, this is why=2E\r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518ea9e4fa2f243=\r\n3c44e74693af3cd9018f5ece13b52fa953ffffb71b777ffa0e25 \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015183b3cc0c3f30e=\r\n436433c367ff042d3890808122395e67f4344c779240b0fb96b2 \r\n\r\nREPEAT AFTER ME=2E=2E=2E\r\n\r\nWHAT TO SAYTO YOUR FRIEND WHO GREW UP ON DEEP DISH PIZZA…\r\n\r\nIn it to win it=2E Last night, the \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518eb92f32ee674=\r\nc5ae648f0c76b48d0f861ccc7e253e391b9fc24ffef3c6d321fb \r\nChicago Cubs tied up the World Series  by beating the Cleveland Indians in =\r\nGame 6=2E They won 9-3=2E Now all eyes are on Game 7 tonight=2E The Cubs ha=\r\nven't won the series since 1908=2E The Indians haven't won since 1948=2E So=\r\n it's not like there's a lot on the line or anything=2E Break out the peanu=\r\nts=2E\r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f0151855235b31e076=\r\n45b8ad0f96b07b7084f725abc8922a18c65775c160f2b08e39a2 \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518bba5a4134f1a=\r\nd3263f4b6e4af51516c63ab1f22dacc4f432de7db7b200638eeb \r\n\r\nWHAT TO SAY WHEN YOU FIND OUT YOURTWO FRIENDS WHO YOU INTRODUCED ARE GETTIN=\r\nG TOGETHERWITHOUT YOU=2E=2E=2E\r\n\r\nWhat's going on here? Earlier this week, \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518335c687702c2=\r\na91cdae5c278dd891b638aef063583271a5bb0b6836c0953412f \r\na federal judge ordered  the Republican party to explain any deals it mayha=\r\nve made with GOP nominee Donald Trump's campaign to monitor the polls durin=\r\ng this election season=2E Reminder: for months, Trump has been saying this =\r\nelection is\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015181282a946ce17=\r\nef3bccb16f5ee0ab70200fc13d574dfa402946fb8f75a3a9f0f7 \r\n\";rigged\"; against him, and encouraging his supporters to be on the lookout=\r\n for voter fraud=2E Early voting opened recently in a lot of states=2E And =\r\nthere have been reports of Trump supporters allegedly photographing and \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015187d61ecd759d7=\r\nb0730c22d96dc27b5bbdf7ca9040c492390b0af040f590c882b5 \r\nharassing  people at the polls=2E That's according to a string of different=\r\n lawsuits inArizona, Ohio, and Nevada flaggingconcerns that some voters are=\r\n being intimidated=2E Last week, the Democratic party filed a separate comp=\r\nlaint with a federal court=2E Now, the Republican party has until the end o=\r\nf today to turn over any evidence that could be related to collaboratingwit=\r\nh the Trump campaign=2EStay tuned=2E\r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518dc0466271971=\r\n9d7c96a196b6493a840195c679622ee661714c17971d712ff8e3 \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518e34779768729=\r\n1180617bc85052596e23d29e50c671041e605e72fc704ed3d004 \r\n\r\nWHAT TO SAY TO WHEN YOUR AM TRAIN GETS DELAYED FOR THE FIFTH TIME…\r\n\r\nI give up=2E Yesterday, Russia put Syrian peace talks on hold=2E \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518f6d20f368728=\r\neb2eed5d4f06ed7405a177f0acf26b236762f262bfa4aa7f8044 \r\nIndefinitely=2E  For years, Syria has been going through a violent civil wa=\r\nr between Syrian President Bashar al-Assad (backed by Russia) and rebel gro=\r\nups (some backed by the US) who want him out of power=2E Hundreds of thousa=\r\nnds of people have been killed=2E Millions have been forced to leave home, =\r\ncausing the EU's worst migrant and refugee crisis since World War II=2E The=\r\n two sides have been trying and failing to hash out a peace deal for a whil=\r\ne now=2E Earlier this month, Russia agreed to a \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f0151881e18b7235c5=\r\n73a07dea1d6f66d4e48cd2742d456c53c4803082f69bb0806362 \r\n\";humanitarian pause\";  to airstrikes in Aleppo - the key rebel stronghold =\r\nwhere a lot of thefighting has been focused=2E Meanwhile, rebel's have been=\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015189ef9643b4194=\r\n48cc8bcbcc8372e8c961fd45cc9331da20df64ae665f2298570e \r\nfighting back in the city =2E And today, Russia announced it would give reb=\r\nels \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518afb1f76fc2f9=\r\ne26bd6f8888964170d459663fedc1855df664b5efa241f0c9f2b \r\neven more time  to leave Aleppo=2EBut it's still walking away from the peac=\r\ne table for the foreseeable future=2E So, Syria's still a peace of work=2E\r\n=\r\n\r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f0151842b260af3e1b=\r\nc0d9404642d2e41e7037c150074a140f84985b3132dcecd8d9b8 \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f0151878e0dacd88d3=\r\nd84627a4a4c82c3debbbab214d0eab433d4640110ad6d8f2c6e4 \r\n\r\nWHAT TO SAY WHEN YOUR DATE SAYS HE NEEDS A NOISE MACHINE ON HIGH VOLUME TO =\r\nFALL ASLEEP=2E=2E=2E\r\n\r\nHaving second thoughts=2E That's what Gannett is telling Tronc (the artist =\r\nformerly known as Tribune Publishing)=2E For months, Gannett, which owns ab=\r\nout a hundred news outlets including USA Today, has been \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518bdef4b851c42=\r\n6335e6df0d77034c6d82d93d562a29526e8f2686412ced239f5f \r\ntrying to buy Tronc , which owns nameslike the LA Times and Chicago Tribune=\r\n=2E For years, print news companies have been struggling to keepad dollars =\r\nand readers on board=2E Gannett was hoping that the power of two media comp=\r\nanies combined would be more attractive than one, especially to advertisers=\r\n=2E The dealwould have created one of the largest media groups in the count=\r\nry=2E So earlier this year, Gannett saddled up to Tronc with a buyout offer=\r\n of around $400 million=2E Tronc said 'no, thanks, we're not that cheap=2E'=\r\n Yesterday, after a lot ofback and forth, Gannett \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518d2bffab0c234=\r\nf17b032b9701c23e88c1cd13f48f1fdc3ceff250bcb5f1df018d \r\nwalked away from the deal=2E  This mightalso be because Gannett had 'meh' e=\r\narnings last quarter, meaning they couldn't get the cash together to write =\r\nthe check=2E Either way, both companies are still single AF=2E\r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015182da3184f75be=\r\n5fdc54fecb1ac3b84096d5f14c82b9c1885506577545687ffc20 \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f0151819c57a5144e9=\r\n444db885889da1a6f479ee69473d5dc1540bded14eb2254d7a61 \r\n\r\nWHAT TO SAY TO YOUR FRIEND WHO ONLY DOES SQUATS AT THE GYM =2E=2E=2E\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f0151874e24bacd196=\r\n875a37d50bd496ccbfdb973ca05810b522ba89490afa4a8a9a45 \r\nSay goodbye to your fave emoji=2E \r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518989cbf077489=\r\n0fc15ee0775e5993d2e6f4301ecd0fb28b902dce8178e03e765a \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015186faa6a631bb0=\r\nb3f66a437bac5f5683830b4a05e4edd8e9289820f4606048f5a4 \r\n\r\nSKIMM THE VOTE\r\n\r\nIn case you somehow missed it, you have to vote on Tuesday=2E Catch up on w=\r\nho's on your ballot and what's at stake=2E Then plan your trip to the polls=\r\n=2E \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015188c8432bed622=\r\n7d27e00e716a22c0a3cb51beaa8ddacfe063b60ac233b6606f25 \r\nQuestions, answered=2E \r\n\r\nSkimm This\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518d25ca1d7e82b=\r\nf2beb04f9ea0fd4fac2a209c9e8fa3162db1c538ebe60b2cb8be \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518fa56ed304e70=\r\ndc7806f79721441d3f2dba8a0d21fe6daaa263e2640a05536a61 \r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015181e724ef34cab=\r\nd7ed78134c92f94f694fbdb799777126a5594c4ea20eaa22ed06 \r\n\r\nSKIMM 50\r\n\r\nThanks to these Skimm'bassadors for sharing like it's hot=2E Want to see yo=\r\nur name on this list? \r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f01518ceda3d8e1ed0=\r\nb62e87b8df3fe3ecb8a5f3ad2f674819afcdb1005e18c5f072d3 \r\nClick here  to learn more=2E\r\n\r\nOlivia Simonson, Brittany Berger, Daniela Rivas, Ashley Lawson, Sarah David=\r\nson, L Denease Thompson-Mack, Noreen Deutsch, Jessee Fordham, Natasha Shah,=\r\n Shari Berga, Aloke Prabhu, Natalie Tenzer, Samantha Panchevre, Ania Arseno=\r\nwicz, Ali Wozniak, Reinalyn, Taunya Robinson, Cassie Christensen, Spencer P=\r\nhilips, Abby Bilinski, Aner Zhou, Julia Erdenebold, Mehreen Mazhar, Alexand=\r\nra Rizzo, Alisa Sutton, Maegan Detlefs, Janina Ancona, Chris Drake, Allan M=\r\noss, Jordan Murray, David Benjamin, Leslie Bartula, Hana Muslic, Kelly Wall=\r\nace, Leslie Buteyn, Bethany Fitzgerald, Stuart Ferguson, Emily Paulino, All=\r\nison Ryan, Niki DeMaio, Ria Conti, Nikki Boyd, Amanda Oliver, Lindsay Barth=\r\nel, Shelby Wynn, Brittany Daunno, Emily Rissmiller, Alex Johannes, Andrea B=\r\norod, Ashley Montufar, Karli Von Herbulis, Lexie Rindler, Natalie Weaver, A=\r\nllison Sotelo, Cynthia Lopez, Isabel Taylor, Jess Tolan, Kasie Heiden, Kels=\r\ney Will, Leslie McFayden, Morgan Balavage, Morgan Goracke, Madeline Trainor=\r\n, Alexa Parisi, Uma Sudarshan, Brooke Borkowski, Connie Lin, Jess Ridolfino=\r\n, Jessica Foster, Mallory Mickus, Sapreen Abbass, Kelli Holland\r\n\r\nSKIMM SHARE\r\n\r\nHalfway there=2E Share theSkimm with your work wife whoalways takes a coffe=\r\ne break when you need it=2E\r\n\r\nhttp://www=2Etheskimm=2Ecom/invite/v2/new?email=3Dchristine@spang=2Ecc&utm_=\r\nsource=3Demail&utm_medium=3Dinvite&utm_campaign=3Dbottom \r\n\r\nSKIMM BIRTHDAYS\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D13e1958399f015185aea2b21fa15=\r\n1bb8b11f09d4ab4aefdd45bee2439742b48454e60e800bcdb09f \r\n* indicates Skimm'bassador=2EHammer time=2E\r\n\r\nLouise Cronin (San Francisco, CA); Faith Greenberg(Longwood, FL);*Hatley Th=\r\nompson(Washington, DC);*Monique Ervine(Kindersley, Canada);*Leana Macrito(C=\r\nhicago, IL);*AnnMarie Murtaugh(Houston, TX);*Maria Perwerton(Rochester, MI)=\r\n;*Lauren Valainis(Washington, DC);*Jaycie Moller(San Francisco, CA);*Erin M=\r\nanfull(Iowa City, IA);*Jennifer Rheaume(Boston, MA);*Karishma Tank(New York=\r\n, NY);*Neelima Agrawal(Chicago, IL);*Eliza Webb(Seattle, WA );*Conoly Crave=\r\nns(Atlanta, GA);*Brent Randall(New York, NY);*Alicia Heiser(Spokane, WA);*N=\r\nicole Rodriguez(Sebastian, FL);Sarah Hofschire(Framingham, MA);Aakankhya Pa=\r\ntro(College Station, TX);Kalon Taylor(Memphis, TN);Joyce A(Milford, CT);Nor=\r\na Delay(Bali, Indonesia);Ashleigh Heaton(Astoria, NY);Angie Teates(Washingt=\r\non, DC);Michelle Aclander(Stony Brook, NY);Laura Dominick(Highland Park, NJ=\r\n);Polly Minifie Snyder(Washington, DC);Sarah Minifie Wolfgang(Boston, MA);K=\r\norrie Nickels(Chicago, IL);Kristyn Gelsomini(Houston, TX);Meaghan Horton(Ne=\r\nw York, NY);Dannetta Gibson Ballou(Columbia MD);Suzie Tice(Great Falls, MT)=\r\n;Annette Bani(Limerick, PA);Zoe Berman(Armonk, NY);Rebekah Harper(Raleigh, =\r\nNC);Alessandra Messineo Long(Greenwich, CT);Anne McArthur(Louisville, KY);J=\r\nennifer Wham;Karin Seymour(Fairfield, CT);Lauren DiNicola(New Haven, CT);Lu=\r\ncy Jackoboice(Grand Rapids, MI);Zoe Weiss(New York, NY)\r\n\r\n\r\n\r\n\r\n\r\n\r\nSkimm'd something we missed?\r\n\r\n\r\n\r\nEmail \r\nmailto:SkimmThis@theSkimm=2Ecom \r\nSkimmThis@theSkimm=2Ecom -\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda378a2f7dccb8e2=\r\n6b5b302c761b48c753ae37dd56d06342bd7761205726e794f19c \r\nRead in browser >> \r\n\r\n\r\nSHARE & FOLLOW US\r\n\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda37226dc355defd=\r\n764c4a5bacce1c02696337681417286260dfec3f16ecc90018bc \r\nFacebook\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda37b28f80f69268=\r\n41e373f2063b283fd3960ec2f4f135612ad6b273d8e901e4e7c5 \r\nTwitter\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda3784ba9df91bdb=\r\n28220027be1a35943845c0ff17cf92fc7b3b89cbe9ce6eee6814 \r\nTumblr\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda37254b18b2d0b0=\r\n3609f0ba749107959301d96068b56b4cb840c3dafa14865c8eae \r\nInstagram\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/?qs=3D580bedd5217eda3708ba55eaca8d=\r\nf450cdc2a1ff1d14b5c3865e1d4fe0745f6c13ceab0948df5210 \r\nPinterest\r\n\r\n\r\nCopyright (c) 2016 theSkimm, All rights reserved=2E\r\n\r\n\r\nOur mailing address is: \r\n\r\ntheSkimm Inc=2E\r\n\r\n49 W 23rd Street, 10th Floor\r\n\r\nNew York, NY, 10010, United States\r\n\r\n\r\nhttp://click=2Emorning7=2Etheskimm=2Ecom/profile_center=2Easpx?qs=3Deec37f3=\r\n2b3ca83aeebff6369cdb5b630c8bd88a7221b6c466ba89d97456c40da8f9fd278c3d66e231b=\r\n08d677113d878806c912d8c8d6295d \r\nUpdate Profile \r\nhttp://pages=2Emorning7=2Etheskimm=2Ecom/page=2Easpx?QS=3D773ed3059447707d8=\r\n211c803e40aaa3b9e56b8b07b4622c1&e=3DY2hyaXN0aW5lQHNwYW5nLmNj \r\nUnsubscribe=20\r\n\r\n\r\n\r\n\r\n","2":"\r\n\r\n\r\n\r\n  \r\n    \r\n    \r\n    \r\n    \r\n    \r\n    \r\n\r\n      \r\n\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n    \r\n    Is this email not displaying correctly?\r\n                  View it in your browser=2E\r\n                
\r\n        
\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n                \r\n                \r\n              \r\n            \r\n          
\r\n                  SHARE THIS\r\n                \r\n                  SHARE THIS\r\n                
\r\n        
\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n                  \r\n                    \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                    \r\n                  
\r\n                          3D\"\"\r\n                        
\r\n                          Skimm for November 2nd\r\n                        
\r\n                          3D\"\"\r\n                        
\r\n                
\r\n        
\r\n\r\n  \r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                            Skimm’d w=\r\nhile getting up on what’s at stake in 2016=2E Ready to vote?<=\r\n/a> \r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                            \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                              QUOTE OF THE DAY\r\n                              
\r\n                              
“Moisture harveste=\r\nd from the clouds” - A description of how Sky PA, a new Scottish beer, is ma=\r\nde=2E Cloudy with a chance of hipster=2E
\r\n                              
\r\n                                
\r\n                                
\r\n                                  Skimm This3D\"Like3D\"Tweet\r\n                          3D\"Insta\r\n                                            
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n   \r\n      \r\n         \r\n      \r\n   \r\n
\r\n            \r\n               \r\n                  \r\n                     \r\n                  \r\n               \r\n            
\r\n                        \r\n                           \r\n                              \r\n                                 \r\n                              \r\n                           \r\n                        
\r\n                                    \r\n                                       \r\n                                          \r\n                                             \r\n                                          \r\n                                       \r\n                                    
\r\n\r\n    
GAS PAINS
              =\r\n                               
\r\n                                 
\r\n                     
\r\n         
\r\n      \r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                                                 =\r\n       \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                              
\r\n                                
THE STORY
\r\n
Earlier this week, a major oil pipeline that runs thro=\r\nugh Alabama exploded=2E
\r\n
WAIT…BACK UP=2E
\r\n
The Colonial Pipeline is a system of pipes thousands o=\r\nf miles long that carries millions of barrels of gas, diesel, and jet fuel =\r\na day from Texas to New Jersey=2E It supplies about a third of the Eas=\r\nt Coast’s gas=2E So it’s a BF gassy D=2E But the pipeline’=\r\n;s run into some problems lately=2E Earlier this fall, part of it was shut down f=\r\nor over a week after a leak in Alabama spilled hundreds of thousands of gal=\r\nlons of gas=2E Cue a gas shortage and prices in southern states going up, up, up=\r\n=2E
\r\n
SO WHAT’S THE LATEST?
\r\n
On Monday, there was an explosion along the pipeline i=\r\nn Alabama that caused a major fire=2E One person was killed and several oth=\r\ners were injured=2E Yesterday, Alabama’s governor declared a state of=\r\n emergency to help make sure gas is de=\r\nlivered throughout the state=2E Part of the pipeline’s be=\r\nen shut down and now the company that operates it says it could take days to get=\r\n the whole thing up and running again=2E So for now it looks=\r\n like gas prices could be back on the up and up=2E
\r\n
theSKIMM
\r\n
A major pipeline that millions of people depend on for=\r\n gas every day can’t seem to get off the struggle bus=2E And now this=\r\n latest incident has fuel companies scrambling to stock up on supply=2E If&=\r\n#160;it starts costing more to fill up the tank, this is why=2E
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n   \r\n      \r\n         \r\n      \r\n   \r\n
\r\n            \r\n               \r\n                  \r\n                     \r\n                  \r\n               \r\n            
\r\n                        \r\n                           \r\n                              \r\n                                 \r\n                              \r\n                           \r\n                        
\r\n                                    \r\n                                       \r\n                                          \r\n                                             \r\n                                          \r\n                                       \r\n                                    
\r\n\r\n    
REPEAT AFTER ME=2E=2E=2E
                =\r\n                             
\r\n                                 
\r\n                     
\r\n         
\r\n      \r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY TO =\r\nYOUR FRIEND WHO GREW UP ON DEEP DISH PIZZA…
\r\n
\r\n                              
\r\n                                
In it to win it=2E Las=\r\nt night, the Chicago Cubs tied up the World Series by beating the Cleveland Indian=\r\ns in Game 6=2E They won 9-3=2E Now all eyes are on Game 7 tonight=2E The Cu=\r\nbs haven’t won the series since 1908=2E The Indians haven’t won=\r\n since 1948=2E So it’s not like there’s a lot on the line or an=\r\nything=2E Break out the peanuts=2E
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY WHEN YOU=\r\n FIND OUT YOUR TWO FRIENDS WHO YOU INTRODUCED ARE GETTING TOGETHER=\r\n0;WITHOUT YOU=2E=2E=2E
\r\n                              
\r\n                                
What’s going on =\r\nhere? Earlier this week, a federal judge ordered the Republican party to explain a=\r\nny deals it may have made with GOP nominee Donald Trump’s campai=\r\ngn to monitor the polls during this election season=2E Reminder: for months=\r\n, Trump has been saying this election is “rigged” against hi=\r\nm, and encouraging his supporters to be on the lookout for voter fraud=2E E=\r\narly voting opened recently in a lot of states=2E And there have been repor=\r\nts of Trump supporters allegedly photographing and harassing people at the polls=\r\n=2E That’s according to a string of different lawsuits in Arizon=\r\na, Ohio, and Nevada flagging concerns that some voters are being intim=\r\nidated=2E Last week, the Democratic party filed a separate complaint with a=\r\n federal court=2E Now, the Republican party has until the end of today to t=\r\nurn over any evidence that could be related to collaborating with the =\r\nTrump campaign=2E Stay tuned=2E
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY TO WHEN =\r\nYOUR AM TRAIN GETS DELAYED FOR THE FIFTH TIME…
\r\n                              
\r\n                                
I give up=2E Yesterday=\r\n, Russia put Syrian peace talks on hold=2E Indefinitely=2E For years, Syria has be=\r\nen going through a violent civil war between Syrian President Bashar al-Ass=\r\nad (backed by Russia) and rebel groups (some backed by the US) who want him=\r\n out of power=2E Hundreds of thousands of people have been killed=2E Millio=\r\nns have been forced to leave home, causing the EU’s worst migrant and=\r\n refugee crisis since World War II=2E The two sides have been trying and fa=\r\niling to hash out a peace deal for a while now=2E Earlier this month, Russi=\r\na agreed to a “humanitarian pause” to airstrikes in Aleppo - the key r=\r\nebel stronghold where a lot of the fighting has been focused=2E Meanwh=\r\nile, rebel’s have been fighting back in the city=2E And today, Russia a=\r\nnnounced it would give rebels even more time to leave Aleppo=2E But it’=\r\ns still walking away from the peace table for the foreseeable future=2E So,=\r\n Syria’s still a peace of work=2E
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY WHEN YOU=\r\nR DATE SAYS HE NEEDS A NOISE MACHINE ON HIGH VOLUME TO FALL ASLEEP=2E=2E=2E=\r\n
\r\n                              
\r\n                                
Having second thoughts=\r\n=2E That’s what Gannett is telling Tronc (the artist formerly known a=\r\ns Tribune Publishing)=2E For months, Gannett, which owns about a hundred ne=\r\nws outlets including USA Today, has been trying to buy Tronc, which owns names=\r\n0;like the LA Times and Chicago Tribune=2E For years, print news companies =\r\nhave been struggling to keep ad dollars and readers on board=2E Gannet=\r\nt was hoping that the power of two media companies combined would be more a=\r\nttractive than one, especially to advertisers=2E The deal would have c=\r\nreated one of the largest media groups in the country=2E So earlier this ye=\r\nar, Gannett saddled up to Tronc with a buyout offer of around $400 million=\r\n=2E Tronc said ‘no, thanks, we’re not that cheap=2E’ Yest=\r\nerday, after a lot of back and forth, Gannett walked away from the deal=2E Th=\r\nis might also be because Gannett had ‘meh’ earnings last q=\r\nuarter, meaning they couldn’t get the cash together to write the chec=\r\nk=2E Either way, both companies are still single AF=2E
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY =\r\nTO YOUR FRIEND WHO ONLY DOES SQUATS AT THE GYM =2E=2E=2E
\r\n                              
\r\n                                
Say goodbye to your fave emoji=2E=\r\n
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                                                          \r\n                                \r\n                              \r\n                                                      \r\n                        
\r\n                                                                           =\r\n                     
SKIMM THE VOTE
\r\n                                                              
\r\n                                \r\n                                  \r\n                                    \r\n                                      \r\n                                    \r\n                                    \r\n                                      \r\n                                  \r\n                                
\r\n                                        \r\n                                      
\r\n                                    
\r\n                              
                            
\r\n                                                                
In case you somehow missed it, yo=\r\nu have to vote on Tuesday=2E Catch up on who̵=\r\n7;s on your ballot and what’s at stake=2E Then plan your trip to the =\r\npolls=2E Questions, answered=2E 
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This3D\"Like3D\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                                  3D\"Pledge=\r\n\r\n                                
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                                     =\r\n                   \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                                           =\r\n               
SKIMM 50
\r\n                                                            
\r\n                                                                
Thanks to these Skimm’bassadors for sharing like itR=\r\n17;s hot=2E Want to see your name on this list? Click here to learn more=2E =\r\n
\r\n
Olivia Simonson, Britt=\r\nany Berger, Daniela Rivas, Ashley Lawson, Sarah Davidson, L Denease Thompso=\r\nn-Mack, Noreen Deutsch, Jessee Fordham, Natasha Shah, Shari Berga, Aloke Pr=\r\nabhu, Natalie Tenzer, Samantha Panchevre, Ania Arsenowicz, Ali Wozniak, Rei=\r\nnalyn, Taunya Robinson, Cassie Christensen, Spencer Philips, Abby Bilinski,=\r\n Aner Zhou, Julia Erdenebold, Mehreen Mazhar, Alexandra Rizzo, Alisa Sutton=\r\n, Maegan Detlefs, Janina Ancona, Chris Drake, Allan Moss, Jordan Murray, Da=\r\nvid Benjamin, Leslie Bartula, Hana Muslic, Kelly Wallace, Leslie Buteyn, Be=\r\nthany Fitzgerald, Stuart Ferguson, Emily Paulino, Allison Ryan, Niki DeMaio=\r\n, Ria Conti, Nikki Boyd, Amanda Oliver, Lindsay Barthel, Shelby Wynn, Britt=\r\nany Daunno, Emily Rissmiller, Alex Johannes, Andrea Borod, Ashley Montufar,=\r\n Karli Von Herbulis, Lexie Rindler, Natalie Weaver, Allison Sotelo, Cynthia=\r\n Lopez, Isabel Taylor, Jess Tolan, Kasie Heiden, Kelsey Will, Leslie McFayd=\r\nen, Morgan Balavage, Morgan Goracke, Madeline Trainor, Alexa Parisi, Uma Su=\r\ndarshan, Brooke Borkowski, Connie Lin, Jess Ridolfino, Jessica Foster, Mall=\r\nory Mickus, Sapreen Abbass, Kelli Holland
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                                                                           =\r\n         \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                              
SKIMM SHARE
\r\n                              
\r\n                                                                           =\r\n                     
Halfway there=2E Share theSkimm w=\r\nith your work wife who always takes a coffee break when you need it=2E=\r\n
\r\n
\r\n                              
\r\n                                3D\"Shar=\r\ne\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      
\r\n        \r\n          \r\n            \r\n              
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n    \r\n    \r\n    \r\n    \r\n\r\n      \r\n\r\n\r\n  
\r\n                                                            \r\n                                                                           =\r\n               
SKIMM BIRTHDAYS
\r\n                                                            
\r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n    \r\n    Is this email not displaying correctly?\r\n                  View it in your browser.\r\n                
\r\n        
\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n                \r\n                \r\n              \r\n            \r\n          
\r\n                  SHARE THIS\r\n                \r\n                  SHARE THIS\r\n                
\r\n        
\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n                  \r\n                    \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                    \r\n                  
\r\n                          \"\"\r\n                        
\r\n                          Skimm for November 2nd\r\n                        
\r\n                          \"\"\r\n                        
\r\n                
\r\n        
\r\n\r\n  \r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                            Skimm’d while getting up on what’s at stake in 2016. Ready to vote? \r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                            \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                              QUOTE OF THE DAY\r\n                              
\r\n                              
“Moisture harvested from the clouds” - A description of how Sky PA, a new Scottish beer, is made. Cloudy with a chance of hipster.
\r\n                              
\r\n                                
\r\n                                
\r\n                                  Skimm This\"Like\"Tweet\r\n                          \"Insta\r\n                                            
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n   \r\n      \r\n         \r\n      \r\n   \r\n
\r\n            \r\n               \r\n                  \r\n                     \r\n                  \r\n               \r\n            
\r\n                        \r\n                           \r\n                              \r\n                                 \r\n                              \r\n                           \r\n                        
\r\n                                    \r\n                                       \r\n                                          \r\n                                             \r\n                                          \r\n                                       \r\n                                    
\r\n\r\n    
GAS PAINS
                                             
\r\n                                 
\r\n                     
\r\n         
\r\n      \r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                              
                            
\r\n                                
THE STORY
\r\n
Earlier this week, a major oil pipeline that runs through Alabama exploded.
\r\n
WAIT…BACK UP.
\r\n
The Colonial Pipeline is a system of pipes thousands of miles long that carries millions of barrels of gas, diesel, and jet fuel a day from Texas to New Jersey. It supplies about a third of the East Coast’s gas. So it’s a BF gassy D. But the pipeline’s run into some problems lately. Earlier this fall, part of it was shut down for over a week after a leak in Alabama spilled hundreds of thousands of gallons of gas. Cue a gas shortage and prices in southern states going up, up, up.
\r\n
SO WHAT’S THE LATEST?
\r\n
On Monday, there was an explosion along the pipeline in Alabama that caused a major fire. One person was killed and several others were injured. Yesterday, Alabama’s governor declared a state of emergency to help make sure gas is delivered throughout the state. Part of the pipeline’s been shut down and now the company that operates it says it could take days to get the whole thing up and running again. So for now it looks like gas prices could be back on the up and up.
\r\n
theSKIMM
\r\n
A major pipeline that millions of people depend on for gas every day can’t seem to get off the struggle bus. And now this latest incident has fuel companies scrambling to stock up on supply. If it starts costing more to fill up the tank, this is why.
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n   \r\n      \r\n         \r\n      \r\n   \r\n
\r\n            \r\n               \r\n                  \r\n                     \r\n                  \r\n               \r\n            
\r\n                        \r\n                           \r\n                              \r\n                                 \r\n                              \r\n                           \r\n                        
\r\n                                    \r\n                                       \r\n                                          \r\n                                             \r\n                                          \r\n                                       \r\n                                    
\r\n\r\n    
REPEAT AFTER ME...
                                             
\r\n                                 
\r\n                     
\r\n         
\r\n      \r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY TO YOUR FRIEND WHO GREW UP ON DEEP DISH PIZZA…
\r\n
\r\n                              
\r\n                                
In it to win it. Last night, the Chicago Cubs tied up the World Series by beating the Cleveland Indians in Game 6. They won 9-3. Now all eyes are on Game 7 tonight. The Cubs haven’t won the series since 1908. The Indians haven’t won since 1948. So it’s not like there’s a lot on the line or anything. Break out the peanuts.
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY WHEN YOU FIND OUT YOUR TWO FRIENDS WHO YOU INTRODUCED ARE GETTING TOGETHER WITHOUT YOU...
\r\n                              
\r\n                                
What’s going on here? Earlier this week, a federal judge ordered the Republican party to explain any deals it may have made with GOP nominee Donald Trump’s campaign to monitor the polls during this election season. Reminder: for months, Trump has been saying this election is “rigged” against him, and encouraging his supporters to be on the lookout for voter fraud. Early voting opened recently in a lot of states. And there have been reports of Trump supporters allegedly photographing and harassing people at the polls. That’s according to a string of different lawsuits in Arizona, Ohio, and Nevada flagging concerns that some voters are being intimidated. Last week, the Democratic party filed a separate complaint with a federal court. Now, the Republican party has until the end of today to turn over any evidence that could be related to collaborating with the Trump campaign. Stay tuned.
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY TO WHEN YOUR AM TRAIN GETS DELAYED FOR THE FIFTH TIME…
\r\n                              
\r\n                                
I give up. Yesterday, Russia put Syrian peace talks on hold. Indefinitely. For years, Syria has been going through a violent civil war between Syrian President Bashar al-Assad (backed by Russia) and rebel groups (some backed by the US) who want him out of power. Hundreds of thousands of people have been killed. Millions have been forced to leave home, causing the EU’s worst migrant and refugee crisis since World War II. The two sides have been trying and failing to hash out a peace deal for a while now. Earlier this month, Russia agreed to a “humanitarian pause” to airstrikes in Aleppo - the key rebel stronghold where a lot of the fighting has been focused. Meanwhile, rebel’s have been fighting back in the city. And today, Russia announced it would give rebels even more time to leave Aleppo. But it’s still walking away from the peace table for the foreseeable future. So, Syria’s still a peace of work.
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY WHEN YOUR DATE SAYS HE NEEDS A NOISE MACHINE ON HIGH VOLUME TO FALL ASLEEP...
\r\n                              
\r\n                                
Having second thoughts. That’s what Gannett is telling Tronc (the artist formerly known as Tribune Publishing). For months, Gannett, which owns about a hundred news outlets including USA Today, has been trying to buy Tronc, which owns names like the LA Times and Chicago Tribune. For years, print news companies have been struggling to keep ad dollars and readers on board. Gannett was hoping that the power of two media companies combined would be more attractive than one, especially to advertisers. The deal would have created one of the largest media groups in the country. So earlier this year, Gannett saddled up to Tronc with a buyout offer of around $400 million. Tronc said ‘no, thanks, we’re not that cheap.’ Yesterday, after a lot of back and forth, Gannett walked away from the deal. This might also be because Gannett had ‘meh’ earnings last quarter, meaning they couldn’t get the cash together to write the check. Either way, both companies are still single AF.
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                
WHAT TO SAY TO YOUR FRIEND WHO ONLY DOES SQUATS AT THE GYM ...
\r\n                              
\r\n                                
Say goodbye to your fave emoji.
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                                                          \r\n                                \r\n                              \r\n                                                      \r\n                        
\r\n                                                                                                
SKIMM THE VOTE
\r\n                                                              
\r\n                                \r\n                                  \r\n                                    \r\n                                      \r\n                                    \r\n                                    \r\n                                      \r\n                                    \r\n                                  \r\n                                
\r\n                                        \r\n                                      
\r\n                              
                            
\r\n                                                                
In case you somehow missed it, you have to vote on Tuesday. Catch up on who’s on your ballot and what’s at stake. Then plan your trip to the polls. Questions, answered. 
\r\n                              
\r\n                                
\r\n                                  
\r\n                                    
\r\n                                      Skimm This\"Like\"Tweet\r\n                                                  
\r\n                                  
\r\n                                
\r\n                              
\r\n                                  \"Pledge\r\n                                
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                                                          
SKIMM 50
\r\n                                                            
                            
\r\n                                                                
Thanks to these Skimm’bassadors for sharing like it’s hot. Want to see your name on this list? Click here to learn more. 
\r\n
Olivia Simonson, Brittany Berger, Daniela Rivas, Ashley Lawson, Sarah Davidson, L Denease Thompson-Mack, Noreen Deutsch, Jessee Fordham, Natasha Shah, Shari Berga, Aloke Prabhu, Natalie Tenzer, Samantha Panchevre, Ania Arsenowicz, Ali Wozniak, Reinalyn, Taunya Robinson, Cassie Christensen, Spencer Philips, Abby Bilinski, Aner Zhou, Julia Erdenebold, Mehreen Mazhar, Alexandra Rizzo, Alisa Sutton, Maegan Detlefs, Janina Ancona, Chris Drake, Allan Moss, Jordan Murray, David Benjamin, Leslie Bartula, Hana Muslic, Kelly Wallace, Leslie Buteyn, Bethany Fitzgerald, Stuart Ferguson, Emily Paulino, Allison Ryan, Niki DeMaio, Ria Conti, Nikki Boyd, Amanda Oliver, Lindsay Barthel, Shelby Wynn, Brittany Daunno, Emily Rissmiller, Alex Johannes, Andrea Borod, Ashley Montufar, Karli Von Herbulis, Lexie Rindler, Natalie Weaver, Allison Sotelo, Cynthia Lopez, Isabel Taylor, Jess Tolan, Kasie Heiden, Kelsey Will, Leslie McFayden, Morgan Balavage, Morgan Goracke, Madeline Trainor, Alexa Parisi, Uma Sudarshan, Brooke Borkowski, Connie Lin, Jess Ridolfino, Jessica Foster, Mallory Mickus, Sapreen Abbass, Kelli Holland
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                                                                                    \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                              
SKIMM SHARE
\r\n                              
\r\n                                                                                                
Halfway there. Share theSkimm with your work wife who always takes a coffee break when you need it.
\r\n
\r\n                              
\r\n                                \"Share\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n  \r\n    \r\n      \r\n    \r\n  \r\n
\r\n        \r\n          \r\n            \r\n              \r\n            \r\n          \r\n        
\r\n                \r\n                  \r\n                    \r\n                      \r\n                    \r\n                    \r\n                      \r\n                    \r\n                  \r\n                
\r\n                        \r\n                          \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                            \r\n                              \r\n                            \r\n                                                        \r\n                              \r\n                            \r\n                          \r\n                        
\r\n                                                            \r\n                                                                                          
SKIMM BIRTHDAYS
\r\n                                                            
                            
\r\n                                                                
* indicates Skimm’bassador. Hammer time.
\r\n
Louise Cronin (San Francisco, CA); Faith Greenberg (Longwood, FL); *Hatley Thompson (Washington, DC); *Monique Ervine (Kindersley, Canada); *Leana Macrito (Chicago, IL); *AnnMarie Murtaugh (Houston, TX); *Maria Perwerton (Rochester, MI); *Lauren Valainis (Washington, DC); *Jaycie Moller (San Francisco, CA); *Erin Manfull (Iowa City, IA); *Jennifer Rheaume (Boston, MA); *Karishma Tank (New York, NY); *Neelima Agrawal (Chicago, IL); *Eliza Webb (Seattle, WA ); *Conoly Cravens (Atlanta, GA); *Brent Randall (New York, NY); *Alicia Heiser (Spokane, WA); *Nicole Rodriguez (Sebastian, FL); Sarah Hofschire (Framingham, MA); Aakankhya Patro (College Station, TX); Kalon Taylor (Memphis, TN); Joyce A (Milford, CT); Nora Delay (Bali, Indonesia); Ashleigh Heaton (Astoria, NY); Angie Teates (Washington, DC); Michelle Aclander (Stony Brook, NY); Laura Dominick (Highland Park, NJ); Polly Minifie Snyder (Washington, DC); Sarah Minifie Wolfgang (Boston, MA); Korrie Nickels (Chicago, IL); Kristyn Gelsomini (Houston, TX); Meaghan Horton (New York, NY); Dannetta Gibson Ballou (Columbia MD); Suzie Tice (Great Falls, MT); Annette Bani (Limerick, PA); Zoe Berman (Armonk, NY); Rebekah Harper (Raleigh, NC); Alessandra Messineo Long (Greenwich, CT); Anne McArthur (Louisville, KY); Jennifer Wham; Karin Seymour (Fairfield, CT); Lauren DiNicola (New Haven, CT); Lucy Jackoboice (Grand Rapids, MI); Zoe Weiss (New York, NY)
\r\n                              
\r\n                      
\r\n              
\r\n      
\r\n\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n                  \r\n                    \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                      \r\n                        \r\n                      \r\n                    \r\n                  
\r\n                          Skimm'd something we missed?\r\n                          
\r\n                          
\r\n                          Email SkimmThis@theSkimm.com  • Read in browser »\r\n                        
\r\n                
\r\n        
\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n                  \r\n                    \r\n                      \r\n                        \r\n                      \r\n                    \r\n                  
\r\n                          \r\n                            \r\n                              \r\n                                \r\n                              \r\n                            \r\n                          
\r\n                                  SHARE & FOLLOW US\r\n                                
\r\n                        
\r\n                  \r\n                    \r\n                      \r\n                        \r\n                      \r\n                    \r\n                  
\r\n                          \r\n                            \r\n                              \r\n                                \r\n                              \r\n                              \r\n                                \r\n                              \r\n                            \r\n                          
\r\n                                  \r\n                                    \r\n                                      \r\n                                        \r\n                                      \r\n                                    \r\n                                  
\r\n                                          \"LikeFacebook\r\n                                          \"TweetTwitter\r\n                                          \"TumbleTumblr\r\n                                          \"InstagramInstagram\r\n                                          \"PinPinterest\r\n                                        
\r\n                                
\r\n                        
\r\n                
\r\n        
\r\n\r\n  \r\n    \r\n      \r\n        \r\n      \r\n    \r\n  
\r\n          \r\n            \r\n              \r\n                \r\n              \r\n            \r\n          
\r\n                  \r\n                    \r\n                      \r\n                        \r\n                      \r\n                    \r\n                  
\r\n                          \r\n                            \r\n                              \r\n                                \r\n                              \r\n                              \r\n                                \r\n                              \r\n                              \r\n                                \r\n                              \r\n                            \r\n                          
\r\n                                  \r\n                                    \r\n                                      \r\n                                        \r\n                                      \r\n                                      \r\n                                        \r\n                                      \r\n                                      \r\n                                        \r\n                                      \r\n                                      \r\n                                        \r\n                                      \r\n                                      \r\n                                        \r\n                                      \r\n                                      \r\n                                        \r\n                                      \r\n                                    \r\n                                  
\r\n                                          Copyright © 2016 theSkimm, All rights reserved.\r\n                                        
\r\n                                          Our mailing address is: 
\r\n                                          theSkimm Inc.
\r\n                                          49 W 23rd Street, 10th Floor
\r\n                                          New York, NY, 10010, United States\r\n                                        
\r\n                                          Update Profile
\r\n            Unsubscribe\r\n                                        
\r\n                                
\r\n                        
\r\n                
\r\n        
\r\n\r\n\r\n\"Advertisement\"\r\n\"Advertisement\"\t\r\n\"Advertisement\"\r\n\r\n\r\n\r\n","snippet":" Is this email not displaying correctly? http://click.morning7.theskimm.com/?qs=580bedd5217eda378a2f7dccb8e26b5b302c761b48c753ae37dd56d06342bd7761205726e794f19c","unread":false,"starred":false,"date":"2016-11-02T10:21:52.000Z","folderImapUID":343848,"folderId":"test-folder-id","folder":{"id":"test-folder-id","account_id":"test-account-id","object":"folder","name":null,"display_name":"Test Folder"},"labels":[],"headers":{"delivered-to":["christine@spang.cc"],"received":["by 10.31.236.3 with SMTP id k3csp698913vkh; Wed, 2 Nov 2016 03:21:52 -0700 (PDT)","from mta.morning7.theskimm.com (mta.morning7.theskimm.com. [136.147.177.13]) by mx.google.com with ESMTPS id j59si882394qtb.16.2016.11.02.03.21.50 for  (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Wed, 02 Nov 2016 03:21:52 -0700 (PDT)","by mta.morning7.theskimm.com id h36v3s163hst for ; Wed, 2 Nov 2016 10:21:47 +0000 (envelope-from )"],"x-received":["by 10.55.47.193 with SMTP id v184mr2167122qkh.259.1478082112295; Wed, 02 Nov 2016 03:21:52 -0700 (PDT)"],"return-path":[""],"received-spf":["pass (google.com: domain of bounce-20_html-215009-5489-7208679-430@bounce.morning7.theskimm.com designates 136.147.177.13 as permitted sender) client-ip=136.147.177.13;"],"authentication-results":["mx.google.com; dkim=pass header.i=@morning7.theskimm.com; spf=pass (google.com: domain of bounce-20_html-215009-5489-7208679-430@bounce.morning7.theskimm.com designates 136.147.177.13 as permitted sender) smtp.mailfrom=bounce-20_HTML-215009-5489-7208679-430@bounce.morning7.theskimm.com; dmarc=pass (p=NONE dis=NONE) header.from=theskimm.com"],"dkim-signature":["v=1; a=rsa-sha1; c=relaxed/relaxed; s=200608; d=morning7.theskimm.com; h=From:To:Subject:Date:List-Unsubscribe:MIME-Version:Reply-To:List-ID:Message-ID:Content-Type; i=dailyskimm@morning7.theskimm.com; bh=TfZNUfri1El8i4AUulbXxJgLUCA=; b=H6ZRxb2e2tmobb63CIcQ1ecsfv+X+Ky/G0EW0Kct6kvNmLP/yASdttr9bcgRbUMO45Su7bytynrE U+kbevaHw5nbatKKWQFP7JxGMjzCtQGM6LzOjGV8qCSu7XwdQ0QgUCJo0V4eo7Iu6bafR9h2WJST ct12e4elnzHFZLnF7GM="],"from":["\"theSkimm\" "],"to":[""],"subject":["Daily Skimm: Hey batter batter"],"date":["Wed, 02 Nov 2016 04:21:46 -0600"],"list-unsubscribe":[""],"mime-version":["1.0"],"reply-to":["\"theSkimm\" "],"list-id":["<7208679_5489.xt.local>"],"x-csa-complaints":["whitelistcomplaints@eco.de"],"x-job":["7208679_5489"],"message-id":[""],"content-type":["multipart/alternative; boundary=\"Aw0OfBZoDhFa=_?:\""],"x-gm-thrid":"1549881429115204149","x-gm-msgid":"1549881429115204149","x-gm-labels":["\\Important"]},"headerMessageId":"","subject":"Daily Skimm: Hey batter batter","folderImapXGMLabels":"[\"\\\\Important\"]"}}
\ No newline at end of file
diff --git a/packages/local-sync/spec/message-factory-spec.js b/packages/local-sync/spec/message-factory-spec.js
index bd2ea44f0..ae529090f 100644
--- a/packages/local-sync/spec/message-factory-spec.js
+++ b/packages/local-sync/spec/message-factory-spec.js
@@ -26,10 +26,17 @@ describe('MessageFactory', function MessageFactorySpecs() {
     forEachJSONFixture('MessageFactory/parseFromImap', (filename, json) => {
       it(`should correctly build message properties for ${filename}`, () => {
         const {imapMessage, desiredParts, result} = json;
+        // requiring these to match makes it overly arduous to generate test
+        // cases from real accounts
+        const excludeKeys = new Set(['id', 'accountId', 'folderId', 'folder', 'labels']);
 
         waitsForPromise(async () => {
           const actual = await parseFromImap(imapMessage, desiredParts, this.options);
-          expect(actual).toEqual(result)
+          for (const key of Object.keys(result)) {
+            if (!excludeKeys.has(key)) {
+              expect(actual[key]).toEqual(result[key]);
+            }
+          }
         });
       });
     })
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index f6a628cc9..2a745b3ed 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -1,4 +1,7 @@
 const _ = require('underscore');
+const os = require('os');
+const fs = require('fs');
+const mkdirp = require('mkdirp');
 
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core');
 const {Capabilities} = IMAPConnection;
@@ -265,6 +268,11 @@ class FetchMessagesInFolder {
         imapMessage,
         desiredParts,
       }, `FetchMessagesInFolder: Could not build message`)
+      const outJSON = JSON.stringify({'imapMessage': imapMessage, 'desiredParts': desiredParts, 'result': {}});
+      const outDir = path.join(os.tmpdir(), "k2-parse-errors", this._folder.name)
+      const outFile = path.join(outDir, imapMessage.attributes.uid.toString());
+      mkdirp.sync(outDir);
+      fs.writeFileSync(outFile, outJSON);
     }
   }
 
diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index 61375e801..54fb09540 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -1,7 +1,9 @@
 const utf7 = require('utf7').imap;
 const mimelib = require('mimelib');
 const QuotedPrintable = require('quoted-printable');
-const {Imap} = require('isomorphic-core')
+const {Imap} = require('isomorphic-core');
+const mkdirp = require('mkdirp');
+const striptags = require('striptags');
 
 const SNIPPET_SIZE = 100
 
@@ -59,6 +61,13 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
     subject: parsedHeaders.subject[0],
   }
 
+  // preserve whitespacing on plaintext emails -- has the side effect of monospacing, but
+  // that seems OK and perhaps sometimes even desired (for e.g. ascii art, alignment)
+  if (!body['text/html'] && body['text/plain']) {
+    values.body = `
${values.body}
`;
+  }
+
+  // TODO: strip quoted text from snippets also
   if (values.snippet) {
     // trim and clean snippet which is alreay present (from values plaintext)
     values.snippet = values.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ')
@@ -68,8 +77,7 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
     }
   } else if (values.body) {
     // create snippet from body, which is most likely html
-    // TODO: Fanciness
-    values.snippet = values.body.substr(0, Math.min(values.body.length, SNIPPET_SIZE));
+    values.snippet = striptags(values.body).trim().substr(0, Math.min(values.body.length, SNIPPET_SIZE));
   }
 
   values.folder = folder

From 718cdbe9ae98bfd1245b110512fd57f80e68d379 Mon Sep 17 00:00:00 2001
From: Christine Spang 
Date: Wed, 7 Dec 2016 07:25:48 -0800
Subject: [PATCH 476/800] Enable nodejs completion with tern

---
 .tern-project | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .tern-project

diff --git a/.tern-project b/.tern-project
new file mode 100644
index 000000000..2f4ecf04e
--- /dev/null
+++ b/.tern-project
@@ -0,0 +1,5 @@
+{
+    "plugins": {
+        "node": {}
+    }
+}

From edbf869ff7e4efd607b70c1201618f90aa369575 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 7 Dec 2016 10:10:34 -0800
Subject: [PATCH 477/800] [isomorphic-core] add more auth error states

---
 packages/isomorphic-core/src/auth-helpers.js | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/packages/isomorphic-core/src/auth-helpers.js b/packages/isomorphic-core/src/auth-helpers.js
index d9f9b8e68..45e1f1dc6 100644
--- a/packages/isomorphic-core/src/auth-helpers.js
+++ b/packages/isomorphic-core/src/auth-helpers.js
@@ -25,7 +25,11 @@ const exchangeSettings = Joi.object().keys({
   eas_server_host: [Joi.string().ip().required(), Joi.string().hostname().required()],
 }).required();
 
-const AUTH_500_USER_MESSAGE = "Please contact support@nylas.com. An unforseen error has occurred."
+const USER_ERRORS = {
+  AUTH_500: "Please contact support@nylas.com. An unforseen error has occurred.",
+  IMAP_AUTH: "Incorrect username or password",
+  IMAP_RETRY: "We were unable to reach your mail provider. Please try again.",
+}
 
 module.exports = {
   imapAuthRouteConfig() {
@@ -104,11 +108,14 @@ module.exports = {
         return reply(JSON.stringify(response));
       })
       .catch((err) => {
-        if (err instanceof IMAPErrors.IMAPAuthenticationError) {
-          return reply({message: err.message, type: "api_error"}).code(401);
-        }
         request.logger.error(err)
-        return reply({message: AUTH_500_USER_MESSAGE, type: "api_error"}).code(500);
+        if (err instanceof IMAPErrors.IMAPAuthenticationError) {
+          return reply({message: USER_ERRORS.IMAP_AUTH, type: "api_error"}).code(401);
+        }
+        if (err instanceof IMAPErrors.RetryableError) {
+          return reply({message: USER_ERRORS.IMAP_RETRY, type: "api_error"}).code(408);
+        }
+        return reply({message: USER_ERRORS.AUTH_500, type: "api_error"}).code(500);
       })
     }
   },

From 8307976df881ccd2009f154c65323a772f378c74 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 7 Dec 2016 13:39:41 -0800
Subject: [PATCH 478/800] [local-sync]: Fix label/folder id creation

We were using the stripped version of label/folder names for the id
hash, e.g. [Gmail]/Drafts would be Drafts.
However, we can't do this because it might collide with other names. e.g. if the
user created a Drafts label, it would end up colliding with [Gmail]/Drafts

Minor lint fix
---
 .../src/local-sync-worker/imap/fetch-messages-in-folder.js     | 1 +
 packages/local-sync/src/models/folder.js                       | 3 +--
 packages/local-sync/src/models/label.js                        | 3 +--
 .../local-sync/src/new-message-processor/extract-contacts.js   | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 2a745b3ed..256e71b03 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -1,6 +1,7 @@
 const _ = require('underscore');
 const os = require('os');
 const fs = require('fs');
+const path = require('path')
 const mkdirp = require('mkdirp');
 
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core');
diff --git a/packages/local-sync/src/models/folder.js b/packages/local-sync/src/models/folder.js
index cea714496..885f2547f 100644
--- a/packages/local-sync/src/models/folder.js
+++ b/packages/local-sync/src/models/folder.js
@@ -28,8 +28,7 @@ module.exports = (sequelize, Sequelize) => {
       },
 
       hash({boxName, accountId}) {
-        const cleanName = formatImapPath(boxName)
-        return crypto.createHash('sha256').update(`${accountId}${cleanName}`, 'utf8').digest('hex')
+        return crypto.createHash('sha256').update(`${accountId}${boxName}`, 'utf8').digest('hex')
       },
     },
     instanceMethods: {
diff --git a/packages/local-sync/src/models/label.js b/packages/local-sync/src/models/label.js
index a893b9830..077a7a306 100644
--- a/packages/local-sync/src/models/label.js
+++ b/packages/local-sync/src/models/label.js
@@ -43,8 +43,7 @@ module.exports = (sequelize, Sequelize) => {
       },
 
       hash({boxName, accountId}) {
-        const cleanName = formatImapPath(boxName)
-        return crypto.createHash('sha256').update(`${accountId}${cleanName}`, 'utf8').digest('hex')
+        return crypto.createHash('sha256').update(`${accountId}${boxName}`, 'utf8').digest('hex')
       },
     },
     instanceMethods: {
diff --git a/packages/local-sync/src/new-message-processor/extract-contacts.js b/packages/local-sync/src/new-message-processor/extract-contacts.js
index 34b9c70ed..b3220dc31 100644
--- a/packages/local-sync/src/new-message-processor/extract-contacts.js
+++ b/packages/local-sync/src/new-message-processor/extract-contacts.js
@@ -28,10 +28,10 @@ async function extractContacts({db, message}) {
       const id = cryptography.createHash('sha256').update(c.email, 'utf8').digest('hex');
       const existing = await db.Contact.findById(id);
       const cdata = {
+        id,
         name: c.name,
         email: c.email,
         accountId: message.accountId,
-        id: id,
       };
 
       if (!existing) {

From 56f8d41b8ceb6200406417e7c6dab8bbc4f46e82 Mon Sep 17 00:00:00 2001
From: Halla Moore 
Date: Wed, 7 Dec 2016 14:06:07 -0800
Subject: [PATCH 479/800] [local-sync] feat(send): Add support for attachments

Also move some helper function logic onto the Message model
---
 .../local-sync/src/local-api/routes/send.js   |   8 +-
 .../local-sync/src/local-api/sending-utils.js | 104 +++---------------
 .../src/local-api/sendmail-client.js          |  20 +++-
 packages/local-sync/src/models/message.js     |  92 ++++++++++++++++
 4 files changed, 130 insertions(+), 94 deletions(-)

diff --git a/packages/local-sync/src/local-api/routes/send.js b/packages/local-sync/src/local-api/routes/send.js
index 42cf7aaf8..d31c37542 100644
--- a/packages/local-sync/src/local-api/routes/send.js
+++ b/packages/local-sync/src/local-api/routes/send.js
@@ -81,7 +81,13 @@ module.exports = (server) => {
       try {
         const accountId = request.auth.credentials.id;
         const db = await LocalDatabaseConnector.forAccount(accountId)
-        const draft = await SendingUtils.findOrCreateMessageFromJSON(request.payload, db, false)
+        const draftData = Object.assign(request.payload, {
+          unread: true,
+          is_draft: false,
+          is_sent: false,
+          version: 0,
+        })
+        const draft = await SendingUtils.findOrCreateMessageFromJSON(draftData, db)
         await (draft.isSending = true);
         const savedDraft = await draft.save();
         reply(savedDraft.toJSON());
diff --git a/packages/local-sync/src/local-api/sending-utils.js b/packages/local-sync/src/local-api/sending-utils.js
index 869f8045f..2c3d0905f 100644
--- a/packages/local-sync/src/local-api/sending-utils.js
+++ b/packages/local-sync/src/local-api/sending-utils.js
@@ -1,16 +1,3 @@
-const _ = require('underscore');
-
-const setReplyHeaders = (newMessage, prevMessage) => {
-  if (prevMessage.messageIdHeader) {
-    newMessage.inReplyTo = prevMessage.headerMessageId;
-    if (prevMessage.references) {
-      newMessage.references = prevMessage.references.concat(prevMessage.headerMessageId);
-    } else {
-      newMessage.references = [prevMessage.messageIdHeader];
-    }
-  }
-}
-
 class HTTPError extends Error {
   constructor(message, httpCode, logContext) {
     super(message);
@@ -21,90 +8,25 @@ class HTTPError extends Error {
 
 module.exports = {
   HTTPError,
-  findOrCreateMessageFromJSON: async (data, db, isDraft) => {
-    const {Thread, Message} = db;
+  setReplyHeaders: (newMessage, prevMessage) => {
+    if (prevMessage.messageIdHeader) {
+      newMessage.inReplyTo = prevMessage.headerMessageId;
+      if (prevMessage.references) {
+        newMessage.references = prevMessage.references.concat(prevMessage.headerMessageId);
+      } else {
+        newMessage.references = [prevMessage.messageIdHeader];
+      }
+    }
+  },
+  findOrCreateMessageFromJSON: async (data, db) => {
+    const {Message} = db;
 
     const existingMessage = await Message.findById(data.id);
     if (existingMessage) {
       return existingMessage;
     }
 
-    const {to, cc, bcc, from, replyTo, subject, body, account_id, date, id} = data;
-
-    const message = Message.build({
-      accountId: account_id,
-      from: from,
-      to: to,
-      cc: cc,
-      bcc: bcc,
-      replyTo: replyTo,
-      subject: subject,
-      body: body,
-      unread: true,
-      isDraft: isDraft,
-      isSent: false,
-      version: 0,
-      date: date,
-      id: id,
-    });
-
-    // TODO
-    // Attach files
-    // Update our contact list
-    // Add events
-    // Add metadata??
-
-    let replyToThread;
-    let replyToMessage;
-    if (data.thread_id != null) {
-      replyToThread = await Thread.find({
-        where: {id: data.thread_id},
-        include: [{
-          model: Message,
-          as: 'messages',
-          attributes: _.without(Object.keys(Message.attributes), 'body'),
-        }],
-      });
-    }
-    if (data.reply_to_message_id != null) {
-      replyToMessage = await Message.findById(data.reply_to_message_id);
-    }
-
-    if (replyToThread && replyToMessage) {
-      if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
-        throw new HTTPError(
-          `Message ${replyToMessage.id} is not in thread ${replyToThread.id}`,
-          400
-        )
-      }
-    }
-
-    let thread;
-    if (replyToMessage) {
-      setReplyHeaders(message, replyToMessage);
-      thread = await message.getThread();
-    } else if (replyToThread) {
-      thread = replyToThread;
-      const previousMessages = thread.messages.filter(msg => !msg.isDraft);
-      if (previousMessages.length > 0) {
-        const lastMessage = previousMessages[previousMessages.length - 1]
-        setReplyHeaders(message, lastMessage);
-      }
-    } else {
-      thread = Thread.build({
-        accountId: account_id,
-        subject: message.subject,
-        firstMessageDate: message.date,
-        lastMessageDate: message.date,
-        lastMessageSentDate: message.date,
-      })
-    }
-
-    const savedMessage = await message.save();
-    const savedThread = await thread.save();
-    await savedThread.addMessage(savedMessage);
-
-    return savedMessage;
+    return Message.associateFromJSON(data, db)
   },
   findMultiSendDraft: async (draftId, db) => {
     const draft = await db.Message.findById(draftId)
diff --git a/packages/local-sync/src/local-api/sendmail-client.js b/packages/local-sync/src/local-api/sendmail-client.js
index 659cf85b9..2879d38bb 100644
--- a/packages/local-sync/src/local-api/sendmail-client.js
+++ b/packages/local-sync/src/local-api/sendmail-client.js
@@ -1,3 +1,4 @@
+const fs = require('fs');
 const nodemailer = require('nodemailer');
 const mailcomposer = require('mailcomposer');
 const {HTTPError} = require('./sending-utils');
@@ -42,10 +43,18 @@ class SendmailClient {
       }
     }
     this._logger.error('Max sending retries reached');
+    this._handleError(error);
+  }
 
+  _handleError(err) {
     // TODO: figure out how to parse different errors, like in cloud-core
     // https://github.com/nylas/cloud-core/blob/production/sync-engine/inbox/sendmail/smtp/postel.py#L354
-    throw new HTTPError('Sending failed', 500, error)
+
+    if (err.startsWith("Error: Invalid login: 535-5.7.8 Username and Password not accepted.")) {
+      throw new HTTPError('Invalid login', 401, err)
+    }
+
+    throw new HTTPError('Sending failed', 500, err);
   }
 
   _draftToMsgData(draft) {
@@ -59,7 +68,14 @@ class SendmailClient {
     msgData.html = draft.body;
     msgData.messageId = `${draft.id}@nylas.com`;
 
-    // TODO: attachments
+    msgData.attachments = []
+    for (const upload of draft.uploads) {
+      msgData.attachments.push({
+        filename: upload.filename,
+        content: fs.createReadStream(upload.targetPath),
+        cid: upload.id,
+      })
+    }
 
     if (draft.replyTo) {
       msgData.replyTo = formatParticipants(draft.replyTo);
diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index c482f3597..413e8426c 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -1,3 +1,4 @@
+const _ = require('underscore');
 const cryptography = require('crypto');
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 const {DatabaseTypes: {buildJSONColumnOptions, buildJSONARRAYColumnOptions}} = require('isomorphic-core');
@@ -71,6 +72,21 @@ module.exports = (sequelize, Sequelize) => {
         this.setDataValue('isSending', val);
       },
     },
+    uploads: Object.assign(buildJSONARRAYColumnOptions('testFiles'), {
+      validate: {
+        uploadStructure: function uploadStructure(stringifiedArr) {
+          const arr = JSON.parse(stringifiedArr);
+          const requiredKeys = ['filename', 'targetPath', 'id']
+          arr.forEach((upload) => {
+            requiredKeys.forEach((key) => {
+              if (!upload.hasOwnPropery(key)) {
+                throw new Error(`Upload must have '${key}' key.`)
+              }
+            })
+          })
+        },
+      },
+    }),
   }, {
     indexes: [
       {
@@ -89,6 +105,82 @@ module.exports = (sequelize, Sequelize) => {
       hashForHeaders(headers) {
         return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
       },
+      fromJSON(data) {
+        // TODO: events, metadata??
+        return this.build({
+          accountId: data.account_id,
+          from: data.from,
+          to: data.to,
+          cc: data.cc,
+          bcc: data.bcc,
+          replyTo: data.reply_to,
+          subject: data.subject,
+          body: data.body,
+          unread: true,
+          isDraft: data.is_draft,
+          isSent: false,
+          version: 0,
+          date: data.date,
+          id: data.id,
+          uploads: data.uploads,
+        });
+      },
+      async associateFromJSON(data, db) {
+        const message = this.fromJSON(data);
+        const {Thread, Message} = db;
+
+        let replyToThread;
+        let replyToMessage;
+        if (data.thread_id != null) {
+          replyToThread = await Thread.find({
+            where: {id: data.thread_id},
+            include: [{
+              model: Message,
+              as: 'messages',
+              attributes: _.without(Object.keys(Message.attributes), 'body'),
+            }],
+          });
+        }
+        if (data.reply_to_message_id != null) {
+          replyToMessage = await Message.findById(data.reply_to_message_id);
+        }
+
+        if (replyToThread && replyToMessage) {
+          if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
+            throw new SendingUtils.HTTPError(
+              `Message ${replyToMessage.id} is not in thread ${replyToThread.id}`,
+              400
+            )
+          }
+        }
+
+        let thread;
+        if (replyToMessage) {
+          SendingUtils.setReplyHeaders(message, replyToMessage);
+          thread = await message.getThread();
+        } else if (replyToThread) {
+          thread = replyToThread;
+          const previousMessages = thread.messages.filter(msg => !msg.isDraft);
+          if (previousMessages.length > 0) {
+            const lastMessage = previousMessages[previousMessages.length - 1]
+            SendingUtils.setReplyHeaders(message, lastMessage);
+          }
+        } else {
+          thread = Thread.build({
+            accountId: message.accountId,
+            subject: message.subject,
+            firstMessageDate: message.date,
+            lastMessageDate: message.date,
+            lastMessageSentDate: message.date,
+          })
+        }
+
+        const savedMessage = await message.save();
+        const savedThread = await thread.save();
+        await savedThread.addMessage(savedMessage);
+
+        return savedMessage;
+      },
     },
     instanceMethods: {
       async setLabelsFromXGM(xGmLabels, {Label, preloadedLabels} = {}) {

From a185a8a5fe39595943e50b0d64043164ac487bb4 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 7 Dec 2016 14:43:51 -0800
Subject: [PATCH 480/800] [local-sync] Fix file/contact creation

We were getting sql unique constraint violation errors for ids because
we were attempting to create objects with the same id within the same
transaction.

This commit ensures that only attempt to write a contact with the same
id once, and that we check for all exsiting contacts before hand.

For file, we ensure that we don't attempt to write 2 files with the same
id more than once
---
 packages/local-sync/src/models/contact.js     |  9 ++++-
 .../new-message-processor/extract-contacts.js | 39 +++++++++++--------
 .../new-message-processor/extract-files.js    | 26 +++++++------
 3 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/packages/local-sync/src/models/contact.js b/packages/local-sync/src/models/contact.js
index 8dc894eaf..46c0014be 100644
--- a/packages/local-sync/src/models/contact.js
+++ b/packages/local-sync/src/models/contact.js
@@ -1,3 +1,5 @@
+const crypto = require('crypto')
+
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('contact', {
     id: {type: Sequelize.STRING(65), primaryKey: true},
@@ -12,8 +14,13 @@ module.exports = (sequelize, Sequelize) => {
         fields: ['id'],
       },
     ],
+    classMethods: {
+      hash({email}) {
+        return crypto.createHash('sha256').update(email, 'utf8').digest('hex');
+      },
+    },
     instanceMethods: {
-      toJSON: function toJSON() {
+      toJSON() {
         return {
           id: `${this.publicId}`,
           account_id: this.accountId,
diff --git a/packages/local-sync/src/new-message-processor/extract-contacts.js b/packages/local-sync/src/new-message-processor/extract-contacts.js
index b3220dc31..96784b958 100644
--- a/packages/local-sync/src/new-message-processor/extract-contacts.js
+++ b/packages/local-sync/src/new-message-processor/extract-contacts.js
@@ -1,4 +1,3 @@
-const cryptography = require('crypto');
 
 function isContactMeaningful(contact) {
   // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
@@ -14,32 +13,40 @@ function isContactMeaningful(contact) {
 }
 
 async function extractContacts({db, message}) {
+  const {Contact} = db
   let allContacts = [];
   ['to', 'from', 'bcc', 'cc'].forEach((field) => {
     allContacts = allContacts.concat(message[field])
   })
 
   const meaningfulContacts = allContacts.filter(c => isContactMeaningful(c));
+  const contactsDataById = new Map()
+  meaningfulContacts.forEach(c => {
+    const id = Contact.hash(c)
+    const cdata = {
+      id,
+      name: c.name,
+      email: c.email,
+      accountId: message.accountId,
+    }
+    contactsDataById.set(id, cdata)
+  })
+  const existingContacts = await Contact.findAll({
+    where: {
+      id: Array.from(contactsDataById.keys()),
+    },
+  })
 
   await db.sequelize.transaction(async (transaction) => {
-    const promises = [];
-
-    for (const c of meaningfulContacts) {
-      const id = cryptography.createHash('sha256').update(c.email, 'utf8').digest('hex');
-      const existing = await db.Contact.findById(id);
-      const cdata = {
-        id,
-        name: c.name,
-        email: c.email,
-        accountId: message.accountId,
-      };
-
+    const promises = []
+    for (const c of contactsDataById.values()) {
+      const existing = existingContacts.find(({id}) => id === c.id)
       if (!existing) {
-        promises.push(db.Contact.create(cdata, {transaction}));
+        promises.push(Contact.create(c, {transaction}));
       } else {
-        const updateRequired = (cdata.name !== existing.name);
+        const updateRequired = (c.name !== existing.name);
         if (updateRequired) {
-          promises.push(existing.update(cdata, {transaction}));
+          promises.push(existing.update(c, {transaction}));
         }
       }
     }
diff --git a/packages/local-sync/src/new-message-processor/extract-files.js b/packages/local-sync/src/new-message-processor/extract-files.js
index d208052ed..74bff1dc7 100644
--- a/packages/local-sync/src/new-message-processor/extract-files.js
+++ b/packages/local-sync/src/new-message-processor/extract-files.js
@@ -1,24 +1,28 @@
 
-function collectFilesFromStruct({db, message, struct}) {
+function collectFilesFromStruct({db, message, struct, fileIds = new Set()}) {
   const {File} = db;
   let collected = [];
 
   for (const part of struct) {
     if (part.constructor === Array) {
-      collected = collected.concat(collectFilesFromStruct({db, message, struct: part}));
+      collected = collected.concat(collectFilesFromStruct({db, message, struct: part, fileIds}));
     } else if (part.type !== 'text' && part.disposition) {
       // Only exposes partId for inline attachments
       const partId = part.disposition.type === 'inline' ? part.partID : null;
       const filename = part.disposition.params ? part.disposition.params.filename : null;
-      collected.push(File.build({
-        filename: filename,
-        partId: partId,
-        messageId: message.id,
-        contentType: `${part.type}/${part.subtype}`,
-        accountId: message.accountId,
-        size: part.size,
-        id: `${message.id}-${partId}-${part.size}`,
-      }));
+      const fileId = `${message.id}-${partId}-${part.size}`
+      if (!fileIds.has(fileId)) {
+        collected.push(File.build({
+          id: fileId,
+          partId: partId,
+          size: part.size,
+          filename: filename,
+          messageId: message.id,
+          accountId: message.accountId,
+          contentType: `${part.type}/${part.subtype}`,
+        }));
+        fileIds.add(fileId)
+      }
     }
   }
 

From 878735f52ef5d96dfbc83debfb0eec3e0f18c850 Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Wed, 7 Dec 2016 16:16:45 -0800
Subject: [PATCH 481/800] [local-sync] Add todos

---
 .../local-sync-worker/imap/fetch-messages-in-folder.js   | 4 ++++
 packages/local-sync/src/local-sync-worker/sync-worker.js | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 256e71b03..839367fe2 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -259,6 +259,9 @@ class FetchMessagesInFolder {
           uid: existingMessage.folderImapUID,
         }, `FetchMessagesInFolder: Updated message`)
       } else {
+        // TODO investigate batching processing new messages
+        // We could measure load of actual sync vs load of just message processing
+        // to determine how meaningful it is
         processNewMessage(messageValues, imapMessage)
         this._logger.info({
           message: messageValues,
@@ -323,6 +326,7 @@ class FetchMessagesInFolder {
       } else {
         this._logger.info('FetchMessagesInFolder: fetchedmax == uidnext, nothing more recent to fetch.')
       }
+
       if (savedSyncState.fetchedmin > 1) {
         const lowerbound = Math.max(1, savedSyncState.fetchedmin - FETCH_MESSAGES_COUNT);
         desiredRanges.push({min: lowerbound, max: savedSyncState.fetchedmin})
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index c70fce3ad..d36900023 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -30,6 +30,7 @@ class SyncWorker {
     // the database, because otherwise things get /crazy/ messy and I don't like
     // having counters and garbage everywhere.
     if (!account.firstSyncCompletion) {
+      // TODO extract this into its own module, can use later on for exchange
       this._logger.info("This is initial sync. Setting up metrics collection!");
 
       let seen = 0;
@@ -188,6 +189,13 @@ class SyncWorker {
   }
 
   async syncMessagesInAllFolders() {
+    // TODO prioritize syncing all of inbox first if there's a ton of folders (e.g. imap
+    // accounts). If there are many folders, we would only sync the first n
+    // messages in the inbox and not go back to it until we've done the same for
+    // the rest of the folders, which would give the appearance of the inbox
+    // syncing slowly. This should only be done during initial sync.
+    // TODO Also consider using multiple imap connections, 1 for inbox, one for the
+    // rest
     const {Folder} = this._db;
     const {folderSyncOptions} = this._account.syncPolicy;
 
@@ -196,6 +204,7 @@ class SyncWorker {
     const foldersSorted = folders.sort((a, b) =>
       (priority.indexOf(a.role) - priority.indexOf(b.role)) * -1
     )
+    // TODO make sure this order is correct
 
     return await Promise.all(foldersSorted.map((cat) =>
       this._conn.runOperation(new FetchMessagesInFolder(cat, folderSyncOptions, this._logger))

From d161a30a34c237825c4b784d16fe2f2acfa09627 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 7 Dec 2016 16:36:52 -0800
Subject: [PATCH 482/800] [local-sync] Add global dbs and cleanup orphan
 messages

---
 .../src/local-sync-worker/sync-process-manager.js  | 14 +++++++++++++-
 .../src/local-sync-worker/sync-worker.js           |  6 ++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/packages/local-sync/src/local-sync-worker/sync-process-manager.js b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
index 00f9bc892..a28a09f44 100644
--- a/packages/local-sync/src/local-sync-worker/sync-process-manager.js
+++ b/packages/local-sync/src/local-sync-worker/sync-process-manager.js
@@ -1,3 +1,4 @@
+const _ = require('underscore')
 const SyncWorker = require('./sync-worker');
 const LocalDatabaseConnector = require('../shared/local-database-connector')
 
@@ -28,9 +29,13 @@ class SyncProcessManager {
     this._workers = {};
     this._listenForSyncsClient = null;
     this._exiting = false;
+    this._accounts = []
     this._logger = global.Logger.child();
   }
 
+  /**
+   * Useful for debugging.
+   */
   async start() {
     this._logger.info(`ProcessManager: Starting with ID`)
 
@@ -41,6 +46,10 @@ class SyncProcessManager {
     }
   }
 
+  accounts() { return this._accounts }
+  workers() { return _.values(this._workers) }
+  dbs() { return this.workers().map(w => w._db) }
+
   wakeWorkerForAccount(account) {
     this._workers[account.id].syncNow();
   }
@@ -53,6 +62,7 @@ class SyncProcessManager {
       if (this._workers[account.id]) {
         throw new Error("Local worker already exists");
       }
+      this._accounts.push(account)
       this._workers[account.id] = new SyncWorker(account, db, this);
       this._logger.info({account_id: account.id}, `ProcessManager: Claiming Account Succeeded`)
     } catch (err) {
@@ -68,4 +78,6 @@ class SyncProcessManager {
   }
 }
 
-module.exports = new SyncProcessManager()
+window.syncProcessManager = new SyncProcessManager();
+window.dbs = window.syncProcessManager.dbs.bind(window.syncProcessManager)
+module.exports = window.syncProcessManager
diff --git a/packages/local-sync/src/local-sync-worker/sync-worker.js b/packages/local-sync/src/local-sync-worker/sync-worker.js
index d36900023..c86b9697c 100644
--- a/packages/local-sync/src/local-sync-worker/sync-worker.js
+++ b/packages/local-sync/src/local-sync-worker/sync-worker.js
@@ -236,6 +236,7 @@ class SyncWorker {
       await this.runNewSyncbackRequests();
       await this._conn.runOperation(new FetchFolderList(this._account.provider, this._logger));
       await this.syncMessagesInAllFolders();
+      await this.cleanupOrpahnMessages();
       await this.onSyncDidComplete();
     } catch (error) {
       this.onSyncError(error);
@@ -245,6 +246,11 @@ class SyncWorker {
     }
   }
 
+  async cleanupOrpahnMessages() {
+    const orphans = await this._db.Message.findAll({where: {folderId: null}})
+    return Promise.map(orphans, (msg) => msg.destroy());
+  }
+
   onSyncError(error) {
     this.closeConnection()
 

From 2f786a8e914a37d2bdb026e338046882d5198e37 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Wed, 7 Dec 2016 17:47:49 -0800
Subject: [PATCH 483/800] [local-sync] fix delta sync not getting threads with
 updated labels

---
 packages/local-sync/src/models/message.js | 9 +++++++--
 packages/local-sync/src/models/thread.js  | 3 ++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index 413e8426c..535b4bc42 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -101,7 +101,12 @@ module.exports = (sequelize, Sequelize) => {
         Message.belongsToMany(Label, {through: MessageLabel})
         Message.hasMany(File)
       },
-
+      requiredAssociationsForJSON: ({Folder, Label}) => {
+        return [
+          {model: Folder},
+          {model: Label},
+        ]
+      },
       hashForHeaders(headers) {
         return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
       },
@@ -216,7 +221,7 @@ module.exports = (sequelize, Sequelize) => {
         this.headerMessageId = `<${this.id}-${this.version}@mailer.nylas.com>`
       },
       toJSON() {
-        if (this.folder_id && !this.folder) {
+        if (this.folderId && !this.folder) {
           throw new Error("Message.toJSON called on a message where folder were not eagerly loaded.")
         }
 
diff --git a/packages/local-sync/src/models/thread.js b/packages/local-sync/src/models/thread.js
index a6c27156c..7eef390de 100644
--- a/packages/local-sync/src/models/thread.js
+++ b/packages/local-sync/src/models/thread.js
@@ -126,7 +126,8 @@ module.exports = (sequelize, Sequelize) => {
             await savedThread.addLabel(label)
           }
         }
-        return savedThread;
+
+        return savedThread.save();
       },
       toJSON() {
         if (!(this.labels instanceof Array)) {

From 1ba56c5c0573b0759edf3fc92e5095fa407c4cca Mon Sep 17 00:00:00 2001
From: Juan Tejada 
Date: Thu, 8 Dec 2016 10:28:23 -0800
Subject: [PATCH 484/800] [local-sync] Fix lint error

---
 .../src/local-sync-worker/imap/fetch-messages-in-folder.js      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index 839367fe2..ecdf17094 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -272,7 +272,7 @@ class FetchMessagesInFolder {
         imapMessage,
         desiredParts,
       }, `FetchMessagesInFolder: Could not build message`)
-      const outJSON = JSON.stringify({'imapMessage': imapMessage, 'desiredParts': desiredParts, 'result': {}});
+      const outJSON = JSON.stringify({imapMessage, desiredParts, result: {}});
       const outDir = path.join(os.tmpdir(), "k2-parse-errors", this._folder.name)
       const outFile = path.join(outDir, imapMessage.attributes.uid.toString());
       mkdirp.sync(outDir);

From 7a763b604e9ea911a7f67588399920890a52c171 Mon Sep 17 00:00:00 2001
From: Jackie Luo 
Date: Thu, 8 Dec 2016 13:38:24 -0800
Subject: [PATCH 485/800] feat(tracking): Add routes for open and link tracking

---
 .../src/local-api/sendmail-client.js           | 13 ++++++++++---
 packages/local-sync/src/models/message.js      | 18 +++++++++++++++---
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/packages/local-sync/src/local-api/sendmail-client.js b/packages/local-sync/src/local-api/sendmail-client.js
index 2879d38bb..c3fb325f1 100644
--- a/packages/local-sync/src/local-api/sendmail-client.js
+++ b/packages/local-sync/src/local-api/sendmail-client.js
@@ -64,6 +64,7 @@ class SendmailClient {
         msgData[field] = formatParticipants(draft[field])
       }
     }
+    msgData.date = draft.date;
     msgData.subject = draft.subject;
     msgData.html = draft.body;
     msgData.messageId = `${draft.id}@nylas.com`;
@@ -89,6 +90,12 @@ class SendmailClient {
     return msgData;
   }
 
+  _getBodyWithMessageIds(draft) {
+    return draft.body.replace(/n1cloud\.nylas\.com\/.+MESSAGE_ID/g, (match) => {
+      return match.replace('MESSAGE_ID', draft.id)
+    })
+  }
+
   async buildMime(draft) {
     const builder = mailcomposer(this._draftToMsgData(draft))
     const mimeNode = await (new Promise((resolve, reject) => {
@@ -110,7 +117,7 @@ class SendmailClient {
 
   async sendCustomBody(draft, body, recipients) {
     const origBody = draft.body;
-    draft.body = body;
+    draft.body = this._getBodyWithMessageIds(draft);
     const envelope = {};
     for (const field of Object.keys(recipients)) {
       envelope[field] = recipients[field].map(r => r.email);
@@ -118,8 +125,8 @@ class SendmailClient {
     const raw = await this.buildMime(draft);
     const responseOnSuccess = draft.toJSON();
     draft.body = origBody;
-    await this._send({raw, envelope})
-    return responseOnSuccess
+    await this._send({raw, envelope});
+    return responseOnSuccess;
   }
 }
 
diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index 535b4bc42..0ed9ceb38 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -110,7 +110,15 @@ module.exports = (sequelize, Sequelize) => {
       hashForHeaders(headers) {
         return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
       },
-      fromJSON(data) {
+      getHeadersForId(data) {
+        let participants = "";
+        const emails = _.pluck(data.from.concat(data.to, data.cc, data.bcc), 'email');
+        emails.sort().forEach((email) => {
+          participants += email
+        });
+        return `${data.date}-${data.subject}-${participants}`;
+      },
+      fromJSON(id, data) {
         // TODO: events, metadata??
         return this.build({
           accountId: data.account_id,
@@ -126,14 +134,16 @@ module.exports = (sequelize, Sequelize) => {
           isSent: false,
           version: 0,
           date: data.date,
-          id: data.id,
+          id: id,
           uploads: data.uploads,
         });
       },
       async associateFromJSON(data, db) {
-        const message = this.fromJSON(data);
         const {Thread, Message} = db;
 
+        const messageId = Message.getHeadersForId(data)
+        const message = this.fromJSON(messageId, data);
+
         let replyToThread;
         let replyToMessage;
         if (data.thread_id != null) {
@@ -220,6 +230,7 @@ module.exports = (sequelize, Sequelize) => {
       regenerateHeaderMessageId() {
         this.headerMessageId = `<${this.id}-${this.version}@mailer.nylas.com>`
       },
+
       toJSON() {
         if (this.folderId && !this.folder) {
           throw new Error("Message.toJSON called on a message where folder were not eagerly loaded.")
@@ -250,6 +261,7 @@ module.exports = (sequelize, Sequelize) => {
         };
       },
     },
+
     hooks: {
       beforeUpdate: (message) => {
         // Update the snippet if the body has changed

From 947eb99b8df5e966f04d64de2915d778365d0eaa Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Thu, 8 Dec 2016 14:16:37 -0800
Subject: [PATCH 486/800] [local-sync] fix builds. Routes with trailing slash
 and main extension

---
 packages/local-sync/package.json         | 2 +-
 packages/local-sync/src/local-api/app.js | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index a24ccce17..000ad00ca 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -2,7 +2,7 @@
   "name": "local-sync",
   "version": "0.0.1",
   "description": "The local sync engine for Nylas N1",
-  "main": "./main.es6",
+  "main": "./main",
   "dependencies": {
     "googleapis": "9.0.0",
     "hapi": "13.4.1",
diff --git a/packages/local-sync/src/local-api/app.js b/packages/local-sync/src/local-api/app.js
index e0c9d4ea7..21b1c149d 100644
--- a/packages/local-sync/src/local-api/app.js
+++ b/packages/local-sync/src/local-api/app.js
@@ -78,8 +78,8 @@ const attach = (directory) => {
 server.register(plugins, (err) => {
   if (err) { throw err; }
 
-  attach('./routes/')
-  attach('./decorators/')
+  attach('./routes')
+  attach('./decorators')
 
   server.auth.strategy('api-consumer', 'basic', { validateFunc: validate });
   server.auth.default('api-consumer');

From fae855f0fe24b2771a9277a6cf8e41c55c12b9da Mon Sep 17 00:00:00 2001
From: Jackie Luo 
Date: Thu, 8 Dec 2016 17:48:34 -0800
Subject: [PATCH 487/800] feat(message-ids): Hash message IDs and replace in
 draft before sending

---
 .../local-sync/src/local-api/sending-utils.js |   5 +-
 .../src/local-api/sendmail-client.js          |  11 +-
 packages/local-sync/src/models/message.js     |  91 ----------------
 .../local-sync/src/shared/message-factory.js  | 103 +++++++++++++++++-
 4 files changed, 115 insertions(+), 95 deletions(-)

diff --git a/packages/local-sync/src/local-api/sending-utils.js b/packages/local-sync/src/local-api/sending-utils.js
index 2c3d0905f..ca91edcb9 100644
--- a/packages/local-sync/src/local-api/sending-utils.js
+++ b/packages/local-sync/src/local-api/sending-utils.js
@@ -1,3 +1,6 @@
+const MessageFactory = require('../shared/message-factory')
+
+
 class HTTPError extends Error {
   constructor(message, httpCode, logContext) {
     super(message);
@@ -26,7 +29,7 @@ module.exports = {
       return existingMessage;
     }
 
-    return Message.associateFromJSON(data, db)
+    return MessageFactory.associateFromJSON(data, db)
   },
   findMultiSendDraft: async (draftId, db) => {
     const draft = await db.Message.findById(draftId)
diff --git a/packages/local-sync/src/local-api/sendmail-client.js b/packages/local-sync/src/local-api/sendmail-client.js
index c3fb325f1..f29ebddc5 100644
--- a/packages/local-sync/src/local-api/sendmail-client.js
+++ b/packages/local-sync/src/local-api/sendmail-client.js
@@ -1,3 +1,5 @@
+/* eslint no-useless-escape: 0 */
+
 const fs = require('fs');
 const nodemailer = require('nodemailer');
 const mailcomposer = require('mailcomposer');
@@ -91,7 +93,14 @@ class SendmailClient {
   }
 
   _getBodyWithMessageIds(draft) {
-    return draft.body.replace(/n1cloud\.nylas\.com\/.+MESSAGE_ID/g, (match) => {
+    const serverUrl = {
+      local: 'http:\/\/lvh\.me:5100',
+      development: 'http:\/\/lvh\.me:5100',
+      staging: 'https:\/\/n1cloud-staging\.nylas\.com',
+      production: 'https:\/\/n1cloud\.nylas\.com',
+    }[process.env];
+    const regex = new RegExp(`${serverUrl}\/.+MESSAGE_ID`, 'g')
+    return draft.body.replace(regex, (match) => {
       return match.replace('MESSAGE_ID', draft.id)
     })
   }
diff --git a/packages/local-sync/src/models/message.js b/packages/local-sync/src/models/message.js
index 0ed9ceb38..83a47ede2 100644
--- a/packages/local-sync/src/models/message.js
+++ b/packages/local-sync/src/models/message.js
@@ -1,5 +1,3 @@
-const _ = require('underscore');
-const cryptography = require('crypto');
 const {PromiseUtils, IMAPConnection} = require('isomorphic-core')
 const {DatabaseTypes: {buildJSONColumnOptions, buildJSONARRAYColumnOptions}} = require('isomorphic-core');
 const striptags = require('striptags');
@@ -107,95 +105,6 @@ module.exports = (sequelize, Sequelize) => {
           {model: Label},
         ]
       },
-      hashForHeaders(headers) {
-        return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
-      },
-      getHeadersForId(data) {
-        let participants = "";
-        const emails = _.pluck(data.from.concat(data.to, data.cc, data.bcc), 'email');
-        emails.sort().forEach((email) => {
-          participants += email
-        });
-        return `${data.date}-${data.subject}-${participants}`;
-      },
-      fromJSON(id, data) {
-        // TODO: events, metadata??
-        return this.build({
-          accountId: data.account_id,
-          from: data.from,
-          to: data.to,
-          cc: data.cc,
-          bcc: data.bcc,
-          replyTo: data.reply_to,
-          subject: data.subject,
-          body: data.body,
-          unread: true,
-          isDraft: data.is_draft,
-          isSent: false,
-          version: 0,
-          date: data.date,
-          id: id,
-          uploads: data.uploads,
-        });
-      },
-      async associateFromJSON(data, db) {
-        const {Thread, Message} = db;
-
-        const messageId = Message.getHeadersForId(data)
-        const message = this.fromJSON(messageId, data);
-
-        let replyToThread;
-        let replyToMessage;
-        if (data.thread_id != null) {
-          replyToThread = await Thread.find({
-            where: {id: data.thread_id},
-            include: [{
-              model: Message,
-              as: 'messages',
-              attributes: _.without(Object.keys(Message.attributes), 'body'),
-            }],
-          });
-        }
-        if (data.reply_to_message_id != null) {
-          replyToMessage = await Message.findById(data.reply_to_message_id);
-        }
-
-        if (replyToThread && replyToMessage) {
-          if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
-            throw new SendingUtils.HTTPError(
-              `Message ${replyToMessage.id} is not in thread ${replyToThread.id}`,
-              400
-            )
-          }
-        }
-
-        let thread;
-        if (replyToMessage) {
-          SendingUtils.setReplyHeaders(message, replyToMessage);
-          thread = await message.getThread();
-        } else if (replyToThread) {
-          thread = replyToThread;
-          const previousMessages = thread.messages.filter(msg => !msg.isDraft);
-          if (previousMessages.length > 0) {
-            const lastMessage = previousMessages[previousMessages.length - 1]
-            SendingUtils.setReplyHeaders(message, lastMessage);
-          }
-        } else {
-          thread = Thread.build({
-            accountId: message.accountId,
-            subject: message.subject,
-            firstMessageDate: message.date,
-            lastMessageDate: message.date,
-            lastMessageSentDate: message.date,
-          })
-        }
-
-        const savedMessage = await message.save();
-        const savedThread = await thread.save();
-        await savedThread.addMessage(savedMessage);
-
-        return savedMessage;
-      },
     },
     instanceMethods: {
       async setLabelsFromXGM(xGmLabels, {Label, preloadedLabels} = {}) {
diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index 54fb09540..af45a6c8c 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -1,9 +1,12 @@
+const _ = require('underscore');
+const cryptography = require('crypto');
 const utf7 = require('utf7').imap;
 const mimelib = require('mimelib');
 const QuotedPrintable = require('quoted-printable');
-const {Imap} = require('isomorphic-core');
-const mkdirp = require('mkdirp');
 const striptags = require('striptags');
+const {Imap} = require('isomorphic-core');
+const SendingUtils = require('../local-api/sending-utils');
+
 
 const SNIPPET_SIZE = 100
 
@@ -89,6 +92,102 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
   return values;
 }
 
+function hashForHeaders(headers) {
+  return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
+}
+
+function getHeadersForId(data) {
+  let participants = "";
+  const emails = _.pluck(data.from.concat(data.to, data.cc, data.bcc), 'email');
+  emails.sort().forEach((email) => {
+    participants += email
+  });
+  return `${data.date}-${data.subject}-${participants}`;
+}
+
+function fromJSON(db, data) {
+  // TODO: events, metadata?
+  const {Message} = db;
+  const id = hashForHeaders(getHeadersForId(data))
+  return Message.build({
+    accountId: data.account_id,
+    from: data.from,
+    to: data.to,
+    cc: data.cc,
+    bcc: data.bcc,
+    replyTo: data.reply_to,
+    subject: data.subject,
+    body: data.body,
+    unread: true,
+    isDraft: data.is_draft,
+    isSent: false,
+    version: 0,
+    date: data.date,
+    id: id,
+    uploads: data.uploads,
+  });
+}
+
+async function associateFromJSON(data, db) {
+  const {Thread, Message} = db;
+
+  const message = fromJSON(db, data);
+
+  let replyToThread;
+  let replyToMessage;
+  if (data.thread_id != null) {
+    replyToThread = await Thread.find({
+      where: {id: data.thread_id},
+      include: [{
+        model: Message,
+        as: 'messages',
+        attributes: _.without(Object.keys(Message.attributes), 'body'),
+      }],
+    });
+  }
+  if (data.reply_to_message_id != null) {
+    replyToMessage = await Message.findById(data.reply_to_message_id);
+  }
+
+  if (replyToThread && replyToMessage) {
+    if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
+      throw new SendingUtils.HTTPError(
+        `Message ${replyToMessage.id} is not in thread ${replyToThread.id}`,
+        400
+      )
+    }
+  }
+
+  let thread;
+  if (replyToMessage) {
+    SendingUtils.setReplyHeaders(message, replyToMessage);
+    thread = await message.getThread();
+  } else if (replyToThread) {
+    thread = replyToThread;
+    const previousMessages = thread.messages.filter(msg => !msg.isDraft);
+    if (previousMessages.length > 0) {
+      const lastMessage = previousMessages[previousMessages.length - 1]
+      SendingUtils.setReplyHeaders(message, lastMessage);
+    }
+  } else {
+    thread = Thread.build({
+      accountId: message.accountId,
+      subject: message.subject,
+      firstMessageDate: message.date,
+      lastMessageDate: message.date,
+      lastMessageSentDate: message.date,
+    })
+  }
+
+  const savedMessage = await message.save();
+  const savedThread = await thread.save();
+  await savedThread.addMessage(savedMessage);
+
+  return savedMessage;
+}
+
 module.exports = {
   parseFromImap,
+  fromJSON,
+  associateFromJSON,
 }

From 6e111c073aec2f783a1a585dd24c7256f83ce356 Mon Sep 17 00:00:00 2001
From: Jackie Luo 
Date: Thu, 8 Dec 2016 17:55:39 -0800
Subject: [PATCH 488/800] fix(message-ids): Use correct hashing for headers

---
 .../local-sync/src/shared/message-factory.js  | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index af45a6c8c..ae2aa5b7a 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -17,8 +17,21 @@ function extractContacts(values = []) {
   })
 }
 
+function getHeadersForId(data) {
+  let participants = "";
+  const emails = _.pluck(data.from.concat(data.to, data.cc, data.bcc), 'email');
+  emails.sort().forEach((email) => {
+    participants += email
+  });
+  return `${data.date}-${data.subject}-${participants}`;
+}
+
+function hashForHeaders(headers) {
+  return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
+}
+
 async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder}) {
-  const {Message, Label} = db
+  const {Label} = db
   const body = {}
   const {headers, attributes} = imapMessage
   const xGmLabels = attributes['x-gm-labels']
@@ -43,7 +56,7 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
   }
 
   const values = {
-    id: Message.hashForHeaders(headers),
+    id: hashForHeaders(getHeadersForId(parsedHeaders)),
     to: extractContacts(parsedHeaders.to),
     cc: extractContacts(parsedHeaders.cc),
     bcc: extractContacts(parsedHeaders.bcc),
@@ -92,19 +105,6 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
   return values;
 }
 
-function hashForHeaders(headers) {
-  return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
-}
-
-function getHeadersForId(data) {
-  let participants = "";
-  const emails = _.pluck(data.from.concat(data.to, data.cc, data.bcc), 'email');
-  emails.sort().forEach((email) => {
-    participants += email
-  });
-  return `${data.date}-${data.subject}-${participants}`;
-}
-
 function fromJSON(db, data) {
   // TODO: events, metadata?
   const {Message} = db;

From 4a11bfe9778e99f5564b12708feeab42aa4dfc91 Mon Sep 17 00:00:00 2001
From: Jackie Luo 
Date: Thu, 8 Dec 2016 18:10:17 -0800
Subject: [PATCH 489/800] fix(message-factory): Unlink circular dependency

---
 .../local-sync/src/local-api/routes/send.js   |  3 +-
 .../local-sync/src/local-api/sending-utils.js | 28 ++++---------------
 packages/local-sync/src/shared/errors.js      | 11 ++++++++
 .../local-sync/src/shared/message-factory.js  | 20 +++++++++----
 4 files changed, 33 insertions(+), 29 deletions(-)
 create mode 100644 packages/local-sync/src/shared/errors.js

diff --git a/packages/local-sync/src/local-api/routes/send.js b/packages/local-sync/src/local-api/routes/send.js
index d31c37542..153c15fd6 100644
--- a/packages/local-sync/src/local-api/routes/send.js
+++ b/packages/local-sync/src/local-api/routes/send.js
@@ -1,5 +1,6 @@
 const Joi = require('joi');
 const LocalDatabaseConnector = require('../../shared/local-database-connector');
+const Errors = require('../../shared/errors');
 const SendingUtils = require('../sending-utils');
 const SendmailClient = require('../sendmail-client');
 
@@ -128,7 +129,7 @@ module.exports = (server) => {
         const {to, cc, bcc} = draft;
         const recipients = [].concat(to, cc, bcc);
         if (!recipients.find(contact => contact.email === sendTo.email)) {
-          throw new SendingUtils.HTTPError(
+          throw new Errors.HTTPError(
             "Invalid sendTo, not present in message recipients",
             400
           );
diff --git a/packages/local-sync/src/local-api/sending-utils.js b/packages/local-sync/src/local-api/sending-utils.js
index ca91edcb9..0ca1c73c3 100644
--- a/packages/local-sync/src/local-api/sending-utils.js
+++ b/packages/local-sync/src/local-api/sending-utils.js
@@ -1,26 +1,8 @@
 const MessageFactory = require('../shared/message-factory')
+const Errors = require('../shared/errors')
 
 
-class HTTPError extends Error {
-  constructor(message, httpCode, logContext) {
-    super(message);
-    this.httpCode = httpCode;
-    this.logContext = logContext;
-  }
-}
-
 module.exports = {
-  HTTPError,
-  setReplyHeaders: (newMessage, prevMessage) => {
-    if (prevMessage.messageIdHeader) {
-      newMessage.inReplyTo = prevMessage.headerMessageId;
-      if (prevMessage.references) {
-        newMessage.references = prevMessage.references.concat(prevMessage.headerMessageId);
-      } else {
-        newMessage.references = [prevMessage.messageIdHeader];
-      }
-    }
-  },
   findOrCreateMessageFromJSON: async (data, db) => {
     const {Message} = db;
 
@@ -34,10 +16,10 @@ module.exports = {
   findMultiSendDraft: async (draftId, db) => {
     const draft = await db.Message.findById(draftId)
     if (!draft) {
-      throw new HTTPError(`Couldn't find multi-send draft ${draftId}`, 400);
+      throw new Errors.HTTPError(`Couldn't find multi-send draft ${draftId}`, 400);
     }
     if (draft.isSent || !draft.isSending) {
-      throw new HTTPError(`Message ${draftId} is not a multi-send draft`, 400);
+      throw new Errors.HTTPError(`Message ${draftId} is not a multi-send draft`, 400);
     }
     return draft;
   },
@@ -45,13 +27,13 @@ module.exports = {
     const {to, cc, bcc} = draft;
     const recipients = [].concat(to, cc, bcc);
     if (recipients.length === 0) {
-      throw new HTTPError("No recipients specified", 400);
+      throw new Errors.HTTPError("No recipients specified", 400);
     }
   },
   validateBase36: (value, name) => {
     if (value == null) { return; }
     if (isNaN(parseInt(value, 36))) {
-      throw new HTTPError(`${name} is not a base-36 integer`, 400)
+      throw new Errors.HTTPError(`${name} is not a base-36 integer`, 400)
     }
   },
 }
diff --git a/packages/local-sync/src/shared/errors.js b/packages/local-sync/src/shared/errors.js
new file mode 100644
index 000000000..887793768
--- /dev/null
+++ b/packages/local-sync/src/shared/errors.js
@@ -0,0 +1,11 @@
+class HTTPError extends Error {
+  constructor(message, httpCode, logContext) {
+    super(message);
+    this.httpCode = httpCode;
+    this.logContext = logContext;
+  }
+}
+
+module.exports = {
+  HTTPError,
+}
diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index ae2aa5b7a..14ad4370b 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -5,8 +5,7 @@ const mimelib = require('mimelib');
 const QuotedPrintable = require('quoted-printable');
 const striptags = require('striptags');
 const {Imap} = require('isomorphic-core');
-const SendingUtils = require('../local-api/sending-utils');
-
+const Errors = require('./errors');
 
 const SNIPPET_SIZE = 100
 
@@ -30,6 +29,17 @@ function hashForHeaders(headers) {
   return cryptography.createHash('sha256').update(headers, 'utf8').digest('hex');
 }
 
+function setReplyHeaders(newMessage, prevMessage) {
+  if (prevMessage.messageIdHeader) {
+    newMessage.inReplyTo = prevMessage.headerMessageId;
+    if (prevMessage.references) {
+      newMessage.references = prevMessage.references.concat(prevMessage.headerMessageId);
+    } else {
+      newMessage.references = [prevMessage.messageIdHeader];
+    }
+  }
+}
+
 async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder}) {
   const {Label} = db
   const body = {}
@@ -151,7 +161,7 @@ async function associateFromJSON(data, db) {
 
   if (replyToThread && replyToMessage) {
     if (!replyToThread.messages.find((msg) => msg.id === replyToMessage.id)) {
-      throw new SendingUtils.HTTPError(
+      throw new Errors.HTTPError(
         `Message ${replyToMessage.id} is not in thread ${replyToThread.id}`,
         400
       )
@@ -160,14 +170,14 @@ async function associateFromJSON(data, db) {
 
   let thread;
   if (replyToMessage) {
-    SendingUtils.setReplyHeaders(message, replyToMessage);
+    setReplyHeaders(message, replyToMessage);
     thread = await message.getThread();
   } else if (replyToThread) {
     thread = replyToThread;
     const previousMessages = thread.messages.filter(msg => !msg.isDraft);
     if (previousMessages.length > 0) {
       const lastMessage = previousMessages[previousMessages.length - 1]
-      SendingUtils.setReplyHeaders(message, lastMessage);
+      setReplyHeaders(message, lastMessage);
     }
   } else {
     thread = Thread.build({

From 162dbbd1416835360fa902c03b4af5b3665c8aa6 Mon Sep 17 00:00:00 2001
From: Jackie Luo 
Date: Thu, 8 Dec 2016 18:32:23 -0800
Subject: [PATCH 490/800] fix(error): Treat error as object, not string

---
 packages/local-sync/src/local-api/sendmail-client.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/local-sync/src/local-api/sendmail-client.js b/packages/local-sync/src/local-api/sendmail-client.js
index f29ebddc5..c8f61742f 100644
--- a/packages/local-sync/src/local-api/sendmail-client.js
+++ b/packages/local-sync/src/local-api/sendmail-client.js
@@ -52,7 +52,7 @@ class SendmailClient {
     // TODO: figure out how to parse different errors, like in cloud-core
     // https://github.com/nylas/cloud-core/blob/production/sync-engine/inbox/sendmail/smtp/postel.py#L354
 
-    if (err.startsWith("Error: Invalid login: 535-5.7.8 Username and Password not accepted.")) {
+    if (err.message.startsWith("Error: Invalid login: 535-5.7.8 Username and Password not accepted.")) {
       throw new HTTPError('Invalid login', 401, err)
     }
 

From 587f7787a67f08dcc6f9fa4ce48a04f2dc592486 Mon Sep 17 00:00:00 2001
From: Christine Spang 
Date: Thu, 8 Dec 2016 18:30:57 -0800
Subject: [PATCH 491/800] fix(local-sync): Fix charset interpretation in
 message parsing

Summary:
This commit fixes the following bugs in message parsing:
- we were unilaterally decoding MIME bodies as UTF-8; instead, decode according
  to the charset data in the mimepart header
- '7bit' content-transfer-encoding means us-ascii, NOT utf-7
- only interpret valid content-transfer-encodings (previously we were trying
  to treat various charsets as transfer-encodings)
- clearer naming: s/values/parsedMessage/
- unify snippet cleanup between plaintext & stripped HTML (merging
  whitespace etc.)

Test Plan: units tests coming

Reviewers: juan

Differential Revision: https://phab.nylas.com/D3491
---
 packages/isomorphic-core/src/imap-box.js      |  6 +-
 packages/local-sync/package.json              |  6 +-
 .../imap/fetch-messages-in-folder.js          |  4 +-
 .../local-sync/src/shared/message-factory.js  | 89 ++++++++++++-------
 4 files changed, 66 insertions(+), 39 deletions(-)

diff --git a/packages/isomorphic-core/src/imap-box.js b/packages/isomorphic-core/src/imap-box.js
index 01e17f4b7..10e2545a1 100644
--- a/packages/isomorphic-core/src/imap-box.js
+++ b/packages/isomorphic-core/src/imap-box.js
@@ -68,7 +68,7 @@ class IMAPBox {
           });
 
           stream.once('end', () => {
-            const full = Buffer.concat(chunks).toString('utf8');
+            const full = Buffer.concat(chunks);
             if (info.which === 'HEADER') {
               headers = full;
             } else {
@@ -77,6 +77,10 @@ class IMAPBox {
           });
         });
         imapMessage.once('end', () => {
+          // attributes is an object containing ascii strings, but parts and
+          // headers are undecoded binary Buffers (since the data for mime
+          // parts cannot be decoded to strings without looking up charset data
+          // in metadata, and this function's job is only to fetch the raw data)
           forEachMessageCallback({attributes, headers, parts});
         });
       })
diff --git a/packages/local-sync/package.json b/packages/local-sync/package.json
index 000ad00ca..a82f5968f 100644
--- a/packages/local-sync/package.json
+++ b/packages/local-sync/package.json
@@ -16,16 +16,14 @@
     "joi": "8.4.2",
     "mimelib": "0.2.19",
     "nodemailer": "2.5.0",
-    "quoted-printable": "1.0.1",
     "request": "2.79.0",
     "rx": "4.1.0",
     "sequelize": "3.27.0",
     "sqlite3": "https://github.com/bengotow/node-sqlite3/archive/bengotow/usleep-v3.1.4.tar.gz",
-    "striptags": "2.1.1",
     "underscore": "1.8.3",
-    "utf7": "^1.0.2",
     "striptags": "2.1.1",
-    "vision": "4.1.0"
+    "vision": "4.1.0",
+    "encoding": "0.1.12"
   },
   "scripts": {
     "test": "../../../../node_modules/.bin/electron ../../../../ --test --enable-logging --spec-directory=$(pwd)/spec"
diff --git a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
index ecdf17094..d219e284a 100644
--- a/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
+++ b/packages/local-sync/src/local-sync-worker/imap/fetch-messages-in-folder.js
@@ -187,7 +187,9 @@ class FetchMessagesInFolder {
         if (['text/plain', 'text/html', 'application/pgp-encrypted'].includes(mimetype)) {
           desired.push({
             id: part.partID,
-            encoding: part.encoding,
+            // encoding and charset may be null
+            transferEncoding: part.encoding,
+            charset: part.params ? part.params.charset : null,
             mimetype,
           });
         }
diff --git a/packages/local-sync/src/shared/message-factory.js b/packages/local-sync/src/shared/message-factory.js
index 14ad4370b..48aa282fa 100644
--- a/packages/local-sync/src/shared/message-factory.js
+++ b/packages/local-sync/src/shared/message-factory.js
@@ -1,13 +1,15 @@
 const _ = require('underscore');
 const cryptography = require('crypto');
-const utf7 = require('utf7').imap;
 const mimelib = require('mimelib');
-const QuotedPrintable = require('quoted-printable');
 const striptags = require('striptags');
+const encoding = require('encoding');
+
 const {Imap} = require('isomorphic-core');
 const Errors = require('./errors');
 
-const SNIPPET_SIZE = 100
+// aiming for the former in length, but the latter is the hard db cutoff
+const SNIPPET_SIZE = 100;
+const SNIPPET_MAX_SIZE = 255;
 
 function extractContacts(values = []) {
   return values.map(v => {
@@ -40,32 +42,41 @@ function setReplyHeaders(newMessage, prevMessage) {
   }
 }
 
+/*
+Since we only fetch the MIME structure and specific desired MIME parts from
+IMAP, we unfortunately can't use an existing library like mailparser to parse
+the message, and have to do fun stuff like deal with character sets and
+content-transfer-encodings ourselves.
+*/
 async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder}) {
   const {Label} = db
+  const {attributes} = imapMessage
+
   const body = {}
-  const {headers, attributes} = imapMessage
-  const xGmLabels = attributes['x-gm-labels']
-  for (const {id, mimetype, encoding} of desiredParts) {
-    if (!encoding) {
-      body[mimetype] = imapMessage.parts[id];
-    } else if (encoding.toLowerCase() === 'quoted-printable') {
-      body[mimetype] = QuotedPrintable.decode(imapMessage.parts[id]);
-    } else if (encoding.toLowerCase() === '7bit') {
-      body[mimetype] = utf7.decode(imapMessage.parts[id]);
-    } else if (encoding.toLowerCase() === '8bit') {
-      body[mimetype] = Buffer.from(imapMessage.parts[id], 'utf8').toString();
-    } else if (encoding && ['ascii', 'utf8', 'utf16le', 'ucs2', 'base64', 'latin1', 'binary', 'hex'].includes(encoding.toLowerCase())) {
-      body[mimetype] = Buffer.from(imapMessage.parts[id], encoding.toLowerCase()).toString();
+  for (const {id, mimetype, transferEncoding, charset} of desiredParts) {
+    // see https://www.w3.org/Protocols/rfc1341/5_Content-Transfer-Encoding.html
+    if (!transferEncoding || new Set(['7bit', '8bit']).has(transferEncoding.toLowerCase())) {
+      // NO transfer encoding has been performed --- how to decode to a string
+      // depends ONLY on the charset, which defaults to 'ascii' according to
+      // https://tools.ietf.org/html/rfc2045#section-5.2
+      const convertedBuffer = encoding.convert(imapMessage.parts[id], 'utf-8', charset || 'ascii')
+      body[mimetype] = convertedBuffer.toString('utf-8');
+    } else if (transferEncoding.toLowerCase() === 'quoted-printable') {
+      body[mimetype] = mimelib.decodeQuotedPrintable(imapMessage.parts[id], charset || 'ascii');
+    } else if (transferEncoding.toLowerCase() === 'base64') {
+      body[mimetype] = mimelib.decodeBase64(imapMessage.parts[id], charset || 'ascii');
     } else {
-      return Promise.reject(new Error(`Unknown encoding ${encoding}, mimetype ${mimetype}`))
+      // 'binary' and custom x-token content-transfer-encodings
+      return Promise.reject(new Error(`Unsupported Content-Transfer-Encoding ${transferEncoding}, mimetype ${mimetype}`))
     }
   }
+  const headers = imapMessage.headers.toString('ascii');
   const parsedHeaders = Imap.parseHeader(headers);
   for (const key of ['x-gm-thrid', 'x-gm-msgid', 'x-gm-labels']) {
     parsedHeaders[key] = attributes[key];
   }
 
-  const values = {
+  const parsedMessage = {
     id: hashForHeaders(getHeadersForId(parsedHeaders)),
     to: extractContacts(parsedHeaders.to),
     cc: extractContacts(parsedHeaders.cc),
@@ -74,7 +85,7 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
     replyTo: extractContacts(parsedHeaders['reply-to']),
     accountId: accountId,
     body: body['text/html'] || body['text/plain'] || body['application/pgp-encrypted'] || '',
-    snippet: body['text/plain'] ? body['text/plain'].substr(0, 255) : null,
+    snippet: null,
     unread: !attributes.flags.includes('\\Seen'),
     starred: attributes.flags.includes('\\Flagged'),
     date: attributes.date,
@@ -90,29 +101,41 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
   // preserve whitespacing on plaintext emails -- has the side effect of monospacing, but
   // that seems OK and perhaps sometimes even desired (for e.g. ascii art, alignment)
   if (!body['text/html'] && body['text/plain']) {
-    values.body = `
${values.body}
`;
+    parsedMessage.body = `
${parsedMessage.body}
`;
   }
 
+  // populate initial snippet
+  if (body['text/plain']) {
+    parsedMessage.snippet = body['text/plain'].trim().substr(0, SNIPPET_MAX_SIZE);
+  } else if (parsedMessage.body) {
+    // create snippet from body, which is most likely html. we strip tags but
+    // don't currently support stripping embedded CSS
+    parsedMessage.snippet = striptags(parsedMessage.body).trim().substr(0,
+      Math.min(parsedMessage.body.length, SNIPPET_MAX_SIZE));
+  }
+
+  // clean up and trim snippet
+  if (parsedMessage.snippet) {
   // TODO: strip quoted text from snippets also
-  if (values.snippet) {
-    // trim and clean snippet which is alreay present (from values plaintext)
-    values.snippet = values.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ')
-    const loc = values.snippet.indexOf(' ', SNIPPET_SIZE);
-    if (loc !== -1) {
-      values.snippet = values.snippet.substr(0, loc);
+    parsedMessage.snippet = parsedMessage.snippet.replace(/[\n\r]/g, ' ').replace(/\s\s+/g, ' ')
+    // trim down to approx. SNIPPET_SIZE w/out cutting off words right in the
+    // middle (if possible)
+    const wordBreak = parsedMessage.snippet.indexOf(' ', SNIPPET_SIZE);
+    if (wordBreak !== -1) {
+      parsedMessage.snippet = parsedMessage.snippet.substr(0, wordBreak);
     }
-  } else if (values.body) {
-    // create snippet from body, which is most likely html
-    values.snippet = striptags(values.body).trim().substr(0, Math.min(values.body.length, SNIPPET_SIZE));
   }
 
-  values.folder = folder
+  parsedMessage.folder = folder
+
+  // TODO: unclear if this is necessary given we already have parsed labels
+  const xGmLabels = attributes['x-gm-labels']
   if (xGmLabels) {
-    values.folderImapXGMLabels = JSON.stringify(xGmLabels)
-    values.labels = await Label.findXGMLabels(xGmLabels)
+    parsedMessage.folderImapXGMLabels = JSON.stringify(xGmLabels)
+    parsedMessage.labels = await Label.findXGMLabels(xGmLabels)
   }
 
-  return values;
+  return parsedMessage;
 }
 
 function fromJSON(db, data) {

From 60bca4acb229437c1f42238851f6ec1aeefe2fd8 Mon Sep 17 00:00:00 2001
From: Mark Hahnenberg 
Date: Fri, 9 Dec 2016 11:12:47 -0800
Subject: [PATCH 492/800] [local-sync] Add index for Thread.id (#4)

---
 packages/local-sync/src/models/thread.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/local-sync/src/models/thread.js b/packages/local-sync/src/models/thread.js
index 7eef390de..05616cf85 100644
--- a/packages/local-sync/src/models/thread.js
+++ b/packages/local-sync/src/models/thread.js
@@ -23,6 +23,7 @@ module.exports = (sequelize, Sequelize) => {
     participants: buildJSONARRAYColumnOptions('participants'),
   }, {
     indexes: [
+      { fields: ['id'], unique: true },
       { fields: ['subject'] },
       { fields: ['remoteThreadId'] },
     ],

From f32d0df7e03cf451417f37037fcf001da942e8ca Mon Sep 17 00:00:00 2001
From: Mark Hahnenberg 
Date: Fri, 9 Dec 2016 11:15:04 -0800
Subject: [PATCH 493/800] [local-sync] Increment streamAll offset by chunkSize
 (#5)

Otherwise we'll infinite loop if there are more than 2000 results.
---
 packages/local-sync/src/shared/database-extensions.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/local-sync/src/shared/database-extensions.js b/packages/local-sync/src/shared/database-extensions.js
index 4c64073fb..5863a8b77 100644
--- a/packages/local-sync/src/shared/database-extensions.js
+++ b/packages/local-sync/src/shared/database-extensions.js
@@ -11,7 +11,7 @@ Sequelize.Model.prototype.streamAll = function streamAll(options = {}) {
       this.findAll(opts).then((models = []) => {
         observer.onNext(models)
         if (models.length === chunkSize) {
-          opts.offset = chunkSize;
+          opts.offset += chunkSize;
           findFn(opts)
         } else {
           observer.onCompleted()

From 5a5aeb6bb37133f5e376939b64f2abbb096bbd42 Mon Sep 17 00:00:00 2001
From: Evan Morikawa 
Date: Mon, 12 Dec 2016 09:43:01 -0500
Subject: [PATCH 494/800] [local-private] move old edgehill src/pro into
 packages/local-private

Summary: This is a test

Test Plan: Testing

Reviewers: juan

Differential Revision: https://phab.nylas.com/D3493

[local-private] add old edgehill src/pro into packages/local-private
---
 .arcconfig                                    |   8 +-
 .arclint                                      |  17 +
 arclib/.phutil_module_cache                   |   1 +
 arclib/__phutil_library_init__.php            |   3 +
 arclib/__phutil_library_map__.php             |  14 +
 packages/local-private/README.md              |  12 +
 .../docs/ContinuousIntegration.md             |  70 +++
 .../appveyor/set_win_env.ps1.enc              | Bin 0 -> 368 bytes
 .../appveyor/win-nylas-n1.p12.enc             | Bin 0 -> 3024 bytes
 .../travis/travis-files-mirror.tar.enc        | Bin 0 -> 13328 bytes
 .../travis/travis-files.tar.enc               | Bin 0 -> 13328 bytes
 .../packages/activity-list/assets/icon.png    | Bin 0 -> 16008 bytes
 .../lib/activity-data-source.es6              |  18 +
 .../lib/activity-list-actions.es6             |  11 +
 .../lib/activity-list-button.jsx              |  70 +++
 .../lib/activity-list-empty-state.jsx         |  21 +
 .../lib/activity-list-item-container.jsx      | 141 +++++
 .../activity-list/lib/activity-list-store.jsx | 208 +++++++
 .../activity-list/lib/activity-list.jsx       | 100 +++
 .../packages/activity-list/lib/main.es6       |  21 +
 .../activity-list/lib/plugin-helpers.es6      |  22 +
 .../activity-list/lib/test-data-source.es6    |  18 +
 .../packages/activity-list/package.json       |  21 +
 .../activity-list/spec/activity-list-spec.jsx | 195 ++++++
 .../stylesheets/activity-list.less            | 142 +++++
 .../packages/composer-mail-merge/icon.png     | Bin 0 -> 15465 bytes
 .../lib/listens-to-mail-merge-session.jsx     |  64 ++
 .../lib/mail-merge-body-token.jsx             | 132 ++++
 .../lib/mail-merge-button.jsx                 |  46 ++
 .../lib/mail-merge-composer-extension.es6     |  10 +
 .../lib/mail-merge-constants.es6              |  37 ++
 .../lib/mail-merge-container.jsx              |  39 ++
 .../lib/mail-merge-draft-editing-session.es6  | 146 +++++
 .../lib/mail-merge-header-input.jsx           |  67 ++
 .../mail-merge-participants-text-field.jsx    | 138 ++++
 .../lib/mail-merge-send-button.jsx            | 115 ++++
 .../lib/mail-merge-subject-text-field.jsx     | 132 ++++
 .../lib/mail-merge-table.jsx                  |  43 ++
 .../lib/mail-merge-token-dnd-handlers.es6     |  51 ++
 .../lib/mail-merge-token.jsx                  |  46 ++
 .../lib/mail-merge-utils.es6                  | 204 ++++++
 .../lib/mail-merge-workspace.jsx              | 159 +++++
 .../packages/composer-mail-merge/lib/main.es6 |  53 ++
 .../lib/selection-state-reducers.es6          | 122 ++++
 .../lib/send-many-drafts-task.es6             | 171 +++++
 .../lib/table-state-reducers.es6              |  95 +++
 .../lib/token-data-source.es6                 | 114 ++++
 .../lib/token-state-reducers.es6              | 122 ++++
 .../lib/workspace-state-reducers.es6          |  14 +
 .../packages/composer-mail-merge/package.json |  23 +
 .../composer-mail-merge/spec/fixtures.es6     |  38 ++
 .../mail-merge-draft-editing-session-spec.es6 | 108 ++++
 .../spec/mail-merge-utils-spec.es6            | 229 +++++++
 .../spec/selection-state-reducers-spec.es6    | 182 ++++++
 .../spec/send-many-drafts-task-spec.es6       | 251 ++++++++
 .../spec/table-state-reducers-spec.es6        | 142 +++++
 .../spec/token-state-reducers-spec.es6        | 155 +++++
 .../spec/workspace-state-reducers-spec.es6    |  29 +
 .../stylesheets/mail-merge.less               | 218 +++++++
 .../packages/composer-scheduler/README.md     |  27 +
 .../assets/ic-composer-scheduler@1x.png       | Bin 0 -> 1197 bytes
 .../assets/ic-composer-scheduler@2x.png       | Bin 0 -> 1445 bytes
 .../assets/ic-eventcard-description@2x.png    | Bin 0 -> 230 bytes
 .../assets/ic-eventcard-link@2x.png           | Bin 0 -> 788 bytes
 .../assets/ic-eventcard-location@2x.png       | Bin 0 -> 1260 bytes
 .../assets/ic-eventcard-notes@2x.png          | Bin 0 -> 511 bytes
 .../assets/ic-eventcard-people@2x.png         | Bin 0 -> 1095 bytes
 .../assets/ic-eventcard-reminder@2x.png       | Bin 0 -> 779 bytes
 .../assets/ic-eventcard-time@2x.png           | Bin 0 -> 1358 bytes
 .../packages/composer-scheduler/icon.png      | Bin 0 -> 14879 bytes
 .../proposed-time-calendar-data-source.es6    |  16 +
 .../lib/calendar/proposed-time-event.jsx      |  51 ++
 .../lib/calendar/proposed-time-picker.jsx     | 170 +++++
 .../lib/composer/email-b64-images.es6         |   7 +
 .../lib/composer/email-images.json            |   5 +
 .../lib/composer/event-datetime-input.jsx     |  68 ++
 .../lib/composer/event-prep-helper.es6        |  20 +
 .../lib/composer/new-event-card-container.jsx | 136 ++++
 .../lib/composer/new-event-card.jsx           | 257 ++++++++
 .../lib/composer/new-event-helper.es6         |  77 +++
 .../lib/composer/new-event-preview.jsx        | 123 ++++
 .../lib/composer/proposed-time-list.jsx       | 251 ++++++++
 .../lib/composer/remove-event-helper.es6      |  32 +
 .../composer/scheduler-composer-button.jsx    | 162 +++++
 .../composer/scheduler-composer-extension.es6 |  83 +++
 .../packages/composer-scheduler/lib/main.es6  |  59 ++
 .../composer-scheduler/lib/proposal.es6       |  11 +
 .../lib/proposed-time-calendar-store.es6      | 185 ++++++
 .../lib/scheduler-actions.es6                 |  30 +
 .../lib/scheduler-constants.es6               |  14 +
 .../packages/composer-scheduler/package.json  |  25 +
 .../spec/composer-scheduler-spec-helper.es6   |  67 ++
 .../spec/new-event-card-spec.jsx              | 218 +++++++
 .../spec/proposed-time-picker-spec.jsx        | 344 ++++++++++
 .../spec/scheduler-composer-button-spec.jsx   | 142 +++++
 .../scheduler-composer-extension-spec.es6     | 149 +++++
 .../spec/test-proposal-data-source.es6        |  24 +
 .../stylesheets/scheduler.less                | 170 +++++
 .../packages/link-tracking/README.md          |   4 +
 .../assets/ic-tracking-unvisited@1x.png       | Bin 0 -> 52653 bytes
 .../assets/ic-tracking-unvisited@2x.png       | Bin 0 -> 50738 bytes
 .../assets/ic-tracking-visited@1x.png         | Bin 0 -> 50790 bytes
 .../assets/ic-tracking-visited@2x.png         | Bin 0 -> 52136 bytes
 .../assets/linktracking-icon@2x.png           | Bin 0 -> 635 bytes
 .../packages/link-tracking/icon.png           | Bin 0 -> 15667 bytes
 .../lib/link-tracking-button.jsx              |  47 ++
 .../lib/link-tracking-composer-extension.es6  |  60 ++
 .../lib/link-tracking-constants.es6           |   5 +
 .../lib/link-tracking-message-extension.jsx   |  69 ++
 .../lib/link-tracking-message-popover.jsx     |  51 ++
 .../packages/link-tracking/lib/main.es6       |  32 +
 .../packages/link-tracking/package.json       |  31 +
 .../link-tracking-composer-extension-spec.es6 | 114 ++++
 .../link-tracking/stylesheets/main.less       |  77 +++
 .../lib/analytics-store.es6                   | 217 +++++++
 .../nylas-private-analytics/lib/main.es6      |   5 +
 .../nylas-private-analytics/package.json      |  19 +
 .../fonts/Nylas-Pro-Blond.otf                 | Bin 0 -> 171736 bytes
 .../fonts/Nylas-Pro-Hair.otf                  | Bin 0 -> 173956 bytes
 .../fonts/Nylas-Pro-Light.otf                 | Bin 0 -> 172912 bytes
 .../fonts/Nylas-Pro-Medium.otf                | Bin 0 -> 171528 bytes
 .../fonts/Nylas-Pro-Normal.otf                | Bin 0 -> 169000 bytes
 .../fonts/Nylas-Pro-SemiBold.otf              | Bin 0 -> 175380 bytes
 .../fonts/Nylas-Pro-Thin.otf                  | Bin 0 -> 171796 bytes
 .../packages/nylas-private-fonts/lib/main.es6 |   4 +
 .../packages/nylas-private-fonts/package.json |  14 +
 .../stylesheets/nylas-fonts.less              |  45 ++
 .../nylas-private-salesforce/.gitignore       |  29 +
 .../nylas-private-salesforce/README.md        |   3 +
 .../nylas-private-salesforce/icon.png         | Bin 0 -> 3304 bytes
 .../keymaps/salesforce.json                   |   3 +
 .../lib/composer/contact-search-results.jsx   |  64 ++
 .../lib/composer/participant-decorator.jsx    |  53 ++
 .../composer/salesforce-composer-picker.jsx   |  60 ++
 .../lib/contact/salesforce-contact-info.jsx   | 195 ++++++
 .../lib/form/fetch-empty-schema-for-type.es6  | 148 +++++
 .../lib/form/form-data-helpers.es6            | 130 ++++
 .../generated-form-to-salesforce-adapter.es6  |  68 ++
 .../lib/form/pending-salesforce-object.es6    |  28 +
 .../lib/form/remove-controls.jsx              |  68 ++
 .../lib/form/salesforce-object-form.jsx       | 257 ++++++++
 .../lib/form/salesforce-object-picker.jsx     | 359 +++++++++++
 .../lib/form/salesforce-schema-adapter.es6    | 589 ++++++++++++++++++
 .../lib/form/salesforce-window-launcher.es6   | 109 ++++
 .../lib/form/smart-fields.es6                 | 351 +++++++++++
 .../nylas-private-salesforce/lib/main.jsx     | 205 ++++++
 .../lib/metadata-helpers.es6                  | 206 ++++++
 .../lib/models/salesforce-object.es6          |  83 +++
 .../lib/models/salesforce-schema.es6          |  38 ++
 .../lib/related-object-helpers.es6            |  43 ++
 .../lib/salesforce-actions.es6                |  35 ++
 .../lib/salesforce-api-error.es6              |  32 +
 .../lib/salesforce-api.jsx                    | 135 ++++
 .../lib/salesforce-constants.es6              |   5 +
 .../lib/salesforce-contact-crawler.es6        |  88 +++
 .../lib/salesforce-data-reset.es6             |  31 +
 .../lib/salesforce-env.es6                    | 146 +++++
 .../lib/salesforce-error-reporter.es6         |  22 +
 .../lib/salesforce-intro-notification.jsx     |  61 ++
 .../salesforce-metadata-cleanup-listener.es6  | 115 ++++
 .../lib/salesforce-new-mail-listener.es6      |  56 ++
 .../lib/salesforce-oauth.jsx                  | 160 +++++
 .../lib/salesforce-object-helpers.es6         | 205 ++++++
 .../lib/salesforce-related-object-cache.es6   | 325 ++++++++++
 .../lib/salesforce-sync-worker.es6            |  77 +++
 .../search/salesforce-search-bar-results.jsx  |  81 +++
 .../lib/search/salesforce-search-indexer.es6  |  28 +
 .../open-in-salesforce-btn.jsx                |  29 +
 .../lib/shared-components/salesforce-icon.jsx |  43 ++
 .../salesforce-login-prompt.jsx               |  36 ++
 .../destroy-message-on-salesforce-task.es6    |  99 +++
 .../tasks/destroy-salesforce-object-task.es6  |  66 ++
 .../ensure-message-on-salesforce-task.es6     | 285 +++++++++
 ...manually-relate-salesforce-object-task.es6 | 130 ++++
 ...ual-relation-to-salesforce-object-task.es6 |  87 +++
 .../tasks/sync-salesforce-objects-task.es6    | 116 ++++
 ...ync-thread-activity-to-salesforce-task.es6 | 121 ++++
 .../tasks/syncback-salesforce-object-task.es6 | 227 +++++++
 .../upsert-opportunity-contact-role-task.es6  | 127 ++++
 .../lib/thread/related-objects-for-thread.jsx | 404 ++++++++++++
 ...lesforce-manually-relate-thread-button.jsx | 192 ++++++
 ...esforce-manually-relate-thread-popover.jsx | 116 ++++
 .../lib/thread/salesforce-sync-label.jsx      |  92 +++
 .../thread/salesforce-sync-message-status.jsx |  87 +++
 .../lib/thread/sync-thread-toggle.jsx         |  60 ++
 .../menus/salesforce.json                     |  19 +
 .../nylas-private-salesforce/package.json     |  24 +
 .../fixtures/opportunity-layouts-alt.json     |   1 +
 .../spec/fixtures/opportunity-layouts.json    |   1 +
 .../spec/form-builder-spec.jsx                |  52 ++
 .../spec/generate-test-data.es6               | 194 ++++++
 .../spec/salesforce-schema-adapter-spec.es6   |  68 ++
 .../syncback-salesforce-object-task-spec.es6  |   7 +
 .../static/images/cancel-button@2x.png        | Bin 0 -> 3260 bytes
 .../ic-salesforce-cloud-btn-large@2x.png      | Bin 0 -> 681 bytes
 .../ic-salesforce-cloud-btn-small@2x.png      | Bin 0 -> 420 bytes
 .../images/icon-salesforce-action@2x.png      | Bin 0 -> 1193 bytes
 .../images/icon-salesforce-cloud@2x.png       | Bin 0 -> 1354 bytes
 .../images/icon-salesforce-searchloupe@2x.png | Bin 0 -> 1085 bytes
 .../icon-salesforce-segment-error@2x.png      | Bin 0 -> 1012 bytes
 .../static/images/icons/account_120.png       | Bin 0 -> 1713 bytes
 .../static/images/icons/announcement_120.png  | Bin 0 -> 1511 bytes
 .../static/images/icons/apps_120.png          | Bin 0 -> 1743 bytes
 .../static/images/icons/article_120.png       | Bin 0 -> 1183 bytes
 .../static/images/icons/avatar_120.png        | Bin 0 -> 1598 bytes
 .../images/icons/avatar_loading_120.png       | Bin 0 -> 1500 bytes
 .../static/images/icons/campaign_120.png      | Bin 0 -> 3093 bytes
 .../images/icons/campaign_members_120.png     | Bin 0 -> 3764 bytes
 .../static/images/icons/canvas_120.png        | Bin 0 -> 1794 bytes
 .../static/images/icons/case_120.png          | Bin 0 -> 669 bytes
 .../static/images/icons/case_email_120.png    | Bin 0 -> 1588 bytes
 .../static/images/icons/client_120.png        | Bin 0 -> 1817 bytes
 .../images/icons/connected_apps_120.png       | Bin 0 -> 1423 bytes
 .../static/images/icons/contact_120.png       | Bin 0 -> 1470 bytes
 .../static/images/icons/custom_120.png        | Bin 0 -> 1245 bytes
 .../static/images/icons/dashboard_120.png     | Bin 0 -> 2399 bytes
 .../static/images/icons/default_120.png       | Bin 0 -> 1356 bytes
 .../static/images/icons/document_120.png      | Bin 0 -> 943 bytes
 .../static/images/icons/drafts_120.png        | Bin 0 -> 1422 bytes
 .../static/images/icons/email_IQ_120.png      | Bin 0 -> 2162 bytes
 .../static/images/icons/email_chatter_120.png | Bin 0 -> 1673 bytes
 .../static/images/icons/emailmessage_120.png  | Bin 0 -> 1673 bytes
 .../static/images/icons/empty_120.png         | Bin 0 -> 636 bytes
 .../static/images/icons/event_120.png         | Bin 0 -> 1608 bytes
 .../static/images/icons/file_120.png          | Bin 0 -> 1349 bytes
 .../static/images/icons/folder_120.png        | Bin 0 -> 927 bytes
 .../images/icons/generic_loading_120.png      | Bin 0 -> 1052 bytes
 .../static/images/icons/goals_120.png         | Bin 0 -> 1845 bytes
 .../static/images/icons/groups_120.png        | Bin 0 -> 2312 bytes
 .../static/images/icons/hierarchy_120.png     | Bin 0 -> 902 bytes
 .../static/images/icons/home_120.png          | Bin 0 -> 680 bytes
 .../static/images/icons/lead_120.png          | Bin 0 -> 1586 bytes
 .../static/images/icons/lead_convert_120.png  | Bin 0 -> 1242 bytes
 .../static/images/icons/link_120.png          | Bin 0 -> 2044 bytes
 .../static/images/icons/log_a_call_120.png    | Bin 0 -> 1773 bytes
 .../static/images/icons/note_120.png          | Bin 0 -> 1878 bytes
 .../static/images/icons/opportunity_120.png   | Bin 0 -> 1717 bytes
 .../static/images/icons/orders_120.png        | Bin 0 -> 2100 bytes
 .../static/images/icons/people_120.png        | Bin 0 -> 1598 bytes
 .../static/images/icons/photo_120.png         | Bin 0 -> 1786 bytes
 .../static/images/icons/poll_120.png          | Bin 0 -> 1107 bytes
 .../static/images/icons/portal_120.png        | Bin 0 -> 1269 bytes
 .../static/images/icons/post_120.png          | Bin 0 -> 1629 bytes
 .../static/images/icons/process_120.png       | Bin 0 -> 1481 bytes
 .../static/images/icons/product_120.png       | Bin 0 -> 941 bytes
 .../static/images/icons/quotes_120.png        | Bin 0 -> 1568 bytes
 .../static/images/icons/recent_120.png        | Bin 0 -> 2230 bytes
 .../static/images/icons/record_120.png        | Bin 0 -> 1357 bytes
 .../static/images/icons/report_120.png        | Bin 0 -> 1366 bytes
 .../static/images/icons/social_120.png        | Bin 0 -> 1713 bytes
 .../static/images/icons/solution_120.png      | Bin 0 -> 1506 bytes
 .../static/images/icons/task2_120.png         | Bin 0 -> 689 bytes
 .../static/images/icons/task_120.png          | Bin 0 -> 1164 bytes
 .../static/images/icons/team_member_120.png   | Bin 0 -> 2226 bytes
 .../images/icons/thanks_loading_120.png       | Bin 0 -> 1366 bytes
 .../static/images/icons/today_120.png         | Bin 0 -> 2295 bytes
 .../static/images/icons/topic_120.png         | Bin 0 -> 2231 bytes
 .../static/images/icons/user_120.png          | Bin 0 -> 1598 bytes
 .../static/images/salesforce-icon@2x.png      | Bin 0 -> 2290 bytes
 .../static/images/salesforce-logo@2x.png      | Bin 0 -> 42918 bytes
 .../static/images/toolbar-chevron@2x.png      | Bin 0 -> 48907 bytes
 .../static/images/toolbar-templates@2x.png    | Bin 0 -> 48713 bytes
 .../stylesheets/open-in-salesforce-btn.less   |  20 +
 .../stylesheets/salesforce-association.less   |  28 +
 .../stylesheets/salesforce-composer.less      |  18 +
 .../stylesheets/salesforce-contact.less       |  31 +
 .../stylesheets/salesforce-icon.less          |  84 +++
 .../stylesheets/salesforce-object-form.less   |  53 ++
 .../stylesheets/salesforce-object-picker.less | 272 ++++++++
 .../stylesheets/salesforce-picker.less        |  37 ++
 .../salesforce-related-object.less            |  70 +++
 .../stylesheets/salesforce-sync-label.less    |  18 +
 .../salesforce-sync-message-status.less       |  11 +
 .../stylesheets/salesforce-welcome-view.less  |  15 +
 .../stylesheets/search-results.less           |   7 +
 .../stylesheets/sync-thread-toggle.less       |   5 +
 .../NYLAS_UI_Confirm_v1.ogg                   | Bin 0 -> 15078 bytes
 .../NYLAS_UI_HitSend_v1.ogg                   | Bin 0 -> 4693 bytes
 .../NYLAS_UI_NewMail_v1.ogg                   | Bin 0 -> 16342 bytes
 .../nylas-private-sounds/NYLAS_UI_Send_v1.ogg | Bin 0 -> 16594 bytes
 .../nylas-private-sounds/lib/main.es6         |  17 +
 .../nylas-private-sounds/package.json         |  14 +
 .../packages/open-tracking/README.md          |   4 +
 .../assets/InMessage-opened@1x.png            | Bin 0 -> 1383 bytes
 .../assets/InMessage-opened@2x.png            | Bin 0 -> 1941 bytes
 .../assets/icon-composer-eye@1x.png           | Bin 0 -> 1738 bytes
 .../assets/icon-composer-eye@2x.png           | Bin 0 -> 2799 bytes
 .../assets/icon-tracking-opened@2x.png        | Bin 0 -> 975 bytes
 .../packages/open-tracking/icon.png           | Bin 0 -> 16804 bytes
 .../packages/open-tracking/lib/main.es6       |  36 ++
 .../lib/open-tracking-button.jsx              |  52 ++
 .../lib/open-tracking-composer-extension.es6  |  36 ++
 .../lib/open-tracking-constants.es6           |   5 +
 .../open-tracking/lib/open-tracking-icon.jsx  |  89 +++
 .../lib/open-tracking-message-popover.jsx     |  51 ++
 .../lib/open-tracking-message-status.jsx      |  79 +++
 .../packages/open-tracking/package.json       |  31 +
 .../open-tracking-composer-extension-spec.es6 |  91 +++
 .../spec/open-tracking-icon-spec.jsx          |  71 +++
 .../open-tracking-message-status-spec.jsx     |  49 ++
 .../open-tracking/stylesheets/main.less       |  88 +++
 .../packages/send-later/icon.png              | Bin 0 -> 15742 bytes
 .../packages/send-later/lib/main.es6          |  23 +
 .../send-later/lib/send-later-button.jsx      | 143 +++++
 .../send-later/lib/send-later-constants.es6   |   4 +
 .../send-later/lib/send-later-popover.jsx     |  48 ++
 .../send-later/lib/send-later-status.jsx      |  43 ++
 .../packages/send-later/package.json          |  22 +
 .../spec/send-later-button-spec.jsx           |  97 +++
 .../spec/send-later-popover-spec.jsx          |  27 +
 .../send-later/stylesheets/send-later.less    |  29 +
 .../packages/send-reminders/icon.png          | Bin 0 -> 15329 bytes
 .../packages/send-reminders/lib/main.es6      |  37 ++
 ...nd-reminders-account-sidebar-extension.es6 |  13 +
 .../lib/send-reminders-composer-button.jsx    |  97 +++
 .../lib/send-reminders-constants.es6          |   4 +
 .../lib/send-reminders-headers.jsx            |  69 ++
 .../send-reminders-mailbox-perspective.es6    |  46 ++
 .../lib/send-reminders-popover-button.jsx     |  86 +++
 .../lib/send-reminders-popover.jsx            |  64 ++
 .../lib/send-reminders-query-subscription.es6 |  42 ++
 .../lib/send-reminders-store.es6              |  63 ++
 .../send-reminders-thread-list-extension.es6  |  19 +
 .../lib/send-reminders-thread-timestamp.jsx   | 105 ++++
 .../lib/send-reminders-toolbar-button.jsx     |  36 ++
 .../lib/send-reminders-utils.es6              | 117 ++++
 .../packages/send-reminders/package.json      |  21 +
 .../stylesheets/send-reminders.less           |  89 +++
 .../thread-sharing/lib/copy-button.jsx        |  52 ++
 .../thread-sharing/lib/external-threads.es6   |  56 ++
 .../packages/thread-sharing/lib/main.es6      |  15 +
 .../lib/thread-sharing-button.jsx             |  56 ++
 .../lib/thread-sharing-constants.es6          |   5 +
 .../lib/thread-sharing-popover.jsx            | 138 ++++
 .../packages/thread-sharing/package.json      |  18 +
 .../thread-sharing/stylesheets/main.less      |  51 ++
 .../nylas-private-error-reporter.js           | 104 ++++
 337 files changed, 18918 insertions(+), 2 deletions(-)
 create mode 100644 .arclint
 create mode 100644 arclib/.phutil_module_cache
 create mode 100644 arclib/__phutil_library_init__.php
 create mode 100644 arclib/__phutil_library_map__.php
 create mode 100644 packages/local-private/README.md
 create mode 100644 packages/local-private/docs/ContinuousIntegration.md
 create mode 100755 packages/local-private/encrypted_certificates/appveyor/set_win_env.ps1.enc
 create mode 100755 packages/local-private/encrypted_certificates/appveyor/win-nylas-n1.p12.enc
 create mode 100644 packages/local-private/encrypted_certificates/travis/travis-files-mirror.tar.enc
 create mode 100644 packages/local-private/encrypted_certificates/travis/travis-files.tar.enc
 create mode 100644 packages/local-private/packages/activity-list/assets/icon.png
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-data-source.es6
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-list-actions.es6
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-list-button.jsx
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-list-empty-state.jsx
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-list-item-container.jsx
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-list-store.jsx
 create mode 100644 packages/local-private/packages/activity-list/lib/activity-list.jsx
 create mode 100644 packages/local-private/packages/activity-list/lib/main.es6
 create mode 100644 packages/local-private/packages/activity-list/lib/plugin-helpers.es6
 create mode 100644 packages/local-private/packages/activity-list/lib/test-data-source.es6
 create mode 100644 packages/local-private/packages/activity-list/package.json
 create mode 100644 packages/local-private/packages/activity-list/spec/activity-list-spec.jsx
 create mode 100644 packages/local-private/packages/activity-list/stylesheets/activity-list.less
 create mode 100644 packages/local-private/packages/composer-mail-merge/icon.png
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/listens-to-mail-merge-session.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-body-token.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-button.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-composer-extension.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-constants.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-container.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-draft-editing-session.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-header-input.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-participants-text-field.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-send-button.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-subject-text-field.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-table.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-token-dnd-handlers.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-token.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-utils.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/mail-merge-workspace.jsx
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/main.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/selection-state-reducers.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/send-many-drafts-task.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/table-state-reducers.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/token-data-source.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/token-state-reducers.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/lib/workspace-state-reducers.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/package.json
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/fixtures.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/mail-merge-draft-editing-session-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/mail-merge-utils-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/selection-state-reducers-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/send-many-drafts-task-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/table-state-reducers-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/token-state-reducers-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/spec/workspace-state-reducers-spec.es6
 create mode 100644 packages/local-private/packages/composer-mail-merge/stylesheets/mail-merge.less
 create mode 100644 packages/local-private/packages/composer-scheduler/README.md
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-composer-scheduler@1x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-composer-scheduler@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-description@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-link@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-location@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-notes@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-people@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-reminder@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/assets/ic-eventcard-time@2x.png
 create mode 100644 packages/local-private/packages/composer-scheduler/icon.png
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-calendar-data-source.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-event.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-picker.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/email-b64-images.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/email-images.json
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/event-datetime-input.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/event-prep-helper.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/new-event-card-container.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/new-event-card.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/new-event-helper.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/new-event-preview.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/proposed-time-list.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/remove-event-helper.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/scheduler-composer-button.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/composer/scheduler-composer-extension.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/main.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/proposal.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/proposed-time-calendar-store.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/scheduler-actions.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/lib/scheduler-constants.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/package.json
 create mode 100644 packages/local-private/packages/composer-scheduler/spec/composer-scheduler-spec-helper.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/spec/new-event-card-spec.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/spec/proposed-time-picker-spec.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/spec/scheduler-composer-button-spec.jsx
 create mode 100644 packages/local-private/packages/composer-scheduler/spec/scheduler-composer-extension-spec.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/spec/test-proposal-data-source.es6
 create mode 100644 packages/local-private/packages/composer-scheduler/stylesheets/scheduler.less
 create mode 100644 packages/local-private/packages/link-tracking/README.md
 create mode 100644 packages/local-private/packages/link-tracking/assets/ic-tracking-unvisited@1x.png
 create mode 100644 packages/local-private/packages/link-tracking/assets/ic-tracking-unvisited@2x.png
 create mode 100644 packages/local-private/packages/link-tracking/assets/ic-tracking-visited@1x.png
 create mode 100644 packages/local-private/packages/link-tracking/assets/ic-tracking-visited@2x.png
 create mode 100644 packages/local-private/packages/link-tracking/assets/linktracking-icon@2x.png
 create mode 100644 packages/local-private/packages/link-tracking/icon.png
 create mode 100644 packages/local-private/packages/link-tracking/lib/link-tracking-button.jsx
 create mode 100644 packages/local-private/packages/link-tracking/lib/link-tracking-composer-extension.es6
 create mode 100644 packages/local-private/packages/link-tracking/lib/link-tracking-constants.es6
 create mode 100644 packages/local-private/packages/link-tracking/lib/link-tracking-message-extension.jsx
 create mode 100644 packages/local-private/packages/link-tracking/lib/link-tracking-message-popover.jsx
 create mode 100644 packages/local-private/packages/link-tracking/lib/main.es6
 create mode 100644 packages/local-private/packages/link-tracking/package.json
 create mode 100644 packages/local-private/packages/link-tracking/spec/link-tracking-composer-extension-spec.es6
 create mode 100644 packages/local-private/packages/link-tracking/stylesheets/main.less
 create mode 100644 packages/local-private/packages/nylas-private-analytics/lib/analytics-store.es6
 create mode 100644 packages/local-private/packages/nylas-private-analytics/lib/main.es6
 create mode 100644 packages/local-private/packages/nylas-private-analytics/package.json
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-Blond.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-Hair.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-Light.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-Medium.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-Normal.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-SemiBold.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/fonts/Nylas-Pro-Thin.otf
 create mode 100644 packages/local-private/packages/nylas-private-fonts/lib/main.es6
 create mode 100755 packages/local-private/packages/nylas-private-fonts/package.json
 create mode 100644 packages/local-private/packages/nylas-private-fonts/stylesheets/nylas-fonts.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/.gitignore
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/README.md
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/icon.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/keymaps/salesforce.json
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/composer/contact-search-results.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/composer/participant-decorator.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/composer/salesforce-composer-picker.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/contact/salesforce-contact-info.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/fetch-empty-schema-for-type.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/form-data-helpers.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/generated-form-to-salesforce-adapter.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/pending-salesforce-object.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/remove-controls.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/salesforce-object-form.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/salesforce-object-picker.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/salesforce-schema-adapter.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/salesforce-window-launcher.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/form/smart-fields.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/main.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/metadata-helpers.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/models/salesforce-object.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/models/salesforce-schema.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/related-object-helpers.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-actions.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-api-error.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-api.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-constants.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-contact-crawler.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-data-reset.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-env.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-error-reporter.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-intro-notification.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-metadata-cleanup-listener.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-new-mail-listener.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-oauth.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-object-helpers.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-related-object-cache.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/salesforce-sync-worker.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/search/salesforce-search-bar-results.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/search/salesforce-search-indexer.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/shared-components/open-in-salesforce-btn.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/shared-components/salesforce-icon.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/shared-components/salesforce-login-prompt.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/destroy-message-on-salesforce-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/destroy-salesforce-object-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/ensure-message-on-salesforce-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/manually-relate-salesforce-object-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/remove-manual-relation-to-salesforce-object-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/sync-salesforce-objects-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/sync-thread-activity-to-salesforce-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/syncback-salesforce-object-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/tasks/upsert-opportunity-contact-role-task.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/thread/related-objects-for-thread.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/thread/salesforce-manually-relate-thread-button.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/thread/salesforce-manually-relate-thread-popover.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/thread/salesforce-sync-label.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/thread/salesforce-sync-message-status.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/lib/thread/sync-thread-toggle.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/menus/salesforce.json
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/package.json
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/spec/fixtures/opportunity-layouts-alt.json
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/spec/fixtures/opportunity-layouts.json
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/spec/form-builder-spec.jsx
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/spec/generate-test-data.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/spec/salesforce-schema-adapter-spec.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/spec/syncback-salesforce-object-task-spec.es6
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/cancel-button@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/ic-salesforce-cloud-btn-large@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/ic-salesforce-cloud-btn-small@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icon-salesforce-action@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icon-salesforce-cloud@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icon-salesforce-searchloupe@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icon-salesforce-segment-error@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/account_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/announcement_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/apps_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/article_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/avatar_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/avatar_loading_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/campaign_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/campaign_members_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/canvas_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/case_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/case_email_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/client_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/connected_apps_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/contact_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/custom_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/dashboard_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/default_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/document_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/drafts_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/email_IQ_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/email_chatter_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/emailmessage_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/empty_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/event_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/file_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/folder_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/generic_loading_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/goals_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/groups_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/hierarchy_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/home_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/lead_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/lead_convert_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/link_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/log_a_call_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/note_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/opportunity_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/orders_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/people_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/photo_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/poll_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/portal_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/post_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/process_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/product_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/quotes_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/recent_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/record_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/report_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/social_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/solution_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/task2_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/task_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/team_member_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/thanks_loading_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/today_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/topic_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/icons/user_120.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/salesforce-icon@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/salesforce-logo@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/toolbar-chevron@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/static/images/toolbar-templates@2x.png
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/open-in-salesforce-btn.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-association.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-composer.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-contact.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-icon.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-object-form.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-object-picker.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-picker.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-related-object.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-sync-label.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-sync-message-status.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/salesforce-welcome-view.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/search-results.less
 create mode 100644 packages/local-private/packages/nylas-private-salesforce/stylesheets/sync-thread-toggle.less
 create mode 100644 packages/local-private/packages/nylas-private-sounds/NYLAS_UI_Confirm_v1.ogg
 create mode 100644 packages/local-private/packages/nylas-private-sounds/NYLAS_UI_HitSend_v1.ogg
 create mode 100644 packages/local-private/packages/nylas-private-sounds/NYLAS_UI_NewMail_v1.ogg
 create mode 100644 packages/local-private/packages/nylas-private-sounds/NYLAS_UI_Send_v1.ogg
 create mode 100644 packages/local-private/packages/nylas-private-sounds/lib/main.es6
 create mode 100644 packages/local-private/packages/nylas-private-sounds/package.json
 create mode 100644 packages/local-private/packages/open-tracking/README.md
 create mode 100644 packages/local-private/packages/open-tracking/assets/InMessage-opened@1x.png
 create mode 100644 packages/local-private/packages/open-tracking/assets/InMessage-opened@2x.png
 create mode 100644 packages/local-private/packages/open-tracking/assets/icon-composer-eye@1x.png
 create mode 100644 packages/local-private/packages/open-tracking/assets/icon-composer-eye@2x.png
 create mode 100644 packages/local-private/packages/open-tracking/assets/icon-tracking-opened@2x.png
 create mode 100644 packages/local-private/packages/open-tracking/icon.png
 create mode 100644 packages/local-private/packages/open-tracking/lib/main.es6
 create mode 100644 packages/local-private/packages/open-tracking/lib/open-tracking-button.jsx
 create mode 100644 packages/local-private/packages/open-tracking/lib/open-tracking-composer-extension.es6
 create mode 100644 packages/local-private/packages/open-tracking/lib/open-tracking-constants.es6
 create mode 100644 packages/local-private/packages/open-tracking/lib/open-tracking-icon.jsx
 create mode 100644 packages/local-private/packages/open-tracking/lib/open-tracking-message-popover.jsx
 create mode 100644 packages/local-private/packages/open-tracking/lib/open-tracking-message-status.jsx
 create mode 100644 packages/local-private/packages/open-tracking/package.json
 create mode 100644 packages/local-private/packages/open-tracking/spec/open-tracking-composer-extension-spec.es6
 create mode 100644 packages/local-private/packages/open-tracking/spec/open-tracking-icon-spec.jsx
 create mode 100644 packages/local-private/packages/open-tracking/spec/open-tracking-message-status-spec.jsx
 create mode 100644 packages/local-private/packages/open-tracking/stylesheets/main.less
 create mode 100644 packages/local-private/packages/send-later/icon.png
 create mode 100644 packages/local-private/packages/send-later/lib/main.es6
 create mode 100644 packages/local-private/packages/send-later/lib/send-later-button.jsx
 create mode 100644 packages/local-private/packages/send-later/lib/send-later-constants.es6
 create mode 100644 packages/local-private/packages/send-later/lib/send-later-popover.jsx
 create mode 100644 packages/local-private/packages/send-later/lib/send-later-status.jsx
 create mode 100644 packages/local-private/packages/send-later/package.json
 create mode 100644 packages/local-private/packages/send-later/spec/send-later-button-spec.jsx
 create mode 100644 packages/local-private/packages/send-later/spec/send-later-popover-spec.jsx
 create mode 100644 packages/local-private/packages/send-later/stylesheets/send-later.less
 create mode 100644 packages/local-private/packages/send-reminders/icon.png
 create mode 100644 packages/local-private/packages/send-reminders/lib/main.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-account-sidebar-extension.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-composer-button.jsx
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-constants.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-headers.jsx
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-mailbox-perspective.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-popover-button.jsx
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-popover.jsx
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-query-subscription.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-store.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-thread-list-extension.es6
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-thread-timestamp.jsx
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-toolbar-button.jsx
 create mode 100644 packages/local-private/packages/send-reminders/lib/send-reminders-utils.es6
 create mode 100644 packages/local-private/packages/send-reminders/package.json
 create mode 100644 packages/local-private/packages/send-reminders/stylesheets/send-reminders.less
 create mode 100644 packages/local-private/packages/thread-sharing/lib/copy-button.jsx
 create mode 100644 packages/local-private/packages/thread-sharing/lib/external-threads.es6
 create mode 100644 packages/local-private/packages/thread-sharing/lib/main.es6
 create mode 100644 packages/local-private/packages/thread-sharing/lib/thread-sharing-button.jsx
 create mode 100644 packages/local-private/packages/thread-sharing/lib/thread-sharing-constants.es6
 create mode 100644 packages/local-private/packages/thread-sharing/lib/thread-sharing-popover.jsx
 create mode 100644 packages/local-private/packages/thread-sharing/package.json
 create mode 100644 packages/local-private/packages/thread-sharing/stylesheets/main.less
 create mode 100644 packages/local-private/src/error-logger-extensions/nylas-private-error-reporter.js

diff --git a/.arcconfig b/.arcconfig
index 00ab68a97..c3db3501f 100644
--- a/.arcconfig
+++ b/.arcconfig
@@ -1,4 +1,8 @@
 {
-  "project_id" : "K2",
-  "conduit_uri" : "https://phab.nylas.com/"
+  "project_id" : "N1",
+  "conduit_uri" : "https://phab.nylas.com/",
+  "load" : [
+    "arclib"
+  ],
+  "lint.engine": "ArcanistConfigurationDrivenLintEngine"
 }
diff --git a/.arclint b/.arclint
new file mode 100644
index 000000000..a96d36a22
--- /dev/null
+++ b/.arclint
@@ -0,0 +1,17 @@
+{
+  "linters": {
+    "coffeescript-linter": {
+      "type": "script-and-regex",
+      "script-and-regex.script": "script/grunt coffeelint:target --target",
+      "script-and-regex.regex": "/^  ((?P✖)|(?P⚠)) *line (?P\\d+) +(?P.*)$/m",
+      "include": "{\\.(e?coffee|cjsx)}"
+    },
+    "eslint-regex-based": {
+      "type": "script-and-regex",
+      "include": ["(\\.jsx?$)", "(\\.es6$)"],
+      "exclude": ["(src\\/K2)", "(node_modules)"],
+      "script-and-regex.script": "sh -c '([ -e ./node_modules/.bin/eslint ]) && (./node_modules/.bin/eslint -f compact \"$0\" || true)'",
+      "script-and-regex.regex": "/^(?P.*): line (?P[0-9]*), col (?P[0-9]*), (?PWarning|Error) - (?P.*?)(\\((?P[a-z-]+)\\))?$/m"
+    }
+  }
+}
diff --git a/arclib/.phutil_module_cache b/arclib/.phutil_module_cache
new file mode 100644
index 000000000..b7d92402c
--- /dev/null
+++ b/arclib/.phutil_module_cache
@@ -0,0 +1 @@
+{"__symbol_cache_version__":11}
\ No newline at end of file
diff --git a/arclib/__phutil_library_init__.php b/arclib/__phutil_library_init__.php
new file mode 100644
index 000000000..93cfbaabb
--- /dev/null
+++ b/arclib/__phutil_library_init__.php
@@ -0,0 +1,3 @@
+ 2,
+  'class' => array(),
+  'function' => array(),
+  'xmap' => array(),
+));
diff --git a/packages/local-private/README.md b/packages/local-private/README.md
new file mode 100644
index 000000000..8872f5696
--- /dev/null
+++ b/packages/local-private/README.md
@@ -0,0 +1,12 @@
+# Nylas Mail
+
+This repo contains proprietary Nylas plugins and other extensions to N1
+
+It is included as a submodule of the open source N1 repo at
+`pro/nylas`
+
+From the root of N1, run `script/grunt add-nylas-build-resources` to manually
+copy the files from this repo into the appropriate places within N1.
+
+That script is run as part of the N1 `build` task. Machines that have access
+this repo will automatically include the proprietary plugins.
diff --git a/packages/local-private/docs/ContinuousIntegration.md b/packages/local-private/docs/ContinuousIntegration.md
new file mode 100644
index 000000000..02b7f8ef0
--- /dev/null
+++ b/packages/local-private/docs/ContinuousIntegration.md
@@ -0,0 +1,70 @@
+# Building N1 with Continuous Integration
+
+    script/grunt ci
+
+N1 is designed to be built into a production app for Mac, Windows, and Linux.
+Only Nylas core team members currently have access to produce a production
+build.
+
+Production builds are code-signed with a Nylas, Inc. certificate and include a
+handful of other proprietary assets such as custom fonts and sounds.
+
+We currently use [Travis](https://travis-ci.org/nylas/N1) to build on Mac &
+Windows and AppVeyor to build on Windows.
+
+A build can be run from a local machines by Jenkins or manually; however,
+several environment variables must be setup.:
+
+**ALL ENVIRONMENT VARIABLES ARE ENCRYPTED**
+
+They exist in an encrypted file that only Travis can read in
+`build/resources/certs/set_env.sh`
+
+**IMPORTANT** Do NOT remove the `2>/dev/null 1>/dev/null` in the
+`before_install` scripts. If any of commands fail we don't want to leak
+sensitive data in the output.
+
+That file must be decrypted and `source`d before the environment variables can
+use.
+
+If not building on Travis, the environment variables must be manually decrypted
+via gpg and sourced
+
+We use [Travis encryption](https://docs.travis-ci.com/user/encrypting-files/)
+and AppVeyor encryption to store the certificates, keys, and passwords
+
+To login to GitHub and clone the Nylas submodule with private assets you need
+to clone recursively (or `git submodule init; git submodule update`) with a
+valid SSH key or login username and password.
+
+We have a CI GitHub account: https://github.com/nylas-deploy-scripts
+The password for that account is stored in the environment variable:
+- `GITHUB_CI_ACCOUNT_PASSWORD`
+
+For signing builds on Mac only when the certificates are already in the
+Keychain (not Travis):
+- `XCODE_KEYCHAIN` - The name of the Mac keychain that contains the
+  certificates and private key.
+- `XCODE_KEYCHAIN_PASSWORD` - Th password to that keychain.
+- `KEYCHAIN_ACCESS` - Alternatively, the `XCODE_KEYCHAIN` and
+  `XCODE_KEYCHAIN_PASSWORD` in a single colon-separated string.
+
+Alternatively, on Travis we decrypt the actual certificate files and create a
+temporary keychain. To do this we need the password to the private key. That's
+stored in:
+- `APPLE_CODESIGN_KEY_PASSWORD`
+
+For signing builds on Windows only:
+- `CERTIFICATE_FILE` - The Windows certificate
+- `CERTIFICATE_PASSWORD` - The password for the private key on the cert
+
+To download Electron:
+- `NYLAS_GITHUB_OAUTH_TOKEN` - The OAuth token to use for GitHub API requests. See
+  https://github.com/atom/grunt-download-electron
+
+To upload built artifacts to S3:
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+
+To notify when builds are done:
+- `NYLAS_INTERNAL_HOOK_URL` - Nylas internal Slack token and url
diff --git a/packages/local-private/encrypted_certificates/appveyor/set_win_env.ps1.enc b/packages/local-private/encrypted_certificates/appveyor/set_win_env.ps1.enc
new file mode 100755
index 0000000000000000000000000000000000000000..484a3a3d8a55f7b4314aa62cabfb5ae371fa039a
GIT binary patch
literal 368
zcmV-$0gwLelOf&aXh&{NGvFxM$guNKaH?I@sd48oI4zGyt2v9UIJ;1pG(^3mvesge
zTIRWIBmmT;e(oGpIZ?=eUnNO(Z;^rK+|3X+6Z>p-FWwsbp`n2qRoBA`aViJ{*(kQ4
z!Bw}OE$B{_X`c;kt+N)x)I|>AQi3a3NZYgzindaOS2vk9(u>oHVu4xd2UGdn!+lJ`
z#wzCMowC+|{?psLmjh-RsIuIL@)hpBI=cdGs&D>Lfc?juMv24qdL8zcT{V5x?CXRj
z6nTm_X0{>aA3wl>U;Bmg3ommgPwy^Kjnaj~;M;VoJ1@G?O^Q;e85fpfs~>nx)+t2x
z=K#pY1+>B>eS?w|v+?^Sun?0g!nnx|Hp_9^5lP(^7i9$`XYwNR(jOQE-{q|G(mG}
O_24wLDYIC4@oyzI>c5Tv

literal 0
HcmV?d00001

diff --git a/packages/local-private/encrypted_certificates/appveyor/win-nylas-n1.p12.enc b/packages/local-private/encrypted_certificates/appveyor/win-nylas-n1.p12.enc
new file mode 100755
index 0000000000000000000000000000000000000000..210cb2777d1f897b745de564335f06474aa52997
GIT binary patch
literal 3024
zcmV;>3orEXZN)GeW9Mjl6?`SD7Si?Kfc_7crJT6k7zxNo%wiUk_`QUnvxsGpl}gtm
zkWG|Nl0>iHH4+dd?9oROhffnlK-?!_w+lj*b3_s}Tj@2+l}K)D6NxQ52uu&TBPrg3
zKl7)Dbj#B3(19Zu8n666=#>B>Ac>MTc_lO=t*3ILT_X-b)1Rb+}gip$!QHin(=$QHN%Z@kqJqW^Y=!HKLrd{g_j`^zQSz_TV6V!7+CZ4Y=NSYwhY9^C*P!|)6dSfI&%{0Z#wl=w(!8*sF;`%;rO
z8b#6gs)M)3h6>0YD+(t>9#gZb!e0uOHV&APnhFi3?ZE2v&91NYs#Xlm)B!}lR9wI#
zTQ~fuG?=W}`;HH@NNgt843839tPM5~jj{e(rFl@3;QyhSH{{#-ZATHb`J<2l19#lVAp|A|8k$bQ2Uqc)$iqV(%{6#ece
z88>tj8s>bobC1F$Zj@i@w`7GfBx7<;in4wRQ+n%$O}nR@Qm#SQ*IwNOo4^H@p+8DR
zMwQfwn+gRG1^Ds7x$%h++b5d)KtS*Csat+vgMjEU~4_#Ed`sh*a^$)
z@dNXB?F)4ggLZ)v;RG_yp>>;6xl0=JAvrOlaz*N1RY@)Gx_d#he*?>3jW$#$C)Ijs
zwpu2&9T|@G{kN%6Y7NPA2gb0C4OG-MQwB2GXYvi`aE=^BBvcaW4ofBY-D%6*96`VM
zN~y`Co8AU}4`879-b(JE9NA7R-U6v0^tP00WB}p?x&zC+Y^Y)9<|^etcnWGAeoSMy
z^n}A!_#DWmkXIi9uT&fjmN9KhY(51iE@aLzl`AhH#Y!{3E%-6YN;wX-pMjIUz%`x*U&i3g{z#cL_>=Yx~M(@!RN&vsV_SlQNmV+ImWj+CK0haqs
z9f`AvNH`TkZKk(vltfGe&mM7R0}m+0gx8NTA4{mb%XLQ~&uw$73f?&m1Uq(Aly|bG
zK0L_JCmpSPN4d|)jLpc9d=md9pNhpfqZ;ogd4#R!Rq`*)_t>E8c$=nTplYzE7h^~w
zZczMS{qg<=NpVy2%cxU4qq^$%jH+AI2EoWNTJ~_Ggh5OJL~`!HE(f>(5G@g55NREM
zeDON2FbtS)Qm8F3clKc~|8tCNA5*(JPRj;TQ7J&jFjP3q=}}w|Kp9jdWLh34_c{9l
zk5b6RiarK(&7_WQP2a;`s|Q3%;JUArglC~%xlXcs$21lo;PK#0$fwZiGG?9na6Jf+
z*85&GHQww{WL7La8k?KriReErg$*PVzMWeSYa3~FZbxGAVk_mI6M0ZTdkmLx2jYPN
zJJg@nB1O}6r%%4vBeY^`&39`DOgH;FC`In4=*Y(nDKXv&V;;GSmQtV^mXtD4%|
zhgb_(;Xg9k6~!t%amg7^XTyYD1#*CCG>8Sn7Y;UWYn$7o+XHpL6b??usdGLG304;+
zJXPLgxjI>lAdrY}O-LS`G3yJ3`-hxyRc?37f
zW(X@O-EpwwwoS*6(5(N|G2Ef@EcQ9dG4+PY_kits%1cDEH7=e;9vVPAerP)H7W;p2
zpYPF8-xX!I1r?eoR^f9A72dmcL+i7;&bcTkT2)&G+mx3nNR^Vd!Y5KED5hvSK218q
z>IJ?Hf#6X0X4T)DQ6Egg@g!;7J&=EeLAQ1HT8)CGwq)OMwXskWupeZD63U&5+j^r8
zDI}MIF%Te~D0*Y^`EQ=Bt>lc`aY&U4`3?&D>A0h1wi=FcPgRu0CNUDV32Pg$5yT
zrA!6Fm{z0<^#|-!&1k>JhNGujqENcJ(_SS797Z^M4o-O6&v0|0s_u&%h!i
zHZ~UXJdGcegq~2W5CkH~zW82&!mywBIHREP@CELfRn5H~S{#FM^`UKI{lP!v9bL1T
zT(aFrRUQSyDaV|j(kj`Clu!=7){5T7<3#kQJ4t4HS>v^^TZ$+>@oUd!7_AO6`lrko
z6>OxzjBD6LQmWpHW0Yso9d`5oM<7xA!PBXDc##>PQ#uBy7|GY7>M`m{5ewDdcW#fY3&aGXP&jC-6
z{E=QJ$FjBSh;}cTb*m1oC$ONkFmQ)_Q9M^GA#l<>fTw*H3jvU_hUwmdOcv6juiZR()9R~2qSuEo+amI36^}@F5z=ZR42KP%6
z`0PtF89{mOm-MOmsjM8nb~nS6M#V~)7KJ4kp;L)pEN0ueJGyt{-Sg0b&To#?F`#19
z*Xz@gu~n3eYy4c;zeMP8XJRdzBi{QYsXDQtt4rrQE{+1JXDh2fshJ)&In++AvGUlp
zN*gf$9oquz;M=b%6v2SOFthnW6!LTSPC7u;I2QdHY!uTG;Wjf3n92@J4+b+32C6Uh
z(L(Tyd&}_2>;*`a#!{Pp@xBT_XU)CR*-C$EeobQ
zd#orwnem(H2{R4j?>ZYx>0@gFFc+c2=7i_F-mRVp(K+7(Pl&FQpTaLFM2SrWChzps
zSuKVUvZ9b9-qUCl!*^Zt6eVmXLE>(K12gzumlUNzf=Va`I~;Pj6E7?H05zaWB#ZT%
zx!?oHa~a81a0uu!YJz2atrGoLla%(pVYbzL^-p~L(${In$u6~bwShff?d9q{Vupl8w{L6NsRSaw;l~S-1MU(|DIO`A{b^iKMQsL8+W=pdDyz
z1LV&R)QfAZof7<9L1&l3-8%=Dg&3ByZuXDInEAL&{KG-V9W@J5ez*?&Q(L}IczO%B
z$=B1l;zh~Y9g-8Rp0cLbeaYp=a-eFyj78;vI5qVj7CV$_hQe?1jYDRSBBAG$^<9Vm
z*`DC7W2H?2QkGBl#L?iuZw81AamGD#5dkl>0KnBk9u^P-h}k4HD^O+f2^0VYB{mi6
zgU+ERC+v{2Vsgzng7qK#&X#v#PE#_TYutMpF1=~Y_*?66N3hx&f~tUvh5z`8NQy<%
zwTAQ%c^GcHKO`~dESSbfrdWdSn?JDFgyGCkGk5bBF1tGOSgaM72n6mka6?T}stNRJ
zxpLVE-6wv79tV0C&6O_E66+tWAZOg_$+wifq9#iFsG#y|&NI}jqB)F3|EsXNkkm<7
zF?_h1!+v6|TQg`nVssqFlu081Se5GNC}l3kV=D5ZUBEGyzA_&?Nm26g3s6EXa95!Q
z^W5Yj;zXB@M*m(lv;y=;xPFVZr1
zvS~cBuQH~S<2_JJ_MCjJW&j%!20AQhkdR`$(pSgzGNpnIT0w6CMc5s$d(S9$>3Nt?
zh_&Ce4#}Y(pGP~Vv9k8Jl8Bc5*)i5^8QQ!|Gv8%knq5Ci@&-ON-lL3{tc%GlZX8jp
ze50i$j|b-K#p@$1HZTKi4@$a7_m`_A7=TitT#lP(G!NUhGHd{z;<~dS2heypV{OROzo83+J={R0ErqI(PMh!;Cxjzxo0Z9HeHQ~;_Q`4LE_iuqQ!14EvtV!)UEO1FT
z);UG+ZPRlrh2its0Q_QQli2q{3?6&Y3V9)Z|90)@u~)>O9FcpZC5*l;)XAb@{132O
zv#m4BE9*OA557Q+P&TOOlA*rCzxd4>%6%;vhLY08)Bf|I>pHj$4`Wm@;E@f!WaDN#
zrAmY$8oAS8MvUflow~0B>Vrw3w-cp8C{*Y!ooClc4X27xU%#mWwZboV4Y-8rOlr#o
zBJj*z0v=C>kLczJ;ThT6C&C2pE$YXY$A|5&F*N$|j09fUlwV>BXFb^C(<1_GSo~=S
zL7RxOT8#h0CM&KosQTZqV4@$*3E_k92uWl?BT+ssEL;^k@wYF#wVybt1km
zYQwvX<3mkP#&!r@9MZ^`b)>Pc^|*vOJ5L0N3X_(+U4sDNdHfW!MB~V>&zwncJLx7n
zLvil%dVD~#eG~y&fHsrY#y!1yeJZiAq96c!xLJKH3YP-S&%dQ
z^_1>Y7CYthq9yq>ZiYg10(JMvb$Vp~G&b8q11ohl=-oMU{3A9}>ra
zR}Uquq6;d^d82c7{CapqI%Y}gNPRc3ULVueU4VXFcOu|KG6}rkM?KLJc*NBs2a69NDd}ICRU-h{Zz{=r}RjoHN+fkU7JRhj^DWpwyNsds2k}p5{u}Ym&82*Hc
z7(-z;!rc&*ncCb5&TV(y20TnVPfiSr27*#cw0E7cv;3wTE5u+-<-&=o|Kj)j;)qC9
z@wwh`!O7ma+-_ysbz{}8%Y?q*xS;DrM4+gs>(qdbOl6=ce)Ggz(Vk67!nsgGV9a!
zkW9V^Z1s^J<2&q}-iz-Q9aas|X;1HIf`()w=^tDCCDVs@8q4DUNL<7bFctp`jM^>N
z_1q#iWkoO^j2i-EReqCNwya}6I{QPfG&iSyZ^N8PD4ZhoOnPg|8u$Xje=6hLp1)LE
z@L?XM*YwP>^wSn9XC;{YHOz7;ym2!eKfm`5B`hLdNn6w9jOJitbYhjrMGl(4*&W)|Nj3XlkpJGNTK24%ZYByPMgG)Wkjh
zjsOl;r0Ae26Pm61tj~@+XgS&pPxju9*g>U`+6(@8}>RhikHpiN>lX=YLJhwexZO__sCg-l_&dIJ-Q~rRf&Wch%`uqlN8#C%lG!-mo^er{1
zgv14utZ^8C4_v*MKY7>apk2-X&w`>8zQZS&RP6foab)Vr-e`nNuW|!wL16T42(E=Q
zGtPMl^MSPA(c}v_wkFrI`rWwCWue%<@kI$rNQh({5>@t89kus&ZgV#mWrdQ0?w*ed
z!TE?@*0GNuMOf~>=~fa)d)7N+A|&0Ql_zR0tbWYSJ3Y)?71OX=M+|cRx;{x>eM(4l
zP#oZXe#Tv~T(xGT**eww-dn|X>TfM7hXDz7}
z(y;I_F`EoKG+d8ZK&^ItvV2Kt28={yc-4>CzW~z|!(wgWp>WMv%cx|0#zcU^;)O?z
zx)T5t>(M(2Z?A1f=X!|wPj(uA*l7zdm#zELi;uW?g*7sEjg-Y8M}t&2+&kFA(W+YZn9TEF>hT`l*omK*#hO{$-Zeq^LF&cm1n`vD(Wn8I(oH<5R
ztDHYsJIZlK`Dt(L;ira!>lfbLAz~9`J$uRM7hm14ZVp&cPWqfQD|*1OlCf&Mjq0M*
zipwsVEvKQ8v=zG(T2oB?T_^8S#(HyETMe3b%pNRU
z^!j2Loh9$jdZ}=^jX=Le+QL=p!(}g>DlJxQHD@6w@9F#OfUM2cC6olhuaaGRrqGgp
z%#)od{QFQsnUA6_@vB^&M2NBm0Um|TxpnA40;*N5JRo=%z9xHM?<(%u6$(_?UKJ_3
zjl-)BC8N4}gZ5lY0@%X-fq3k0sVy#beD$aQE$vNNS=LQLN5Wf;H@yJ_Z$2qZk!Hr#~%PQ^mrn$ZLV3V$WU@preftX3BfaCT~NU{;lN
zEPBDLroRU;sgqQ98<5~kC27_8`5w016IsW=?H;xLZ(czd%2PMOT4=#^9@@IG)$~XQ
zxeShyCu{1Qc@n-(w5Gm5zr;*j*1yPOe&+AfY%JtI<`o(Ki3e}#4CFKHzu_LiX0x1%
zt4_fXPTGUd@^;T)dwVWhVAT|NQZp4nEiPQCj5D<1J{{rf8Hng9hOd3-bi2S(cu4|#
zb^9|E?rJ-}R|V75fDarK+oQl&h!f{%Sx$o+#rfVF;DKnzyrOU&?yc{o9-_=mDbtfN
zXv#ULj;~JNYSY}}0BSsZgK?OeGUh!*33n;#^`@@I{UQY(tDT;bG=S?1t(VT{_6dc0
z9|VU8?&IL*%){`e?I>U$O~V)oxvCqt?B*67`kb3pl8O&lQ$U{yN#iT%vY2Dq-Y
z7ct3}1gIY3+un1e{iOXV2oOIL{QjgErf9dV;zFFBX5r9%$a+q
zEM2xol%WRxy2LuSWlQyVn*_+e^thi|6@wq3rJA9DoNy~1*O9}HxR#<%t!nI8_2U`d
zO4|)9&`y25?sKs>m1U_UJ?;TG<%o2`%yh%dsfb3{JS)~$rff4e8Ccf7j
z%}FflU;WG`lV@igRK=?MvdaHwrX=6JjMe;bGe>M@IaN(z_27X~0|nvEiOFY8QF|0_NV^P%XDykd1G^aqsUqL!I^ssKVyGe7vS(C@zz^L
z6h_R@v2c$~`yEMlsPKJQ`tcE>PSWU*l>T5R%=1R8^;&SyG7hI;e+^ApAK41;79Va&
zXvWPjhv4PaohT88m%n5KA^R_tJ!+)zRht
z*KeHPQ6txyh&oF%kE1^@tf_>x(6(nU$h*pCd}}%{dJ#e^x64yo&1=yAsmXl|c)#H=
z$Z@R?ZMR9=<(Akwi2I?J-HZJ1VUilm2~~*-aeeC;V6Kn(!B~uo*Pm`AQ1Ni32-48Z
zt+>=rRZ!c6GE1Hh5R`odJa7PmN31C|iSgA><};NI$qOM(#AzV>D6S*kbF&u%plKP=
zPQzBKu(#T~Lzq4Zhi~l@iDPFW1t|!t#f3$#itF7rJemQR
zyvRI+p6xbVQC9^gKH&3@X+?SWU<9o8!f)cLZlvsb)`9%YDr=2iwY8Q%CL^?A@`Nkw
z#4%GfR1qs`uv|3a*FlnSs}Hz$m>n&;gi2{6OeWxM;;!p&1&FSCGAiggE>g!83z*8z|&lwwfONl=4V`L-(Is
zR+p2a$2wTL*-1*`AA;k%Rj9^m$(wgO@ij%XpS
z6m>>nhNUd5uB;s2fx}Rp+A(NRyQlJrP3#5o`r?D&R9JJPOoh^*KSu`eloKiLHm2f>{a72uC%Ml4#z?`4k~?8svm~&zXif>MMc}J`%M__!b_)
zVICM;4MDSZRIQ%%wRAIuExV+m#JMDS*4Pu;p$x}pO2&!pdTCz(PH_W@
z>PeT6!Rm76HVY?=kE-RCJ=IGPZsx;M`K^JZyMBxq#LrzRBIKaJE<7zV
z;)3Cbzd^KW)+W6LhdypmHcz
z@svyS_43-+9B)wTs$+Y#M;6GQHnE?>wBBXIkjNm@XrtOip_Te$5)&bKm(-3
zdT5Ujr%~=Zfo(B1A%_qFZJl|2xi+5jK1bQ?5t%t+Xj7@B8ac~OFs)c!Fr)f#hq=f0
zdhv6@8aq3+RvC2t_T1Fh(d}`MC-k
z8Rtz2%^FYw++wi3D=66#))SXyAv$U*up304I#i-Y
z<8188_j*~!EO^c{#zI;wnqGJQxMypCUPC=DJTp+A>W?0@C?08!N?D`;KWTo)doP}5
z8Rxm0=P}NNDxeC}gzSu;$pUDaQ@-&AXgM^iJEA?4kUNTMOEV2qvy7$6d$bFuQ6fEM
zGKb@Pb@|}HONhAXp?18V(2z#IV{u=r3B
za%wvPhJBUKJDkQT%Vd4bZ`$fW%Wohq3J-R(#m=
z_;69IpFaMgGETR0UH}O?8#J@yL8AiQ>dqv5&PJ11T2+BhSS=-LpO)&~@8xP>rdH*|
zr*Wr2TzW3WE|0W53tFv~iOE)N(P8nltTd|ELOw-KuA7O6VUYqg0o7KiH6HZv0f5$kYI>2MZ8h&Z@U;I0`j%ELw+NM^mM
z(;p*5sPFJ5Z~wWpQ8T!_AkK`{m>12lIYq2h<6?(|eey85&4{~KMWDtM;1~?EPjAW2
znjyrNHT#_%v%t{=N7W5SgYx&ByOaoja3EEguWGs(2yaXfbHfTb@q3dIAyLs&7@UQ$
z`2R~1fP^1-EQjGMD@YvpQqK>;HOGDKof-gdhwk8p^u?>&G{70xuuTpWBk%Fn?r76m
z`OpvYK-*16fc!>jf9fR^`oLZm_{o3@C-eoB$DX0~z}wR+LN!q7aRO?Q5;Hso4cajj
zZKg?cmPt<1$;F}D>HkS}wZK%p3*+Z#eFPVHU$0jSh&e?MAd905zQvXi7T#YAx_LjH
zOWxbc_L&l1Q?K(;JgiA%;?<^2F5hYNJVA}_6Cksh&y|jHxDO33QO`Y6hD3w;19(H>
zeUBqq$(o6UP=HBm(?#QS6KjN7dvJ5jj?qJ&iGHnZT7nHnzJvPGGs$8h#s|+g+)eMW
z{Xe}C;4WS44~_X-G#}AEV{jo$0@5t$zxFaK#0r2CxGIpz6c?CY76tbSqN6h5$gPEd
zyQtQ;;-xtb|1un$BH)A|?EP~T@^HmaG-CKBM3R&buj-xT9U5lU8|5(9Ubx^Kz2N03
z{!+OrBb?}sz!yhx_B=u`>LU6tPA2#OE9flp7``g$J(h2&d9ji7Jhsu5%ndq4-q8eG
zPj)yV>^v7nYBPC=9V^J1O=&v#aug__k}Orf;OtxB44V>R+23{Lh9z+a#!+HqCArIm
z?QVj~CTNalA_j#3~3$$r}b
zsb?UlZjpsG@YJ>3s4l@!t1t{x3h=Gf3z@fUU~|&(Ax#95Bd@JejtqF$k*db8oE8z)
zh$JSZQ%O5fgvE0Y9PD3>El%DbIhYbwdMU4gGs8Q<7rMb}2YsZ17_UcMfx4FD-&GLc=d{3geoAONw
zyNVnGO#rvCs<|XNW)P@?;1hAh+6!WI)R~P@dx&{7G|F9(WIiaVc$LJPclxFgbDS+U
zE+QhXmr4d54Y;Br1Mu|OuyEcsJg-2to$E=t%e&lq4o-o(f5+#qJ3RkTi?!#FPXzy*
z@|P(6Rg4t6C|_lXg}OfOE7^v&op7gVpY~$%@$*7Q5KO-ZzR6r&1YG+OGmy7_E|vW=
zil?xw-Gro3CHC-yDSIU#Q{+UGPJIcC^lkZXEt)kXqd8ixvjs$cH$8
zPrbZjA1I|wB`|>@5yp5bb_i&dMB%L=_&cQ=fIa!bJCS<{vi#j+c9W(*`Cvp+X8d8=
zf(S0*o|@r#{RQ~C8o^Z-dUxaU
z_s6&sa?~dbXkXHK>ARxpXG^>N$yc0r=rYdf4YL?I!R$c`MSm~`G?9%ANS(*>!h6n_
zQ+dEqTw5J2{*TojIW7L5e5A7Ka<=C%yIS{^AHwAIySnO0nWh5c!3W1vu?z4;r$=lhcT()YCb-u5NZ(nf`l(fT
z0+hGj&Uj4_&U|Dly;Nz0LZ2kIIDte`{@Cu!#m>OUp@IttU7TpZb&RLjRelASAPtg?
zu9WBGLPR8HB&ZcNuEfhMB}~cQc>G^)wk7a#O-3HUTkuAvRrd7@2Luyu4hPk4gbnJb
z{fD#y9ku8rR6oQf_~oAKS8UURr~vh=@eg)wlde5gV&lHU5px%7jr%ZNlHeG{)gUUfGpN#s9=Qt`)L=EbCpKq;
z%xakuv1tM-^rGTle``+$O|f32V&Ke~o4Zj&UO#H)pdd}?)FHs4ZpSiCTMwczpX)}9
zjT-#4WV)_-#>0QF1WLBZ%3HI|LINi$KqhQf#-)H*kZmVgWE7oU;XhxNJCDuA)cpJq
z>p%h7NwsII;)~*K_veXYalPW)LFsyH2_T6mk)iO`4Bwxhej&4#qhQ05fj(T3*db^c
z2trXod19QUe6>c5KDK$Bsl_|z`{1LNH=ppA4F=z1p2iX?D#JHv#?g)NeC(K@su{Et
z&)(xF@?Rg+kkV#xjgv)neFC}`!cym{pic)HMA&R33S6CZ8I(r%NHR{bu7xSQ$qmar
zIudOFEa4qRWy3p#W^CE@!J>2dOn_86QO`RajEF+f;C7z
z&n>o%4(JBY@WR#IAC=in3w$jfksKLqXhJ*#GvX)+28#rXm+i29Djfy8s^_CZZ0rn&
zM>Wk%b)K!_pe3O6!LuIl`(e?xu6)#59}}BI5=v%_YU-VnX>FkW6sj2KODeinTfOqo
z7%Y{hdM#V{j^r{f?rjZ@6CRN*!3F2I+=T%HdHydjrs*8tVjWZAJx1^y@8(cfkfe<;
zGiYFH&aNkUIkvBRe#KBnsz=%fu3D#g10np>VUlRy4J0V88|E9~
z!)w{*%pqGR9?HCw9$*#WeDuC-xlU=rq0{oJJ)JV2_u)yQuymB_)FnwL3>wfGpsH1?
zGz1{4BO$FYJ_%O%OCR4wf9iE_!NQ#NxRk+3L|U;!r5g|D$iFxc8p6>@(qgcBxsGI-=tFJXfuH5csecH}|FNi%P=-PWr($%@;Y7d0mJ$Ddqhp67QX3M<
zwKfr_5N!Dw+Nh~x77d5V#v4Fq<}1Sas<3nkdego9I7quSatoTV!ctXR+sR28kCp`{
zBJ?5P6y^kSp9}Qk?%Bskx_xt|3)kk~wxfL}0zOtzP{xi9}R
zkf`9oLPt+GDr6^)e236Sx;dM->3vIu5K;aw
z7CuzVct0Tl4{E!A4CZbf9J~L?j&xd?^xqE37Fgf^+97p&Av=}SFj2$(JE}tjb4dw{
z3Y~}Si<~|x!_mTM5t3>iFB4j|f&ut)&K#msnEROBj
z8yy=^}0kIi;SOKGHT+kz!UQFX?y^YW|F$h`bGn*?Jimg_%4DQQ083~!LmU$zn;
z=}=xu$eRw9swd9%$q&C{KH*&Iif{n{3daJZ9zdFkA13Z1F+7;NlKn?a
zyZ9QP^)aQLstqA<&S=BcxLI
z$URGRG2nL^kTzSP3T>P-BO&@R_g-G6d+t`oL0shL^414Srbp2%SL+PKCIW)RzOA<*nl
zS$6AG$!-qgu(>EFj|8HTkCKcfb?d+~#d_jpVX*UKb;LdaWad^wP~tRE4m{;y;j?@I
zb#&oN%1*3nt1heV>0uW%-_4AI3Ff+v^h
zC-Nh=T3DgdngE7etGXB!Gt~CoI_km;91`d&D%cSG2HO{AdT9e^hETx)UQmHr
zdw%}_PcvCLh4&Djum=bk(}xoN?dg-l6*PI~77z*1)bCeKf$wIOB1&Em(;~iCCL$JX
zY){q26ODIJo#JoJ(g+h$>aGqC%$@PRxzI>legJv4z6BK}BqZCIWYgbM4bRIiB{$qF
zi-5kN(BqaJ9FgSW8}{=0&pW24Q}(c0IkQ|PQaU9!ve!{}@nSf3T5f`dad|0tTYTjQ
zSjn^aAm6@t64WdA6yk47pX3y*|9eotu6M
zgVM4yZTr>(i>41+$ylex4)r0{p^%0JQ@6G(F+mYjzDu`|CplMIg?2kDC2-lZO?Pgm
zDN>&EVK)Z7U
ztEHSn8W_1u%RuP+&O%>xH;-|}eq6z=ScWdG4i?FduaGi9YqTd5><3L!DU_WJsu-oc
z9@RXpkcE>j*;o48b(X?%^-X4fgp7Pq1mwWyh(}JEc^O={Fv~|;V9$hM6_Htj-fvYl
z@X!owq1BVM@CbN465~OHUhy~7E&gw`hwc$q-tNLeXWG*=%fQ`?uSZdOPMwmr77-Wy
zjt(KSqctddkg*-Bq0;$&>vM8W;d
z%BBJE-FKZz;RAqqLrqb5yM}jZL^pm*#^gc=)Ewzk%e5b=XVQ|BGgqRAY@XVxCuro+
zEz;mYqH#FkXGk|rg9b>p5Frxi0O;3|KQeHke_d5f^QpmM&a!@FmEW@PT*cj0cgSQU
zIEPlav%jzAU8$u%`N4Ihu>ebGNeD2CPf!#
zNI{{kU?j&Q>k_G4{((gq@G9*2W|tS8;}Ff!3hg0b-8QY2Hw;m6RP0e|$YkD?JABP9
z3FwW%+8^YTp+|sv@8{x*r#Xr%xWWwD_*$cb$
z6!90`EU^TIF``f*e~^eWG0BiBLek0k)KXJ=FmP$Ady5S5zB(=Ds$_4IQ{qSzVzy6n
zaBFHS{h^pyFQrc+^Gxg*+w8%P{T!2Tdp?i@ZU!ii+C?e7h;%9xzG=6NGu7q*ExOWr
z+wr^#BqSIQT-oxcwF)cy{dEw1i@bH$#9x}HZ(x4>%Sq1(9aH}zvFJ+n+95w4phjI3
z{WXb%fSCEJWTiLmO>sWG`3Y=2{C!&XeQT$Q^~o79P9&9>(>9$qL3}wJ&4g4CPc$>}
z8}y1+#21
z6p5N&!JMrs0qF)d7Q|;=VPJkvF#O%6gY-4`d#Bsej_0!StI^{`rC*%h-8lxRk(^6K5SHmyaJ$!DZ_9-U<`H_bBFj32MYWw>f_4TV+_zI5$8TD6
zh_u8D;)f$?(?mq)G##fR9Z3Z9k?{U*LjaN%~qyaR@NOlEo_7DTys*fajI
z?iz~F)FM#N;DSoIAuRcgW6Ji{&0^*@S|2We`sq&)Zj=@&0aNA1x9_*Q6uu9YTg<9UOA{>f3H6cbyRXyK>qh$>w`qS
z-}*5HFRsE2P~?<`-kGu8uwqUFrqx)%7umb=Au2dm7hguWKwhvz>Orq)Mer`H2?u`Q
z8c}a_z&(n?MqMLYEfH;Q<|0*WBc}wcT1wJ5aZLtye@_bp1>(mAuh!f$nPrM-oU_Y1
zL~GE!btS^RIEeg&y4Wk-xZ#q~{K?g0n$~C=U-^!ZWS!s7i%dZO#sqedBtwih-nwcA
zJyLLAqr^1n#fQ{QXd>ued|h_7272b}#1UZmZ{l;0
z=ZIURg2>o+Lk4=+5PP}b>V~1)|MdYIdaK*6nMO`vmO+X5F_6(E0Q^d-}06MW0MNHgZif0Tnc%(sJJ`Aik|
z=Uj(apbRS2w|V{3$O^5&8Y5_yb4uH
z3E5Zr5uKfxM6)OEZoai0A)v^f2|ST~9*7UTIv=5q43VOPdItyn4A4cGpTA}r4>a{=
zV6QbdJ(=$Q4Rif&zFTdwATOfTq9=Pt`@XYL3`i_y9Zji9A99mmdZkFb43IvvNEE)|5#Vl
z?U0J&f!^9&RVhd}j!Bq#ihF16@p+W4qkY
z7}MgCPGevsG61_~-gQ;dMq%Kse3L23LCb$6H__!p`IiUYk;ADuY7dvIjc-@Z^Q&%zP9DIOd6ZA9=?lpIB1{J}oQ{
zgy&Ws&i90A6^xh{>7+i$BFu((HDW$>wQw&2M>Jjb@$T9kpO-zlW
znhvG=1y@6(sMa;qMZ3{#1uZ!mj*&R0Zx_ocgc5w0)Rfa-H>1~VS!_!d8e-VUQ#nBZ
zR-5fHA!=zqRD)xXem_ZsRaV$by`7c9miF4i>40RA+v<2l&s!p?F>3vwC?t@jgBJB^q-8D*(IUd=I^}xCOa@qDCG;@S
zv@n&Qeynz-4fJz9r_>k^>meI$*x292SvVJrWiM|V?1$m9dv~~F@RR|}v{~1~2W%gt
z5g(gttw&dlCKQ1^ioUHFJCnz2q3`-%c>EvI#27+kUKK^IAFN1dR`Y|vd|~ulCTnsV
zq6IzB2xsO71jba|QHtonW+<#bsg!U$l6yiVzF8VNmDy`6<=$I8TAKWa%+WKXXcZPN
zJIHt>U2Tw}Vh@0JS%D(K2=xrz!M#bxbuuq26K
zM@Kct_nqq=``tP*Dd%lHIU1s5-T8SJp39ATVp)HtDSGQ{GHgT5Nq5+Ci$#w;(k2m0
z5-YA9i6f_ggW|HH7T@dE(dcyYN_!4rZzatE8p->$F3&7MFhe$!lXb!Po={+f@U`F*
zK+6NR1UuONf4ihdC2fzIzw511UMBLk4k>=jbzph?oXD|WhHSzcRiTre7r@3uhxt0i
zVFy_weUl|O{o_@*r+B=NJ+iAS2Y(;cQX%*3)rbbTtx_kO0um|JZPTLz&^9|WNo-)z
zm7?kz!2sq=w$kE8_MSv%j;g)f-~Z@AWPo^fLoBd;AxBgsYa=Q#6WH?BPSO|pmW2ph
zVV$Em!H%I_2W!DA*=}4^9B|Z#7(0x0DGwTR=SZT#XO(#aE1Z>c{NcL011l5u=W72s
zp5aDrXRPC1`XaW#w(puj)12c*=Rw_k-jSlbRtZNW_z{YTf2zCOW7dnlgs+P%n~4ZF
zSiNKYk;g(HaIB*rQhzrzWYayJg?Ee3Y(*;|sQ5Mb{c|rB8Q09R4l@5@*eP5&hN1$O
zDrQ?HQp4vm50MsD6O0Fnu3nm4s0W27%(m&k@NF-27M_Cpa_%UQe@_Z8-hnCAVMRZ-
z?S-rr+kENX83h_$c%e()^{kUYWZus#$Sm)!A0@ESfqF)_{@QcRO{W?$!5S^Bp@3bIj4r$p5x1VZDch=EApik)
z^o5^0`@9Xj&E1ajbbtCKL}6zvh66Ob;!194Yrt0
zi%tQ6^1%DrM3jSAPYt!v?kwR?oxMP~I?1pk#n6f7acc2Ca7JhOH~syyg7&2~Y`h+8
z)(*W}NnVemr&IFhzg!anyeCSMG%jaP=d_~r8i@&DVOy+mdJqr>6Vp=j2*XYYY6dv5
zAJp7T?vp_FVKxEMzx|rWOq-+37`MPlAe;RlAs4z%Y>H_~B>X4f@wmPa>WG&Sm{7G@
z;;I?kt2$_7n0qRm@Ft=O#AYm$07MZ>lRn;k?ouA^=-jjPPi4X;FP9DGat0%(zuUN1
z5kYa&m%nb+Z`B5A>FMFIBKMUN%apS>C{hdFc`j&{3U+~;hM|`sRp|@g5c$M*nzf1S
z^e2`pgz$M2mOKYE&s+5;13;+quYb)eDAD<_?Jl950|{P5*rydPL%`NXIGc$YEawlr
z_OmDuoqJ>Hr6zP=aVk
z7Z)AEguABYFfh_TbU(N*7vt}f(cpJiPRz-#v+=&e)uSt<7=Jn=bv3#TOxONx?`j_jDCipA8_x0}7a=$#9<6Kebp
zCwEz|RlE@vxHlnler?37Nu`|FD|q(R0?N~^DRk@d|F51SARpLcIuOJ9$e6!3#QU#O
z*jg7&wYHora&Ipa*qI~wwCh(qGFVna)$=PL{G2?*NKH`mXj~_AziJWyPQr5a(>H?6
z&a_axckQFj?C!pGH0A{rAj&h@X6gjSBe}MP@9@Fe1{C(Mmrj~pm~sD@R%(p9q2EXN
z609;V#j#GhVQ_ZC8GF0?Vi}O4nR;|^^zBPPd?KnVAy*Od_w?%HK`(9uQO9#ud}j|u
zuylArg-^;EzXjPD0xf$jwjhE;--4@gq?Vfk%F{{zBBYk4!M_aK!O2K{C5A^|hN6<5
zWt+{&xDti6R7`hftN7H+gXuQY&vqzYPkkX;*JxWb4QK}h%29KH
z3=ZE~*j{j$%v0B1k@NGaDyS)1%*9{pX(&`T;WL$hRmE+JybSg@kK153h#~kzdemti
z95~V{6p^>L2OoQ|Qlg(Ed6gtoCzl42sp%I%Q6;NmJ@=}|-I(6^HX7s(Lg-ojXF`uh
zj21Wd-4-6aCC-LCFm03>m==(!t;|U_
zrBR2e3gFI6fyyU#D|5JL{mYxF-r1A^5q1L`YrDt8K+$|-(m*KxO#&BqOD<7?yYdnvcHR97N28Z%9gtOp*tj>qZr$o>3_8SN
z?rm{KqFJ(eq;;g;On$ZC#Q|L*+KDa(e83<(nLS2;Glj@@;!L&_9&{JMEbU~lx|Z_1
z7iCE`*})S1U(PLbIiRq-4=kWcvX&Tyc^EmMh#)B}=NvX*QtC_~7
z4~;gPi4|D5)!V`TNoU~WE8nAom?$v=r-$7kIW*UDVk9~THpn$D#^#0n-{?mX$Fp)3
zcE~1)BY31}#rJcF2;uPF<~>YDrNTKlkUqj`2&1cH=y#u|I)h86lRIKBt7>Lxwbx-`
z7F(!=hJzj%$xKKOS88-EMyUhTq+v0y9J!FRUP+lFH9Omk6pI+v
zcF!-X1pDGkn%FN)p<%B;)ZT)Cs^4~wb8=8uOo4y}WNO+E?+?K24}~WmSPg7T+a|Y|
zweDH=BKG>EmOxFlSsvTwX@mvuN7oZcuHH^c(}AuYkrYE_gd~J4Au_a6xVdIkOr+hp
ziRb~tyVCXMX+qe5P
zmz6J-&-h5Z8s@^&PUe;#Eey)_?uNK3^~el$n$NU2B~w`AjnETs3Q_)H+(dy&w-Gc4
z?4=%EB`b49-U#V!&zD?CAj&;{0BR~L$;KY1O}TGp?#KCNc_q#_5c4v#75bjt!0{|D
zl;Ixb<@9m@vzHcA%raA$L-6ql*wkClRLZQq0^H!r{+N=t_Qpy#O>?*rC|-nbMemkK7|Z8dV}un_j=0ErM3)@d3!s_DO+!lemBl
z+dKnptlF)kYqPr~bB<+fGpQLZZszQxd8KW>db~}ppCT$6C;H$mQ7;e}f&5{tu%}oS
zUMnfb=XE4c5VvsoXiM)Rb}g+AFKlc9lKoHJ*x?-yhdzCOsBpKeODoG9FW|rY$a)(a
zXR($Mh2(7J3c*V{(w89A09tJsp=ZZ<-z?gL3~nuoJKYI)w<@Ca{KFFWaJ%26%zF2T
ztT6`M$~$|In>%j&9AB5ZIR0Kel0UA3l@j4Z7;E+!a2eHw4_|0btg)v%B1{PAJ=xl`!P=%DihGL)sloC>8wC#a8-JPpMUV5zbCZyA&H)kM*JQoh
zHN~&aYQGv%x_FoF#6IHr5SW#e1|$228Sh#?syBf$uAX9uYu#Igh{#429r8^W6Cp@3
z0>iF%O?XWxb|*u12Jjc@^+O;k-#i+)zuxHc(6!BOGrG>$cWe%@r;xR>N#xhd^P5)*
zsQNVy5Rv!b4&DqNJ`bBp6+3;_h#E#a{H^VP4qfIC&uIEIG^4~5F)(9oY>tc{ov;YH
zN{9F5SFG==uN7eXARQmTiiMwI{KPRgo$%*$XT2eL%>ca-v)Ns2hG}=cQPXj)*>(Yq
zv!;TLf|G5>-D%egXWa%l*iu%=u>I7EG7@BMCRVzZiY&00}GM&9Yyk!WM5}o-7a~m+Grwxz_3^
zn=)+)3o?1QV2k!>#fooKF@D{a8ObM<@%h%$;o?K5Nc=u$;I*Gt8-aqYifzYclE8`x
z2G&*MA$tzFg&bk!^xST<#{06S5WNK6Gd^5Y;gmPVW6;Bzs?sB45oQ0-vh<+C-z&r`
zH2uvLvAMtiCgM8aAn=C?8sU>r?1cR+NGkM|*ICLDUKxfv62XXDP12sK+VJv5KFMO1
z<)=QFn}*sCR9pE1;R?Cqa8~xd08@lkWm#{mHozK`>U?G78shZu=!q#%m;A0Czvp+`
zDmAT6vae=9c}T^h#BJ(GRX3&x7SMEen@&z*q^cj;mPp+Tu6T8}q$GJqIP=FIc359r
zB;`{QQ_s3gQ7y;B!{z|Cj57k)rw~`hxc2R<(kM~xzCB}%ubk+%hVH=eW
zv#G}%S|H&EG)Wc!&P^anA3Hy7-u!qxE;nP{R5ia&9}qKuK81#`E@s`?_BN2wlbLt-
z7#$V9t>^nJqfl>awO=)WmfNBcFcEn{nsG8R5-`dJ{AvZd*29joTl`TJ^%@L0Q>)?`
zmSRf$BE%H*!wDAI4MG&By^eA%hfNVh(S&M-BXZ3
z5PY@?cw{Hvh(IDcBh;q~oy{#eYAz$p|8Jsaqcx&9+$XK0b!cO%wZfJncS>*N=L+jm
z^4P#;tb<+IfRM|%VAwB}f$b2S_-_>CY}=o*=kI*N1u*sCY4`+R8Mdn=Aa4liG{yV3
z;R@E31CReulqWHX8}K>}|@p5bgDwpG3`Qw*uluSjEoPf-k1c9gD2cT&#C`FuuE_isgW0lQD(c$!9{*d9F2q&`sF#uX7cwCT)Q&MZxn^m?QK!e=?2aH
za1)(|>bKcUjUJ;&3_K+0%sT9@pHMQ;$P}KS#wfl0tfO(uif~PTJfR#v&7}Y3Bjnr0^A#m{R*E*sn~9
zVj0N1a8g)l`18!mo6ihyw-}^q5Cg6DmG&SMNWiAf@liG)#|I9NjZ)o~i-sb=>0XsCt)vUs~HKwebYq{mD)V-w9mn
z^Kc-3vxT!f87$1#Lp)hK?DdL8Y$!^kzOY}MwW;hm`7OVa3KPG}w$X;v9!^jF8Jk+Z
z;rbtN`qjfXzShcfag?>Xv8zr^zpI&^K{>5>%2p)l)Lkj?5bvV`aTxwUu78YWd8!J&
zQH1Z`sz_KZp;#d7n#6%VXL1BRZu>t+4+hdD2v~D|vM(ug5L)W~OL87pw8p?Kn|VIaLatSyobDwtLy<`3DgG(0iF!#)ry+JD
z`Wa%^XoBeO9slh^%J4#~9SMIjaAf6eU2~X?P6$UKaq{!Aihr#+r~|#ig?PTCZ>n6&DTfT7>3Pz0*~NoemNB
z@~9_f`5_iKFchcsShl7Ig$l%umNBnmpYy$r=KuaK+oS^itCDamu?AQy%wCL4*o^48
zm_BFzkr>FR%oD*eB)dXXzI=l`v9i%z$fU#N6l#3G6EET|6}k>EdM@4CJ5Rt-q#>b-
zz=@jU0AnRJu(B`UvBP)gxbzSjE`Mc89!MzJOe5dzzl8s(!Ms
zPSUiz8OL||COM%YjmPZH3R<^gR%FocTzH5Jz9%8!)Q6x#@_UENDU<~VMl&)|Z_xnCB6F*3`v6qLkG^TGfGrn}jsvHjITw3gH
zTWg;)*Q`^lx6%x*AWC5B+*U$~*dmZ{VY9@A}+H&sVot;DkIAI4C
zeLr)ydG@YI1s>-=GGI?=t6m;q@`{D(BSi9uz?-Kt0+j^`ddzQq6b6QBFKprHwf}0@JQ7Ve?>Y}7Y5&#g#b~aY
zd)m)&j=ygP3{7VpO6v{%{E-Be#bd{S#E3j3&^{n9zN;X0e26W3Eqr~RHsaeU`)kov
ztjt8*1`_)2olo=RT2G&M^6m0)>X#nL(JfIu>nNl{3Jwbo8FdP{Fnh|bnmg4yk~M0h
z839)8in1-olQB=0jTPD!7S@`T?D*-5ZR}*Xh(~>4&C-R)m5cG7t9N0{9q0h-*XBiK
zoFeN12OEGyOTib(D>Zzf*&i{DpwNZUs8Yai-M*V`F58>jXFTctLfLt!uOpgWi4d5yprf*OXNe_tIOY0GsK$8vMP*RscpK
zl9pRhVDZ8$B4ZxIqfNS$0@2|m7eeIbgh56E%=z0B5szoSlWCYD`8MrrPnV}Ft;5rk
z%vc{q4MCknd;KP(Rc1Fp^^YfbP``9h+Nyu_5M__GmL-#enNXTiqpTts=8030gl_ttmwJzoA93QkT!e1y6ks53FaqkQph}z6|yeB4ytrT*uDQ#@54sk@p|-`1I&z
zHfOU;=cmy1Hm7$ESZ1+Oqr~)pKbIT;V4-eoQnXk$bhS~npbDwyf+-5(gjvmImOgq~
zc^vZvk;7ILk4gNr3KSDfPwJH)BmsnxasET#vER}1+@K{Zset%$-7`#J0^7hV4iyb^
zq>w%#&A(k&1Hn3VWqI(M!s%bg1+XI7*l5HSu@KZyLfc84#R()izx&gXPVUJ(pT9&~
z6oJ%X9boN#99Wr)mxYfu9dUqv+x(T(+~StO2@$SeQxdu+%lb&66>JYGu!~z=8MT$x
zZ{W)a_jVctc_0!0$pXIR39`qB@q5=D|I|S5W$dH7AG&elM#Z7kw&xZZX_fB+
zGrXDC<}&g|tyChJ$h*Kw>Ph0vXadZcO{Qi`IC*_DmJO(z%X;kw9f*L#AfOfL
zE`3}xpFP8~EKwtVX|@#ooE2u;!>MP#w^Zy@$C3VAxs;S-TB*TPGCvxJ=IvMz2Eijlb6Q^lvHOgXNRw^PqWXuyxQ+}H
z8oOCY4N57o*7-m@#N>1d?1wB-Gs2}LiQE+IpOc(8iW(a1AelGU=GH9jrv=?@I4i6OR2pFs`dwAZ8=~o~#rrEJ*sTT&6)}4EXPI
z?xx6dJC6BXKY(q<@F5+-_(=8yL*XKPdUmry0qT`=9J#p_ES*?YKBVvv
zBO)-2L4vf_wfOJnA@=1e1}j6|a~_*+7-AVsXYu~n-rQK@;u^IJ2So@ry#*7F^dNnd
zO0Awi!5BUA@uLU9lgc2S@21Ixh`AxUTe7Qm|RK%@?TchH9NZN+zoz5wh%^wJ
zIT_WJ3PejD86)wgMrnL1^kbqs4<2DYCGLQ8``|P#o3oRg4Ge%HeCuz~Vvb0ck-TL=~C+ox|Wn>H*
zp;VmmjC<`*a1f{KM*g1Z=t=U=fu~Pk@p`T9Z;5%cgyR)+VJ+d%x-;1sxE8FbosTFt
z7xZ;H-6La}RER?oh>fE&KWNlosPMw4<)c8)hs-H0u4fa3ky|nf#OjHT6mNSwHIT-<
zp(ab13RW0G??Uk2_?L>XB*p-LGrFB*M}`fj`pZ8H?9l!JQ6n!^3>5UL&ypoGnKhHi
zB{#_dW=W-GWCIONERdmDp>i8!-#Zv?{IGdY@4C&S;zCt3=t|}9KosuE
zKj-WZNLgdT*?x_ZUb8lGTp^Z^woBcL^w<|)TQ~bP!0rpM=P&q4(fQmRO2+pJ`QJ0K
z{5~eNoSf)j%mELMl~3&{yQ%HlT3h~fkYr8%+H{?Vg$d}0s=CoSQG^?7j~G7R&t>eN
zg5F8>_H+8sVtl5?f3C-e0L?yb>^~+k+ae(6Dut>0-+>3n%ofm;)-cP%$3DEIpbLb#F4ak_(RvR}~QV#%DTjgH)p;?a~(0MT$4-;uET6
zrJQ#%@jx-D7rd^&mM?vqk17!-
zCA^{FOVx2M%8nj4m^a6|g^9S$1LL5B>=TZODUHfCAS|IO+u>sbje(BaOEJ%ORxjFD
z3>>4Yi1UEvokIjy5Xzm|{V8)8oqbjolZ_crlo#pvGcIu@M_5W3UcbM=0dDN^rUc61
zmJAi0{`m|HvCk5EoGi2j^6p-+U$WG8AM?luz^b8w+mpfkrH`sn!N5}3O&zJ!M#42#
z>(_(?2?pi+69AiHQ*8*EGopLUo<37w^=<%_jp54j6K3Kxq2*~DU1Cez26?rdFILH7
zeH}L*gh9uNA`@hqM^9aLc`FbKj&(2_ZIOa=AAx|#3CuN-q
z8r_EkVY>t416^2HFYyZ2;|o!|b<_zEMt}`lOP{6ibCxH$u*mD;DD=l1w2U*r=!b1y
zn7cC4p3-P;h)v>_ONFaT7OnIA)F})%@|}a(i`uPabebsLV7(#-pu(|;3E8QrYt|&+
z_SC2^AC;N|Uu%XyhsAvpLy?GlL{|6zOF>-<3s9#qA;eMG)n?@xbC+#fg>+q{W+P5tzYECmN?TbW(`%JN2<}w(^SLp@0R|u=
z;g~?;FWH-)%JrWNo0L!^tFh}rLF%zF(9o=Si|$lVDLMcChu>Bmq=FTP3;r;7IF5*x
z;{%{yWIs3D13VdpNEv^JaW2K)671#v_!^-;P(HY5W~lNFG07l|b|Ak1fD?#`#VKOn$3C
z|EeuDv3NVDTF&wwE7ole3_sJXT&4yFJ~RJ9!KJ;g9meIB;Z)SgWX*Wb7n}Hf9ddOg
z$extpZQSyq$nE^~zTCJa_N03XngS13`J<;q3IF?t_1V({ozV^3AI>elU|P6
z*@E_3Sn$saW`(;Y?Iz!l@G>NYXB0C>;|P#`P76a$jzZJ!LnP&$n5!8$RkY|H&#n2I
zE=J3l4Ha+<9@q~RJ_oCJk2L@PNTK2LL34tH^B-*n(?^P~5uk9ATB-JRfX27YQK)ZM3V~r+5or#jXaQ-NBYS>4SEg2PUI=T!$MOwL2b&rKy;mb)eV{hvX>
z79j&klSY>;4K4XUtg_T4L*SuY6n^5ld1O>Kt*!$hxD2AA@F-obJ#DTB4T!P0Ld<}K
z1h%|*%JB0;)9DdyJrYZg-DOXTanuZR#zvtUMuQaPAD^JN{~Nzj9SPUlKL(8!zB^R@>J;eNcN>X1qzy`rxGtHyj45F1?kCrbqa-PaW6uUK
zL~lg&9y<&fWH7js1N>nh_PO~*tT|?Q1i)U+1t!C%giaH%&j<4KTep!f#l!kxSQJ)U
zFUPS1IkuN85P(Gma2YhB+q&wIC#fneNNcNF*YC|}HkR6kR3NAZg_eaxkCy!Wc_WfX4@g<_yC4B+@#Wxp2%w
z&DtZuKt*cF1X_J?X(F}p;A4=vQHc_P`DTJ%e;x59b%C*FS|`3LO#hf}*T%!up(OqW
zO#z)kuj4xgebOhZ=CZj6T+#!URcONkq@rfBX<8_(3n)DU_!Gg0kP|#
z17jn)_cyQAEC$j6Xp6h2k1*zrTmXFKiC$*%ODf@LQF{&QDgOv;Qr{{he%!v8B8qEE5h#|C=dQL+M=*o-Mx!x0l?Fhh~-LF@m}Mx9_%V9@h+X2FKGo!~@stBw!S
zwNZNJEZWddnx=9bsp@_Jri|H_gGt`2W2c$;-lys}nD`^wmhcOb8gJpY=aG%QcaQ#z
z`(t?WtqdaU9Wp@5{~Tmtll#Eph`46d?$d@qnGPak#kkJ1TLRYc5^OyQd#E3027-%9
zTG^oTyDBHBXqiO9UiAfZ0~R-hm+GMOQiu7iLX3HvYY4ZQQ{tPp^t&wk=;KwdPQ+>T
z8rQao3@1Gk$gFzs7K)D!3D^4pQ?RaK7j2y%5DY7YqAa7-u2cAPE_3E5%heQD*M3g~oQJlev0Wv}G(^|?M<&?(
zGb0>5_L8ua`hK)e8wcVv{v@9f2YA(V>L5sAh&&0}Vw_f<|M0;sDF*S~zz{FNIB!X+
zvD-XC_RxeqE>{Oj%JUnfL8>#zD8Dn%1>|0o%CE>LXWM@QOu0_6NHo(yjdEPoT%oIR
zZUAkD?FNFB!82zNuW2kOxGaYJNy#t#4Dq)K`V|{6-k^dwafhy)Eg4*sxHd(5I(VFVHWbquG8OCpYM+`i*K;rMS4DV-JyGn|N`@
zYwlx*or2g>U@6hLynth<$Jq6lnn~SfGzwG9!j6O%^+CTO;YeC{OfunBR*@;+qndMA
zkhG~N%vkGKXL8Z&p_OXFuRCpaUcgq+Wiw_xH)CgO5?;R4bS1ZT)2|A-~v(Wm;U$?tu$);&}hzVxf5Y}}WhJX(&^dnbE`HoB)SyFb}oTHQnf
zsX*NB5uXTarFly1r^$12Ckl3uiu24?x!|OWSc5?TL-TQ1Y7o{G?K!iQayPq)KoKI1V6({?0#|2JOTKVH
z$_Dpz(I-NSQIhol7`vlba16MV(Au`%{=Ggy&U|kBXZP^&tH<@RMW;^kGP^Lj_-W~2
z2snEMKpw4E&z=q~;07(F`*@GMm+!wY!j@m7RvL+x9xfU0@}&4VvFEw@CWvCP;NcZZ
zt(zHwrnDX)*6t=GqIss#SKg1Q_w^a_M-Q-M4qn{ch-hFcy9^MiacpeXN3dvih6M9*
zbN@ikn78HV)|j;FPHvrnj+J^-qXPY1$`|w2Qro3cMotaHJN@LPY#f~5C>w*jjrbtY
z*)BW?6TfUDV&~_>06sL8^V8t(JO=wPJ3j9j_}k1%fH^%C@%zXf^Y^ur6V$?PszMnx
zT>moXU%UYC*^piXXYU0m1E&d_Yo^X(hFR9Ye_{wMu4AR;;3R?K&kgY^EBqY5Y<*15
abi2>r9kH(~GStzezW65_j%ktujP3;2N(vVM

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/activity-list/assets/icon.png b/packages/local-private/packages/activity-list/assets/icon.png
new file mode 100644
index 0000000000000000000000000000000000000000..a56f492aa52bc7fc843421b00f1ec7d52ee13a60
GIT binary patch
literal 16008
zcmeI3d0Z3M7RQGr0THSwDp0L4B2Y>)AtWKh5Ksa_A<8PCQphquAe%{$K-Gd)1*Ix|
zs31iwN};%6-Ipp_zy-moV0pL!_PJYesfx?%n-GKxLZ^M7{yF)4lAPT8JLlZ_o^x;J
zpUDZCIm61r-U5OktH1z$82Zi9zs-s0XRcrJHS}wkDj-G!L1U-r-$sF94xJFhc9BLz
zYoi6h9I;YP6-tyMgsPLPP&EW`y>%*~I0ey?L`b4k;YEIQolZd&6ivE9E7VaGn#qPB
z@8a+gjaV&JX{Aa9Ngr1zQl@IX$Ygz@-dE3c$yL3H6q+7(C`DK&RKadk8a$YjMBMA6
zN>$6cmnIRzhzyY<3atk9a~sr8m88@vHA%`LQua;{nHa5!K+v0e(DlmYgQnJK{T84I
zdXNt4sfkEeA#fO?QKqWJh~EPAQYH^}^;&5{Uvc_H($5UJc0?x~!c9M8aO-XbLxs5L
zwQ_vbh)}CkM<|sto}tS^-XSFU`gV5}$yp#2OBH&qA9LY>PJI>ci-+)qT7-voB#T00
zQ0UAEHzvp3gF~ZJXy_*m2x3qUh#^o)qzUPR#b9%2?qCc+IWPvggCs(&@GnIHwLlz+
zn3JGX%Y|B=R4z_$p;eHM%T>=gx%(8wR|(dyNyQ&}fAUF%rn<
zp#xN@RKgK^u*8Tvn@MpOc`zsp2Fr~iWQar*CXLM&F%lRe4~bhhM1cTwe=js&DNfbz
zm!8m)zZn{Ee=qbbwG=(;gfc**U;W=rF-VyAQUypgXosc)os0Hw_h}Z6%Bt-FN8_E!>TQzhLjWR)-CR8KS649RcYrEYLx4+}S1YB7E4JK&!+70N+gxys@8~nYozRU)Fw7#mcRD+(uprJ3=P5;^B3(gPpjlu8*=Lh=s
zxkf6aS{}V8-GE`hOMm_vj)rjbRsrF$m`ryj7alk~&?7{uLuAo>DY}O>`p#rBdVB^>
z4C)v&lp`?F<7g-r@TcBgOT#F7%Yfdkz`eIC{mb0D8UA6T?~4B6z+gB$2}lFe#&KaH
z0tGlOkOroWuDae*{2Z5$USB2a+i0%>5{
zI4(>?pa91O(!jKFT$qSJ0gel#fobEoFcEZ1PX9m
zAPr0#$AyUq6yUf(8kjbY3lkA2z;S^zFl`(cCL&OP;{s`5+BhyuM4$l21=7H@aa@>)
zKmm>mq=9MUxG)ic0vs1e1JlNFVIl$rI4+O|rj6smL<9iPryz6(LAGzhvE4?&y)
z2pXsSI28)UuNsLYrSDtgsC0B0pR@x?r^K
z2YY+5y!x|v@#KHgok~lqO4l}L@BZ@q18D!ogT$?wqKm5^3rRfs?1u31_6T6xhqH2%lyN~k3}iJ*uuk@#5)8Wta<%~2C%7B-=c
z;SG}QW=%8CRFpdXx_5N5G}AfGjJ9OgZm%qBa)h=jMb*r`4iMP~DUd^)BCUJ|!3Blv8`VOL<+uQEZJ-|x$~>)qLzr%h&@eekkFr^#t2u|63&
z*K)z6=S?Xv19K1hD~A#K!0-gaq;=Uob;7w#ai+5=3K8>8)_W@<)`
z@#91}M5>;uYhT%ITP+`6I@!E-aa^eRzZy+U{^JtRDxA5aeF;(abWzd5%8_u>%Ew)|
z4j($w(NLXXwVpg)7cVT?M^7O?OQ@foOL5;_-qf{`)<)5^#(#V5#k%7~nt4;kiixwx
zVbvu8Zv&>3jraRzW$Z`%U1mGX@?O1pP?@P2WBy+GM-&>=i~0KM_YvTEFyi(U#%1gjeNP@}88fHc7NxfAhwD
zBCAjwb;x6^nRW54q6igt_)&`oCRB<|%SzRq)~a6!P;53e#Hn;oX-;mfnOWxj$gjKB
z=4b+bMEq;6_^=L^nDAH=OX5fzU+yup3_CZ4$nqu}S2^Fv)_hmgJ?J}|lXk&vx3e}h
zoT|3^nN41G+FoM)uzJz0{G7lii%D^W0v{*QdB5u4qJxY5-x!Z4TsOP^xZ!(FCo*TA
z&M`B^QN4NI!P_(ykv;Ls?acZqb;PXWStX)%F*d|a%?I*M=DEeWOvv%A%w97&yWa7z
z%a_JAyNIXqd=|!+nmrTwU0t_W5-jm~F?wP|;0LyR%S__AJ?ZxB{MFyb(8I43#Y{~W
znct~|A{j3~DzJF;>og_la*=MlGuhuNmY-W-TlWV~#`_K?TM-tlY>ycCImN-FY}8iA
zL`!<3@lOG3YGY{Z70(3ACe;bJW_w9>V_V~#JUVu+n^PwKPIcx7=}|w0(~eP{cPALRGVdP!X7;11^S1WG
z1MNp2FJi}bU6l`Wb-D8gQrmU5rpD&ZwuEzbTO@qnV_x6yYmbLgCp->pIJPg;6ZG=dQFb`pV+Hi`?MjM#7}Xw2zEhSYDkuzqVsY(V>5)7Kjbvd?l0N6-H9
zb9L>ZHya*r3E6vnQ?+F6*{K%~wLGifUH;%w4YKM+)I8Ju<}&@!+VV;XIWO>8`&;|k
RU-Zw42l~zA@0=dL {
+    const buttonRect = ReactDOM.findDOMNode(this).getBoundingClientRect();
+    Actions.openPopover(
+      ,
+      {originRect: buttonRect, direction: 'down'}
+    );
+  }
+
+  _onDataChanged = () => {
+    this.setState(this._getStateFromStores());
+  }
+
+  _getStateFromStores() {
+    return {
+      unreadCount: ActivityListStore.unreadCount(),
+    }
+  }
+
+  render() {
+    let unreadCountClass = "unread-count";
+    let iconClass = "activity-toolbar-icon";
+    if (this.state.unreadCount) {
+      unreadCountClass += " active";
+      iconClass += " unread";
+    }
+    return (
+      

+        

+          {this.state.unreadCount}
+        

+        
+      

+    );
+  }
+}
+
+export default ActivityListButton;
diff --git a/packages/local-private/packages/activity-list/lib/activity-list-empty-state.jsx b/packages/local-private/packages/activity-list/lib/activity-list-empty-state.jsx
new file mode 100644
index 000000000..55bf4afee
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/activity-list-empty-state.jsx
@@ -0,0 +1,21 @@
+import React from 'react';
+import {RetinaImg} from 'nylas-component-kit';
+
+const ActivityListEmptyState = function ActivityListEmptyState() {
+  return (
+    

+      
+      

+        Enable read receipts  or
+        link tracking  to
+        see notifications here.
+      

+    

+  );
+}
+
+export default ActivityListEmptyState;
diff --git a/packages/local-private/packages/activity-list/lib/activity-list-item-container.jsx b/packages/local-private/packages/activity-list/lib/activity-list-item-container.jsx
new file mode 100644
index 000000000..ede2c2761
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/activity-list-item-container.jsx
@@ -0,0 +1,141 @@
+import React from 'react';
+
+import {DisclosureTriangle,
+  Flexbox,
+  RetinaImg} from 'nylas-component-kit';
+import {DateUtils} from 'nylas-exports';
+import ActivityListStore from './activity-list-store';
+import {pluginFor} from './plugin-helpers';
+
+
+class ActivityListItemContainer extends React.Component {
+
+  static displayName = 'ActivityListItemContainer';
+
+  static propTypes = {
+    group: React.PropTypes.array,
+  };
+
+  constructor() {
+    super();
+    this.state = {
+      collapsed: true,
+    };
+  }
+
+  _onClick(threadId) {
+    ActivityListStore.focusThread(threadId);
+  }
+
+  _onCollapseToggled = (event) => {
+    event.stopPropagation();
+    this.setState({collapsed: !this.state.collapsed});
+  }
+
+  _getText() {
+    const text = {
+      recipient: "Someone",
+      title: "(No Subject)",
+      date: new Date(0),
+    };
+    const lastAction = this.props.group[0];
+    if (this.props.group.length === 1 && lastAction.recipient) {
+      text.recipient = lastAction.recipient.displayName();
+    } else if (this.props.group.length > 1 && lastAction.recipient) {
+      const people = [];
+      for (const action of this.props.group) {
+        if (!people.includes(action.recipient)) {
+          people.push(action.recipient);
+        }
+      }
+      if (people.length === 1) text.recipient = people[0].displayName();
+      else if (people.length === 2) text.recipient = `${people[0].displayName()} and 1 other`;
+      else text.recipient = `${people[0].displayName()} and ${people.length - 1} others`;
+    }
+    if (lastAction.title) text.title = lastAction.title;
+    text.date.setUTCSeconds(lastAction.timestamp);
+    return text;
+  }
+
+  renderActivityContainer() {
+    if (this.props.group.length === 1) return null;
+    const actions = [];
+    for (const action of this.props.group) {
+      const date = new Date(0);
+      date.setUTCSeconds(action.timestamp);
+      actions.push(
+        

+          
+            

+              {action.recipient ? action.recipient.displayName() : "Someone"}
+            

+            

+            

+              {DateUtils.shortTimeString(date)}
+            

+          
+        

+      );
+    }
+    return (
+      

+        {actions}
+      

+    );
+  }
+
+  render() {
+    const lastAction = this.props.group[0];
+    let className = "activity-list-item";
+    if (!ActivityListStore.hasBeenViewed(lastAction)) className += " unread";
+    const text = this._getText();
+    let disclosureTriangle = (
);
+    if (this.props.group.length > 1) {
+      disclosureTriangle = (
+        
+      );
+    }
+    return (
+      
 { this._onClick(lastAction.threadId) }}>
+        
+          
+            

+              
+            

+            {disclosureTriangle}
+            

+              {text.recipient} {pluginFor(lastAction.pluginId).predicate}:
+            

+            

+            

+              {DateUtils.shortTimeString(text.date)}
+            

+          
+          

+            {text.title}
+          

+        
+        {this.renderActivityContainer()}
+      

+    );
+  }
+
+}
+
+export default ActivityListItemContainer;
diff --git a/packages/local-private/packages/activity-list/lib/activity-list-store.jsx b/packages/local-private/packages/activity-list/lib/activity-list-store.jsx
new file mode 100644
index 000000000..96154c6a2
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/activity-list-store.jsx
@@ -0,0 +1,208 @@
+import NylasStore from 'nylas-store';
+import {
+  Actions,
+  Thread,
+  DatabaseStore,
+  NativeNotifications,
+  FocusedPerspectiveStore,
+} from 'nylas-exports';
+import ActivityListActions from './activity-list-actions';
+import ActivityDataSource from './activity-data-source';
+import {pluginFor} from './plugin-helpers';
+
+
+class ActivityListStore extends NylasStore {
+  activate() {
+    this._getActivity();
+    this.listenTo(ActivityListActions.resetSeen, this._onResetSeen);
+    this.listenTo(FocusedPerspectiveStore, this._updateActivity);
+  }
+
+  actions() {
+    return this._actions;
+  }
+
+  unreadCount() {
+    if (this._unreadCount < 1000) {
+      return this._unreadCount;
+    } else if (!this._unreadCount) {
+      return null;
+    }
+    return "999+";
+  }
+
+  hasBeenViewed(action) {
+    if (!NylasEnv.savedState.activityListViewed) return false;
+    return action.timestamp < NylasEnv.savedState.activityListViewed;
+  }
+
+  focusThread(threadId) {
+    NylasEnv.displayWindow()
+    Actions.closePopover()
+    DatabaseStore.find(Thread, threadId).then((thread) => {
+      if (!thread) {
+        NylasEnv.reportError(new Error(`ActivityListStore::focusThread: Can't find thread`, {threadId}))
+        NylasEnv.showErrorDialog(`Can't find the selected thread in your mailbox`)
+        return;
+      }
+      Actions.ensureCategoryIsFocused('sent', thread.accountId);
+      Actions.setFocus({collection: 'thread', item: thread});
+    });
+  }
+
+  getRecipient(recipientEmail, recipients) {
+    if (recipientEmail) {
+      for (const recipient of recipients) {
+        if (recipientEmail === recipient.email) {
+          return recipient;
+        }
+      }
+    } else if (recipients.length === 1) {
+      return recipients[0];
+    }
+    return null;
+  }
+
+  _dataSource() {
+    return new ActivityDataSource();
+  }
+
+  _onResetSeen() {
+    NylasEnv.savedState.activityListViewed = Date.now() / 1000;
+    this._unreadCount = 0;
+    this.trigger();
+  }
+
+  _getActivity() {
+    const dataSource = this._dataSource();
+    this._subscription = dataSource.buildObservable({
+      openTrackingId: NylasEnv.packages.pluginIdFor('open-tracking'),
+      linkTrackingId: NylasEnv.packages.pluginIdFor('link-tracking'),
+      messageLimit: 500,
+    }).subscribe((messages) => {
+      this._messages = messages;
+      this._updateActivity();
+    });
+  }
+
+  _updateActivity() {
+    this._actions = this._messages ? this._getActions(this._messages) : [];
+    this.trigger();
+  }
+
+  _getActions(messages) {
+    let actions = [];
+    this._notifications = [];
+    this._unreadCount = 0;
+    const sidebarAccountIds = FocusedPerspectiveStore.sidebarAccountIds();
+    for (const message of messages) {
+      if (sidebarAccountIds.length > 1 || message.accountId === sidebarAccountIds[0]) {
+        const openTrackingId = NylasEnv.packages.pluginIdFor('open-tracking')
+        const linkTrackingId = NylasEnv.packages.pluginIdFor('link-tracking')
+        if (message.metadataForPluginId(openTrackingId) ||
+          message.metadataForPluginId(linkTrackingId)) {
+          actions = actions.concat(this._openActionsForMessage(message));
+          actions = actions.concat(this._linkActionsForMessage(message));
+        }
+      }
+    }
+    if (!this._lastNotified) this._lastNotified = {};
+    for (const notification of this._notifications) {
+      const lastNotified = this._lastNotified[notification.threadId];
+      const {notificationInterval} = pluginFor(notification.pluginId);
+      if (!lastNotified || lastNotified < Date.now() - notificationInterval) {
+        NativeNotifications.displayNotification(notification.data);
+        this._lastNotified[notification.threadId] = Date.now();
+      }
+    }
+    const d = new Date();
+    this._lastChecked = d.getTime() / 1000;
+
+    actions = actions.sort((a, b) => b.timestamp - a.timestamp);
+    // For performance reasons, only display the last 100 actions
+    if (actions.length > 100) {
+      actions.length = 100;
+    }
+    return actions;
+  }
+
+  _openActionsForMessage(message) {
+    const openTrackingId = NylasEnv.packages.pluginIdFor('open-tracking')
+    const openMetadata = message.metadataForPluginId(openTrackingId);
+    const recipients = message.to.concat(message.cc, message.bcc);
+    const actions = [];
+    if (openMetadata) {
+      if (openMetadata.open_count > 0) {
+        for (const open of openMetadata.open_data) {
+          const recipient = this.getRecipient(open.recipient, recipients);
+          if (open.timestamp > this._lastChecked) {
+            this._notifications.push({
+              pluginId: openTrackingId,
+              threadId: message.threadId,
+              data: {
+                title: "New open",
+                subtitle: `${recipient ? recipient.displayName() : "Someone"} just opened ${message.subject}`,
+                canReply: false,
+                tag: "message-open",
+                onActivate: () => {
+                  this.focusThread(message.threadId);
+                },
+              },
+            });
+          }
+          if (!this.hasBeenViewed(open)) this._unreadCount += 1;
+          actions.push({
+            messageId: message.id,
+            threadId: message.threadId,
+            title: message.subject,
+            recipient: recipient,
+            pluginId: openTrackingId,
+            timestamp: open.timestamp,
+          });
+        }
+      }
+    }
+    return actions;
+  }
+
+  _linkActionsForMessage(message) {
+    const linkTrackingId = NylasEnv.packages.pluginIdFor('link-tracking')
+    const linkMetadata = message.metadataForPluginId(linkTrackingId)
+    const recipients = message.to.concat(message.cc, message.bcc);
+    const actions = [];
+    if (linkMetadata && linkMetadata.links) {
+      for (const link of linkMetadata.links) {
+        for (const click of link.click_data) {
+          const recipient = this.getRecipient(click.recipient, recipients);
+          if (click.timestamp > this._lastChecked) {
+            this._notifications.push({
+              pluginId: linkTrackingId,
+              threadId: message.threadId,
+              data: {
+                title: "New click",
+                subtitle: `${recipient ? recipient.displayName() : "Someone"} just clicked ${link.url}.`,
+                canReply: false,
+                tag: "link-open",
+                onActivate: () => {
+                  this.focusThread(message.threadId);
+                },
+              },
+            });
+          }
+          if (!this.hasBeenViewed(click)) this._unreadCount += 1;
+          actions.push({
+            messageId: message.id,
+            threadId: message.threadId,
+            title: link.url,
+            recipient: recipient,
+            pluginId: linkTrackingId,
+            timestamp: click.timestamp,
+          });
+        }
+      }
+    }
+    return actions;
+  }
+}
+
+export default new ActivityListStore();
diff --git a/packages/local-private/packages/activity-list/lib/activity-list.jsx b/packages/local-private/packages/activity-list/lib/activity-list.jsx
new file mode 100644
index 000000000..e60473e3c
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/activity-list.jsx
@@ -0,0 +1,100 @@
+import React from 'react';
+import classnames from 'classnames';
+
+import {Flexbox,
+  ScrollRegion} from 'nylas-component-kit';
+import ActivityListStore from './activity-list-store';
+import ActivityListActions from './activity-list-actions';
+import ActivityListItemContainer from './activity-list-item-container';
+import ActivityListEmptyState from './activity-list-empty-state';
+
+class ActivityList extends React.Component {
+
+  static displayName = 'ActivityList';
+
+  constructor() {
+    super();
+    this.state = this._getStateFromStores();
+  }
+
+  componentDidMount() {
+    this._unsub = ActivityListStore.listen(this._onDataChanged);
+  }
+
+  componentWillUnmount() {
+    ActivityListActions.resetSeen();
+    this._unsub();
+  }
+
+  _onDataChanged = () => {
+    this.setState(this._getStateFromStores());
+  }
+
+  _getStateFromStores() {
+    const actions = ActivityListStore.actions();
+    return {
+      actions: actions,
+      empty: actions instanceof Array && actions.length === 0,
+      collapsedToggles: this.state ? this.state.collapsedToggles : {},
+    }
+  }
+
+  _groupActions(actions) {
+    const groupedActions = [];
+    for (const action of actions) {
+      if (groupedActions.length > 0) {
+        const currentGroup = groupedActions[groupedActions.length - 1];
+        if (action.messageId === currentGroup[0].messageId &&
+          action.pluginId === currentGroup[0].pluginId) {
+          groupedActions[groupedActions.length - 1].push(action);
+        } else {
+          groupedActions.push([action]);
+        }
+      } else {
+        groupedActions.push([action]);
+      }
+    }
+    return groupedActions;
+  }
+
+  renderActions() {
+    if (this.state.empty) {
+      return (
+        
+      )
+    }
+
+    const groupedActions = this._groupActions(this.state.actions);
+    return groupedActions.map((group) => {
+      return (
+        
+      );
+    });
+  }
+
+  render() {
+    if (!this.state.actions) return null;
+
+    const classes = classnames({
+      "activity-list-container": true,
+      "empty": this.state.empty,
+    })
+    return (
+      
+        
+          {this.renderActions()}
+        
+      
+    );
+  }
+}
+
+export default ActivityList;
diff --git a/packages/local-private/packages/activity-list/lib/main.es6 b/packages/local-private/packages/activity-list/lib/main.es6
new file mode 100644
index 000000000..6ac9bd2b6
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/main.es6
@@ -0,0 +1,21 @@
+import {ComponentRegistry, WorkspaceStore} from 'nylas-exports';
+import {HasTutorialTip} from 'nylas-component-kit';
+import ActivityListButton from './activity-list-button';
+import ActivityListStore from './activity-list-store';
+
+const ActivityListButtonWithTutorialTip = HasTutorialTip(ActivityListButton, {
+  title: "Open and link tracking",
+  instructions: "If you've enabled link tracking or read receipts, those events will appear here!",
+});
+
+export function activate() {
+  ComponentRegistry.register(ActivityListButtonWithTutorialTip, {
+    location: WorkspaceStore.Location.RootSidebar.Toolbar,
+  });
+  ActivityListStore.activate();
+}
+
+
+export function deactivate() {
+  ComponentRegistry.unregister(ActivityListButtonWithTutorialTip);
+}
diff --git a/packages/local-private/packages/activity-list/lib/plugin-helpers.es6 b/packages/local-private/packages/activity-list/lib/plugin-helpers.es6
new file mode 100644
index 000000000..8e9d5a6a4
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/plugin-helpers.es6
@@ -0,0 +1,22 @@
+
+export function pluginFor(id) {
+  const openTrackingId = NylasEnv.packages.pluginIdFor('open-tracking')
+  const linkTrackingId = NylasEnv.packages.pluginIdFor('link-tracking')
+  if (id === openTrackingId) {
+    return {
+      name: "open",
+      predicate: "opened",
+      iconName: "icon-activity-mailopen.png",
+      notificationInterval: 600000, // 10 minutes in ms
+    }
+  }
+  if (id === linkTrackingId) {
+    return {
+      name: "link",
+      predicate: "clicked",
+      iconName: "icon-activity-linkopen.png",
+      notificationInterval: 10000, // 10 seconds in ms
+    }
+  }
+  return undefined
+}
diff --git a/packages/local-private/packages/activity-list/lib/test-data-source.es6 b/packages/local-private/packages/activity-list/lib/test-data-source.es6
new file mode 100644
index 000000000..40022ee02
--- /dev/null
+++ b/packages/local-private/packages/activity-list/lib/test-data-source.es6
@@ -0,0 +1,18 @@
+export default class TestDataSource {
+  buildObservable() {
+    return this;
+  }
+
+  manuallyTrigger = (messages = []) => {
+    this.onNext(messages);
+  }
+
+  subscribe(onNext) {
+    this.onNext = onNext;
+    this.manuallyTrigger();
+    const dispose = () => {
+      this._unsub();
+    }
+    return {dispose};
+  }
+}
diff --git a/packages/local-private/packages/activity-list/package.json b/packages/local-private/packages/activity-list/package.json
new file mode 100644
index 000000000..570429139
--- /dev/null
+++ b/packages/local-private/packages/activity-list/package.json
@@ -0,0 +1,21 @@
+{
+  "name": "activity-list",
+  "main": "./lib/main",
+  "version": "0.1.0",
+  "repository": {
+    "type": "git",
+    "url": ""
+  },
+  "engines": {
+    "nylas": "*"
+  },
+
+  "isOptional": true,
+
+  "title":"Activity List",
+  "icon":"./assets/icon.png",
+  "description": "Get notifications for open and link tracking activity.",
+  "supportedEnvs": ["development", "staging", "production"],
+
+  "license": "GPL-3.0"
+}
diff --git a/packages/local-private/packages/activity-list/spec/activity-list-spec.jsx b/packages/local-private/packages/activity-list/spec/activity-list-spec.jsx
new file mode 100644
index 000000000..5ffdd03ad
--- /dev/null
+++ b/packages/local-private/packages/activity-list/spec/activity-list-spec.jsx
@@ -0,0 +1,195 @@
+import React from 'react';
+import ReactTestUtils from 'react-addons-test-utils';
+import {
+  Thread,
+  Actions,
+  Contact,
+  Message,
+  DatabaseStore,
+  FocusedPerspectiveStore,
+} from 'nylas-exports';
+import ActivityList from '../lib/activity-list';
+import ActivityListStore from '../lib/activity-list-store';
+import TestDataSource from '../lib/test-data-source';
+
+const OPEN_TRACKING_ID = 'open-tracking-id'
+const LINK_TRACKING_ID = 'link-tracking-id'
+
+const messages = [
+  new Message({
+    accountId: "0000000000000000000000000",
+    bcc: [],
+    cc: [],
+    snippet: "Testing.",
+    subject: "Open me!",
+    threadId: "0000000000000000000000000",
+    to: [new Contact({
+      name: "Jackie Luo",
+      email: "jackie@nylas.com",
+    })],
+  }),
+  new Message({
+    accountId: "0000000000000000000000000",
+    bcc: [new Contact({
+      name: "Ben Gotow",
+      email: "ben@nylas.com",
+    })],
+    cc: [],
+    snippet: "Hey! I am in town for the week...",
+    subject: "Coffee?",
+    threadId: "0000000000000000000000000",
+    to: [new Contact({
+      name: "Jackie Luo",
+      email: "jackie@nylas.com",
+    })],
+  }),
+  new Message({
+    accountId: "0000000000000000000000000",
+    bcc: [],
+    cc: [new Contact({
+      name: "Evan Morikawa",
+      email: "evan@nylas.com",
+    })],
+    snippet: "Here's the latest deals!",
+    subject: "Newsletter",
+    threadId: "0000000000000000000000000",
+    to: [new Contact({
+      name: "Juan Tejada",
+      email: "juan@nylas.com",
+    })],
+  }),
+];
+
+let pluginValue = {
+  open_count: 1,
+  open_data: [{
+    timestamp: 1461361759.351055,
+  }],
+};
+messages[0].applyPluginMetadata(OPEN_TRACKING_ID, pluginValue);
+pluginValue = {
+  links: [{
+    click_count: 1,
+    click_data: [{
+      timestamp: 1461349232.495837,
+    }],
+  }],
+  tracked: true,
+};
+messages[0].applyPluginMetadata(LINK_TRACKING_ID, pluginValue);
+pluginValue = {
+  open_count: 1,
+  open_data: [{
+    timestamp: 1461361763.283720,
+  }],
+};
+messages[1].applyPluginMetadata(OPEN_TRACKING_ID, pluginValue);
+pluginValue = {
+  links: [],
+  tracked: false,
+};
+messages[1].applyPluginMetadata(LINK_TRACKING_ID, pluginValue);
+pluginValue = {
+  open_count: 0,
+  open_data: [],
+};
+messages[2].applyPluginMetadata(OPEN_TRACKING_ID, pluginValue);
+pluginValue = {
+  links: [{
+    click_count: 0,
+    click_data: [],
+  }],
+  tracked: true,
+};
+messages[2].applyPluginMetadata(LINK_TRACKING_ID, pluginValue);
+
+
+describe('ActivityList', function activityList() {
+  beforeEach(() => {
+    this.testSource = new TestDataSource();
+    spyOn(NylasEnv.packages, 'pluginIdFor').andCallFake((pluginName) => {
+      if (pluginName === 'open-tracking') {
+        return OPEN_TRACKING_ID
+      }
+      if (pluginName === 'link-tracking') {
+        return LINK_TRACKING_ID
+      }
+      return null
+    })
+    spyOn(ActivityListStore, "_dataSource").andReturn(this.testSource);
+    spyOn(FocusedPerspectiveStore, "sidebarAccountIds").andReturn(["0000000000000000000000000"]);
+    spyOn(DatabaseStore, "run").andCallFake((query) => {
+      if (query._klass === Thread) {
+        const thread = new Thread({
+          id: "0000000000000000000000000",
+          accountId: TEST_ACCOUNT_ID,
+        });
+        return Promise.resolve(thread);
+      }
+      return null;
+    });
+    spyOn(ActivityListStore, "focusThread").andCallThrough();
+    spyOn(NylasEnv, "displayWindow");
+    spyOn(Actions, "closePopover");
+    spyOn(Actions, "setFocus");
+    spyOn(Actions, "ensureCategoryIsFocused");
+    ActivityListStore.activate();
+    this.component = ReactTestUtils.renderIntoDocument();
+  });
+
+  describe('when no actions are found', () => {
+    it('should show empty state', () => {
+      const items = ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item");
+      expect(items.length).toBe(0);
+    });
+  });
+
+  describe('when actions are found', () => {
+    it('should show activity list items', () => {
+      this.testSource.manuallyTrigger(messages);
+      waitsFor(() => {
+        const items = ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item");
+        return items.length > 0;
+      });
+      runs(() => {
+        expect(ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item").length).toBe(3);
+      });
+    });
+
+    it('should show the correct items', () => {
+      this.testSource.manuallyTrigger(messages);
+      waitsFor(() => {
+        const items = ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item");
+        return items.length > 0;
+      });
+      runs(() => {
+        expect(ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item")[0].textContent).toBe("Someone opened:Apr 22Coffee?");
+        expect(ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item")[1].textContent).toBe("Jackie Luo opened:Apr 22Open me!");
+        expect(ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item")[2].textContent).toBe("Jackie Luo clicked:Apr 22(No Subject)");
+      });
+    });
+
+    it('should focus the thread', () => {
+      this.testSource.manuallyTrigger(messages);
+      waitsFor(() => {
+        const items = ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item");
+        return items.length > 0;
+      });
+      runs(() => {
+        const item = ReactTestUtils.scryRenderedDOMComponentsWithClass(this.component, "activity-list-item")[0];
+        ReactTestUtils.Simulate.click(item);
+      });
+      waitsFor(() => {
+        return ActivityListStore.focusThread.calls.length > 0;
+      });
+      runs(() => {
+        setImmediate(() => {
+          expect(NylasEnv.displayWindow.calls.length).toBe(1);
+          expect(Actions.closePopover.calls.length).toBe(1);
+          expect(Actions.setFocus.calls.length).toBe(1);
+          expect(Actions.ensureCategoryIsFocused.calls.length).toBe(1);
+        })
+      });
+    });
+  });
+});
diff --git a/packages/local-private/packages/activity-list/stylesheets/activity-list.less b/packages/local-private/packages/activity-list/stylesheets/activity-list.less
new file mode 100644
index 000000000..13a0d0754
--- /dev/null
+++ b/packages/local-private/packages/activity-list/stylesheets/activity-list.less
@@ -0,0 +1,142 @@
+@import "ui-variables";
+
+.toolbar-activity {
+  order: 100;
+  position: relative;
+
+  .unread-count {
+    display: none;
+    &.active {
+      display: inline-block;
+      background: @component-active-color;
+      text-align: center;
+      color: @white;
+      border-radius: @border-radius-base;
+      font-size: 8px;
+      padding: 0 4px;
+      position: absolute;
+      right: -7px;
+      top: 5px;
+      line-height: 11px;
+    }
+  }
+  .activity-toolbar-icon {
+    margin-top: 20px;
+    background: @gray;
+    &.unread {
+      background: @component-active-color;
+    }
+  }
+}
+
+.activity-list-container {
+  width: 260px;
+  overflow: hidden;
+  font-size: @font-size-small;
+  color: @text-color-subtle;
+  .spacer {
+    flex: 1 1 0;
+  }
+
+  height: 282px;
+  &.empty {
+    height: 182px;
+  }
+
+  .empty {
+      text-align: center;
+      padding: @padding-base-horizontal * 2;
+      padding-top: @padding-base-vertical * 8;
+      img.logo {
+        background-color: @text-color-very-subtle;
+      }
+      .text {
+        margin-top: @padding-base-vertical * 6;
+        color: @text-color-very-subtle;
+      }
+  }
+
+  .activity-list-item {
+    padding: @padding-small-vertical @padding-small-horizontal;
+    white-space: nowrap;
+    border-bottom: 1px solid @border-color-primary;
+    cursor: default;
+    &.unread {
+      color: @text-color;
+      background: @background-primary;
+      &:hover {
+        background: darken(@background-primary, 2%);
+      }
+      .action-message {
+        font-weight: 600;
+      }
+    }
+
+    &:hover {
+      background: darken(@background-secondary, 2%);
+    }
+
+    .disclosure-triangle {
+      padding-top: 5px;
+      padding-bottom: 0;
+    }
+    .activity-icon-container {
+      flex-shrink: 0;
+    }
+    .activity-icon {
+      vertical-align: text-bottom;
+    }
+    .action-message, .title {
+      text-overflow: ellipsis;
+      overflow: hidden;
+    }
+    .timestamp {
+      color: @text-color-very-subtle;
+      text-overflow: ellipsis;
+      overflow: hidden;
+      flex-shrink: 0;
+      padding-left: 5px;
+    }
+  }
+  .activity-list-toggle-item {
+    height: 30px;
+    white-space: nowrap;
+    background: @background-secondary;
+    cursor: default;
+    overflow-y: hidden;
+    transition-property: all;
+    transition-duration: .5s;
+    transition-timing-function: cubic-bezier(0, 1, 0.5, 1);
+    &:last-child {
+      border-bottom: 1px solid @border-color-primary;
+    }
+    .action-message {
+      padding: @padding-small-vertical @padding-small-horizontal;
+      text-overflow: ellipsis;
+      overflow: hidden;
+    }
+    .timestamp {
+      padding: @padding-small-vertical @padding-small-horizontal;
+      color: @text-color-very-subtle;
+      text-overflow: ellipsis;
+      overflow: hidden;
+    }
+  }
+  .activity-toggle-container {
+    &.hidden {
+      .activity-list-toggle-item {
+        height: 0;
+        &:last-child {
+          border-bottom: none;
+        }
+      }
+    }
+  }
+}
+
+body.platform-win32,
+body.platform-linux {
+  .toolbar-activity {
+    margin-right: @padding-base-horizontal;
+  }
+}
\ No newline at end of file
diff --git a/packages/local-private/packages/composer-mail-merge/icon.png b/packages/local-private/packages/composer-mail-merge/icon.png
new file mode 100644
index 0000000000000000000000000000000000000000..fcc7513bed2a74570d19a3acd46fc96ef89f3db1
GIT binary patch
literal 15465
zcmeI3Yg7|w8pnr~Rzx|1^JX&o`$nzRG;{v&E%3AZX|~_U#v+
z81@4MMZQKR>uJ4iCW2EIF=nI;gxG1ZF=_}xCp&Ezo<-0C1Cc>mqlNbl9}@~lW3(_;
zsgvkzF+?UAS70ZS3T7qa1zC8MQ8-y0h&mA_frX$kfzx8PIuK{H(47}yVzwC;3fvMp
zD_W>w3kvkQ1VIdCCj?4ygb0^Js06Agu~ey2DOBMCnMA6DC309U5y|9;LV_q{0`E(x
z4rC(KZZsi@+UeeM%&%x+CQaKA7|zSf6X(grlsyBMMny%z5*aL$i5P{*k#D6jr^xCE
z^(1+gM@u+xJ87dy$|_*YNM%?GI
z<=D;cp&4zv`3#UlcFg{Ce=$xUw5yCnK{wve6@QVuf@`1X$L(mmqFk~
z+N0BvoNpuGM8ZMk*l}W7E^{fNJzYIbnmQ+^OD49b*Rc~$vKJfM$
zJAu)ZJ(;4+YR@W5c!p3A6XRZ0g7G>GC#@{k2`Jp%)H&mw`3NmW6KZBfMu;Q|kxZE^
zRU+~TM4=Q(RER_ZB=IP_<fdU>ENP{!xad9F71w1a024~FU
z;zR@rcw8V2&X~u=i3k+%xIh}5F^`KA5h&nsfiyT{9v3GfP{88?X>i6oE>1+CfX4;W
z;EZ`(oQOaHj|-&18S}U}5rG087f6FM=5cW%0tGxSkOpV`v$z5~9{?n*%u|1P%;SBk
zyE*mDqk#fEE>Q!j;gNX)r!)@?q1;(@iF0dfv*9MBwQXehMC-f5*y;Ud4G3vHLit`6Q^cs8in
zzbT`}zbUIm&@{?mQ(rtPtf=@lN3bxqVNK@5OwqB>b;5O_Pc=*4IP`h?0Ef1-bN21OB7uU
z<91%zg?IEX8T;)NG+|!ust&ndyDaF|?3!V&tA7dm&*zI(h0U>d+NhI5&K+3gC>j@^
zIQR|1^|i}dV|en_vQmBf$^aMq@sx3{F~KT9$&J6jl(-^4ULsGWx;1do&=|kZ4qgP9G`o6(A7<=
z%Xi#K4Bx$}7FqK+=t{<3emYt_{p_(xM6GVoi3R7&&K$f{
zdN^?VTlKFl$G$UnG#nYOtT_C|xn+{m9YFG6+)-%cE$%#y#V>aXAX
z!d7VGnJsG```4x2`eElwiw9+I6|XXtY?W8l9+y?jxR_5@Wd-+ft$GlCbN_-#tLmXG
zpMN6=XyH%y^qQJS!o9KcHyC#9ch!pjk#8(7T~4Hrs2>;f=pIxw
Y|D$&Xl+8(HHarwRZI*WL)byhN11-LP!vFvP

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/composer-mail-merge/lib/listens-to-mail-merge-session.jsx b/packages/local-private/packages/composer-mail-merge/lib/listens-to-mail-merge-session.jsx
new file mode 100644
index 000000000..5ef87a72f
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/listens-to-mail-merge-session.jsx
@@ -0,0 +1,64 @@
+/* eslint no-prototype-builtins: 0 */
+import React, {Component, PropTypes} from 'react';
+import {mailMergeSessionForDraft} from './mail-merge-draft-editing-session'
+
+
+export default function ListensToMailMergeSession(ComposedComponent) {
+  return class extends Component {
+    static displayName = ComposedComponent.displayName
+
+    static containerRequired = false
+
+    static propTypes = {
+      session: PropTypes.object,
+      draftClientId: PropTypes.string,
+      ...ComposedComponent.propTypes,
+    }
+
+    constructor(props) {
+      super(props)
+      this.unlisten = () => {}
+      this.state = {
+        mailMergeSession: mailMergeSessionForDraft(props.draftClientId, props.session),
+      };
+    }
+
+    componentDidMount() {
+      const {mailMergeSession} = this.state;
+      if (mailMergeSession) {
+        this.unlisten = mailMergeSession.listen(() => {
+          this.setState({mailMergeSession})
+        });
+      }
+    }
+
+    componentWillUnmount() {
+      this.unlisten();
+    }
+
+    focus() {
+      if (this.refs.composed) {
+        this.refs.composed.focus()
+      }
+    }
+
+    render() {
+      const {mailMergeSession} = this.state;
+
+      if (!mailMergeSession) {
+        return 
+      }
+      const componentProps = {
+        ...this.props,
+        mailMergeSession: mailMergeSession,
+        sessionState: mailMergeSession.state,
+      }
+      if (Component.isPrototypeOf(ComposedComponent)) {
+        componentProps.ref = 'composed'
+      }
+      return (
+        
+      )
+    }
+  }
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-body-token.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-body-token.jsx
new file mode 100644
index 000000000..72858d9d6
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-body-token.jsx
@@ -0,0 +1,132 @@
+import React, {Component, PropTypes} from 'react'
+import MailMergeToken from './mail-merge-token'
+import {DragBehaviors} from './mail-merge-constants'
+import {tokenQuerySelector} from './mail-merge-utils'
+import ListensToMailMergeSession from './listens-to-mail-merge-session'
+
+/**
+ * MailMergeBodyTokens are rendered by the OverlaidComponents component in the
+ * subject and body of the composer.
+ * The OverlaidComponents' state is effectively the state of the contenteditable
+ * inside those fields, * and it decides what to render based on the
+ * anchor (img) tags that are present in the contenteditable.
+ *
+ * Given this setup, we use the lifecycle methods of MailMergeBodyToken to keep
+ * the state of the contenteditable (the tokens actually rendered in the UI),
+ * in sync with our token state for mail merge (tokenDataSource)
+ */
+class MailMergeBodyToken extends Component {
+  static displayName = 'MailMergeBodyToken'
+
+  static propTypes = {
+    className: PropTypes.string,
+    tokenId: PropTypes.string,
+    field: PropTypes.string,
+    colName: PropTypes.string,
+    sessionState: PropTypes.object,
+    mailMergeSession: PropTypes.object,
+    draftClientId: PropTypes.string,
+    colIdx: PropTypes.oneOfType([PropTypes.string, PropTypes.number]),
+    isPreview: PropTypes.bool,
+  }
+
+  constructor(props) {
+    super(props)
+    this.state = this.getState(props)
+  }
+
+  componentDidMount() {
+    // When the token gets mounted, it means a mail merge token anchor node was
+    // added to the contenteditable, via drop, paste, or any other means, so we
+    // add it to our mail merge state
+    const {colIdx, field, colName, tokenId, mailMergeSession} = this.props
+    const {tokenDataSource} = mailMergeSession.state
+    const token = tokenDataSource.getToken(field, tokenId)
+    if (!token) {
+      mailMergeSession.linkToDraft({colIdx, field, colName, tokenId})
+    }
+  }
+
+  componentWillReceiveProps(nextProps) {
+    this.setState(this.getState(nextProps, this.state.colIdx))
+  }
+
+  shouldComponentUpdate(nextProps, nextState) {
+    return (
+      this.props.isPreview !== nextProps.isPreview ||
+      this.state.colIdx !== nextState.colIdx ||
+      this.props.sessionState.selection !== nextProps.sessionState.selection ||
+      this.props.sessionState.tableDataSource !== nextProps.sessionState.tableDataSource ||
+      this.props.sessionState.tokenDataSource !== nextProps.sessionState.tokenDataSource
+    )
+  }
+
+  componentDidUpdate() {
+    // A token might be removed by mutations to the contenteditable, in which
+    // case the tokenDataSource's state is updated by componentWillUnmount.
+    //
+    // However, when a token is removed from state via other means, e.g. when a
+    // table column is removed, we also want to make sure that we remove it from the
+    // UI. Since the contenteditable is effectively the source of state for
+    // OverlaidComponents, we imperatively remove the token from contenteditable
+    // if it has been removed from our state.
+    const {field, tokenId, sessionState: {tokenDataSource}} = this.props
+    const token = tokenDataSource.getToken(field, tokenId)
+    if (!token) {
+      const node = document.querySelector(tokenQuerySelector(tokenId))
+      if (node) {
+        node.parentNode.removeChild(node)
+      }
+    }
+  }
+
+  componentWillUnmount() {
+    // A token might be removed by any sort of mutations to the contenteditable.
+    // When an the actual anchor node in the contenteditable is removed from
+    // the dom tree, OverlaidComponents will unmount our corresponding token,
+    // so this is where we get to update our tokenDataSource's state
+    const {field, tokenId, mailMergeSession} = this.props
+    mailMergeSession.unlinkFromDraft({field, tokenId})
+  }
+
+  getState(props) {
+    // Keep colIdx as state in case the column changes index when importing a
+    // new csv file, thus changing styling
+    const {sessionState: {tokenDataSource}, field, tokenId} = props
+    const nextToken = tokenDataSource.getToken(field, tokenId)
+    if (nextToken) {
+      const {colIdx, colName} = nextToken
+      return {colIdx, colName}
+    }
+    const {colIdx, colName} = props
+    return {colIdx, colName}
+  }
+
+  render() {
+    const {colIdx, colName} = this.state
+    const {className, draftClientId, sessionState, isPreview} = this.props
+    const {tableDataSource, selection} = sessionState
+    const selectionValue = tableDataSource.cellAt({rowIdx: selection.rowIdx, colIdx}) || "No value selected"
+
+    if (isPreview) {
+      return {selectionValue}
+    }
+
+    return (
+      
+        
+          
+            {selectionValue}
+          
+        
+      
+    )
+  }
+}
+export default ListensToMailMergeSession(MailMergeBodyToken)
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-button.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-button.jsx
new file mode 100644
index 000000000..c3fbbd435
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-button.jsx
@@ -0,0 +1,46 @@
+import classnames from 'classnames'
+import React, {PropTypes} from 'react'
+import {RetinaImg} from 'nylas-component-kit'
+import ListensToMailMergeSession from './listens-to-mail-merge-session'
+
+
+function MailMergeButton(props) {
+  if (props.draft.replyToMessageId) {
+    return ;
+  }
+
+  const {mailMergeSession, sessionState} = props
+  const {isWorkspaceOpen} = sessionState
+  const classes = classnames({
+    "btn": true,
+    "btn-toolbar": true,
+    "btn-enabled": isWorkspaceOpen,
+    "btn-mail-merge": true,
+  })
+
+  return (
+    
+  )
+}
+MailMergeButton.displayName = 'MailMergeButton'
+MailMergeButton.containerRequired = false
+MailMergeButton.propTypes = {
+  draft: PropTypes.object,
+  session: PropTypes.object,
+  sessionState: PropTypes.object,
+  draftClientId: PropTypes.string,
+  mailMergeSession: PropTypes.object,
+}
+
+export default ListensToMailMergeSession(MailMergeButton)
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-composer-extension.es6 b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-composer-extension.es6
new file mode 100644
index 000000000..bb984b971
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-composer-extension.es6
@@ -0,0 +1,10 @@
+import * as Handlers from './mail-merge-token-dnd-handlers'
+
+export const name = 'MailMergeComposerExtension'
+
+export {
+  onDragOver,
+  shouldAcceptDrop,
+} from './mail-merge-token-dnd-handlers'
+
+export const onDrop = Handlers.onDrop.bind(null, 'body')
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-constants.es6 b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-constants.es6
new file mode 100644
index 000000000..4c00e36ef
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-constants.es6
@@ -0,0 +1,37 @@
+import plugin from '../package.json'
+
+export const PLUGIN_ID = plugin.name;
+export const PLUGIN_NAME = "Mail Merge"
+export const DEBUG = false
+export const MAX_ROWS = 150
+
+export const ParticipantFields = ['to', 'cc', 'bcc']
+export const ContenteditableFields = ['subject', 'body']
+export const LinkableFields = [...ParticipantFields, ...ContenteditableFields]
+
+export const DataTransferTypes = {
+  ColIdx: 'mail-merge:col-idx',
+  ColName: 'mail-merge:col-name',
+  DraftId: 'mail-merge:draft-client-id',
+  DragBehavior: 'mail-merge:drag-behavior',
+}
+
+export const DragBehaviors = {
+  Copy: 'copy',
+  Move: 'move',
+}
+
+export const ActionNames = [
+  'addColumn',
+  'removeLastColumn',
+  'addRow',
+  'removeRow',
+  'updateCell',
+  'shiftSelection',
+  'setSelection',
+  'clearTableData',
+  'loadTableData',
+  'toggleWorkspace',
+  'linkToDraft',
+  'unlinkFromDraft',
+]
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-container.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-container.jsx
new file mode 100644
index 000000000..65a9eaef0
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-container.jsx
@@ -0,0 +1,39 @@
+import React, {Component, PropTypes} from 'react'
+import MailMergeWorkspace from './mail-merge-workspace'
+import ListensToMailMergeSession from './listens-to-mail-merge-session'
+
+
+class MailMergeContainer extends Component {
+  static displayName = 'MailMergeContainer'
+
+  static containerRequired = false
+
+  static propTypes = {
+    session: PropTypes.object,
+    sessionState: PropTypes.object,
+    draftClientId: PropTypes.string,
+    mailMergeSession: PropTypes.object,
+  }
+
+  shouldComponentUpdate(nextProps) {
+    // Make sure we only update if new state has been set
+    // We do not care about our other props
+    return (
+      this.props.draftClientId !== nextProps.draftClientId ||
+      this.props.sessionState !== nextProps.sessionState
+    )
+  }
+
+  render() {
+    const {draftClientId, sessionState, mailMergeSession} = this.props
+    return (
+      
+    )
+  }
+}
+
+export default ListensToMailMergeSession(MailMergeContainer)
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-draft-editing-session.es6 b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-draft-editing-session.es6
new file mode 100644
index 000000000..1a80ab511
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-draft-editing-session.es6
@@ -0,0 +1,146 @@
+import NylasStore from 'nylas-store'
+import * as TableStateReducers from './table-state-reducers'
+import * as TokenStateReducers from './token-state-reducers'
+import * as SelectionStateReducers from './selection-state-reducers'
+import * as WorkspaceStateReducers from './workspace-state-reducers'
+import {ActionNames, PLUGIN_ID, DEBUG} from './mail-merge-constants'
+
+
+const sessions = new Map()
+
+function computeNextState({name, args = []}, previousState = {}, reducers = []) {
+  if (reducers.length === 0) {
+    return previousState
+  }
+  return reducers.reduce((state, reducer) => {
+    if (reducer[name]) {
+      const reduced = reducer[name](previousState, ...args)
+      return {...state, ...reduced}
+    }
+    return state
+  }, previousState)
+}
+
+/**
+ * MailMergeDraftEditingSession instances hold the entire state for the Mail Merge
+ * plugin for a given draft, as a single state tree. Sessions trigger when any changes
+ * on the state tree occur.
+ *
+ * Mail Merge state for a draft can be modified by dispatching actions on a session instance.
+ * Available actions are defined by `MailMergeConstants.ActionNames`.
+ * Actions are dispatched by calling the action on a session as a method:
+ * ```
+ *  session.addColumn()
+ * ```
+ *
+ * Internally, the session acts as a Proxy which forwards action calls into any
+ * registered reducers, and merges the resulting state from calling the action
+ * on each reducer to compute the new state tree. Registered reducers are
+ * currently hardcoded in this class.
+ *
+ * A session instance also acts as a proxy for the corresponding `DraftEditingSession`,
+ * instance, and forwards to it any changes that need to be persisted on the draft object
+ *
+ * @class MailMergeDraftEditingSession
+ */
+export class MailMergeDraftEditingSession extends NylasStore {
+
+  constructor(session, reducers) {
+    super()
+    this._session = session
+    this._reducers = reducers || [
+      TableStateReducers,
+      TokenStateReducers,
+      SelectionStateReducers,
+      WorkspaceStateReducers,
+    ]
+    this._state = {}
+    this.initializeState()
+    this.initializeActionHandlers()
+  }
+
+  get state() {
+    return this._state
+  }
+
+  draft() {
+    return this._session.draft()
+  }
+
+  draftSession() {
+    return this._session
+  }
+
+  initializeState(draft = this._session.draft()) {
+    const savedMetadata = draft.metadataForPluginId(PLUGIN_ID)
+    const shouldLoadSavedData = (
+      savedMetadata &&
+      savedMetadata.tableDataSource &&
+      savedMetadata.tokenDataSource
+    )
+    const action = {name: 'initialState'}
+    if (shouldLoadSavedData) {
+      const loadedState = this.dispatch({name: 'fromJSON'}, savedMetadata)
+      this._state = this.dispatch(action, loadedState)
+    } else {
+      this._state = this.dispatch(action)
+    }
+  }
+
+  initializeActionHandlers() {
+    ActionNames.forEach((actionName) => {
+      // TODO ES6 Proxies would be nice here
+      this[actionName] = this.actionHandler(actionName).bind(this)
+    })
+  }
+
+  dispatch(action, prevState = this._state) {
+    const nextState = computeNextState(action, prevState, this._reducers)
+    if (DEBUG && action.debug !== false) {
+      console.log('--> action', action.name)
+      console.dir(action)
+      console.log('--> prev state')
+      console.dir(prevState)
+      console.log('--> new state')
+      console.dir(nextState)
+    }
+    return nextState
+  }
+
+  actionHandler(actionName) {
+    return (...args) => {
+      this._state = this.dispatch({name: actionName, args})
+
+      // Defer calling `saveToSession` to make sure our state changes are triggered
+      // before the draft changes
+      this.trigger()
+      setImmediate(this.saveToDraftSession)
+    }
+  }
+
+  saveToDraftSession = () => {
+    // TODO
+    // - What should we save in metadata?
+    //   - The entire table data?
+    //   - A reference to a statically hosted file?
+    //   - Attach csv as a file to the "base" or "template" draft?
+    const {tokenDataSource, tableDataSource} = this._state
+    const draftChanges = this.dispatch({name: 'toDraftChanges', args: [this._state], debug: false}, this.draft())
+    const serializedState = this.dispatch({name: 'toJSON', debug: false}, {tokenDataSource, tableDataSource})
+
+    this._session.changes.add(draftChanges)
+    this._session.changes.addPluginMetadata(PLUGIN_ID, serializedState)
+  }
+}
+
+export function mailMergeSessionForDraft(draftId, draftSession) {
+  if (sessions.has(draftId)) {
+    return sessions.get(draftId)
+  }
+  if (!draftSession) {
+    return null
+  }
+  const sess = new MailMergeDraftEditingSession(draftSession)
+  sessions.set(draftId, sess)
+  return sess
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-header-input.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-header-input.jsx
new file mode 100644
index 000000000..f17689662
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-header-input.jsx
@@ -0,0 +1,67 @@
+import React, {Component, PropTypes} from 'react'
+import {pickHTMLProps} from 'pick-react-known-prop'
+import MailMergeToken from './mail-merge-token'
+
+
+function getInputSize(value) {
+  return ((value || '').length || 1) + 1
+}
+
+class MailMergeHeaderInput extends Component {
+
+  static propTypes = {
+    draftClientId: PropTypes.string,
+    colIdx: PropTypes.any,
+    tableDataSource: PropTypes.object,
+    defaultValue: PropTypes.string,
+    onBlur: PropTypes.func,
+  }
+
+  constructor(props) {
+    super(props)
+    this.state = {inputSize: getInputSize(props.defaultValue)}
+  }
+
+  componentWillReceiveProps(nextProps) {
+    this.setState({inputSize: getInputSize(nextProps.defaultValue)})
+  }
+
+  onInputBlur = (event) => {
+    const {target: {value}} = event
+    this.setState({inputSize: getInputSize(value)})
+    // Can't override the original onBlur handler
+    this.props.onBlur(event)
+  }
+
+  onInputChange = (event) => {
+    const {target: {value}} = event
+    this.setState({inputSize: getInputSize(value)})
+  }
+
+  render() {
+    const {inputSize} = this.state
+    const {draftClientId, tableDataSource, colIdx, ...props} = this.props
+    const colName = tableDataSource.colAt(colIdx)
+
+    return (
+      

+        
+          
+        
+      

+    )
+  }
+}
+
+export default MailMergeHeaderInput
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-participants-text-field.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-participants-text-field.jsx
new file mode 100644
index 000000000..618c6f4f6
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-participants-text-field.jsx
@@ -0,0 +1,138 @@
+import React, {Component, PropTypes} from 'react';
+import classnames from 'classnames'
+import {DropZone, TokenizingTextField} from 'nylas-component-kit'
+import MailMergeToken from './mail-merge-token'
+import {DataTransferTypes} from './mail-merge-constants'
+import ListensToMailMergeSession from './listens-to-mail-merge-session'
+
+
+function MailMergeParticipantToken(props) {
+  const {token: {tableDataSource, rowIdx, colIdx, colName}} = props
+  const selectionValue = tableDataSource.cellAt({rowIdx, colIdx}) || 'No value selected'
+
+  return (
+    
+      {selectionValue}
+    
+  )
+}
+MailMergeParticipantToken.propTypes = {
+  token: PropTypes.shape({
+    colIdx: PropTypes.any,
+    rowIdx: PropTypes.any,
+    tableDataSource: PropTypes.object,
+  }),
+}
+
+
+class MailMergeParticipantsTextField extends Component {
+  static displayName = 'MailMergeParticipantsTextField'
+
+  static containerRequired = false
+
+  static propTypes = {
+    onAdd: PropTypes.func,
+    onRemove: PropTypes.func,
+    field: PropTypes.string,
+    session: PropTypes.object,
+    className: PropTypes.string,
+    sessionState: PropTypes.object,
+    draftClientId: PropTypes.string,
+    mailMergeSession: PropTypes.object,
+  }
+
+  static defaultProps = {
+    className: '',
+  }
+
+  constructor(props) {
+    super(props)
+    this._tokenWasMovedBetweenFields = false
+  }
+
+  // This is called by the TokenizingTextField when a token is dragged and dropped
+  // between fields
+  onAddToken = (...args) => {
+    const tokenToAdd = args[0][0]
+    if (args.length > 1 || !tokenToAdd) { return }
+
+    const {mailMergeSession} = this.props
+    const {colIdx, colName, tokenId, field} = tokenToAdd
+    // Remove from previous field
+    mailMergeSession.unlinkFromDraft({field, tokenId})
+    // Add to our current field
+    mailMergeSession.linkToDraft({colIdx, colName, field: this.props.field})
+    this._tokenWasMovedBetweenFields = true
+  }
+
+  onRemoveToken = ([tokenToDelete]) => {
+    const {field, mailMergeSession} = this.props
+    const {tokenId} = tokenToDelete
+    mailMergeSession.unlinkFromDraft({field, tokenId})
+  }
+
+  onDrop = (event) => {
+    if (this._tokenWasMovedBetweenFields) {
+      // Ignore drop if we already added the token
+      this._tokenWasMovedBetweenFields = false
+      return
+    }
+    const {dataTransfer} = event
+    const {field, mailMergeSession} = this.props
+    const colIdx = dataTransfer.getData(DataTransferTypes.ColIdx)
+    const colName = dataTransfer.getData(DataTransferTypes.ColName)
+    mailMergeSession.linkToDraft({colIdx, colName, field})
+  }
+
+  focus() {
+    this.refs.textField.focus()
+  }
+
+  shouldAcceptDrop = (event) => {
+    const {dataTransfer} = event
+    return !!dataTransfer.getData(DataTransferTypes.ColIdx)
+  }
+
+  render() {
+    const {field, className, sessionState} = this.props
+    const {isWorkspaceOpen, tableDataSource, selection, tokenDataSource} = sessionState
+
+    if (!isWorkspaceOpen) {
+      return 
+    }
+
+    const classes = classnames({
+      'mail-merge-participants-text-field': true,
+      [className]: true,
+    })
+    const tokens = (
+      tokenDataSource.tokensForField(field)
+      .map((token) => ({...token, tableDataSource, rowIdx: selection.rowIdx}))
+    )
+
+    return (
+      
+         token.tokenId}
+          tokenRenderer={MailMergeParticipantToken}
+          tokenIsValid={() => true}
+          tokenClassNames={(token) => `token-color-${token.colIdx % 5}`}
+          onRequestCompletions={() => []}
+          completionNode={() => }
+          onAdd={this.onAddToken}
+          onRemove={this.onRemoveToken}
+          onTokenAction={false}
+        />
+      
+    )
+  }
+}
+
+export default ListensToMailMergeSession(MailMergeParticipantsTextField)
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-send-button.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-send-button.jsx
new file mode 100644
index 000000000..1ea99de47
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-send-button.jsx
@@ -0,0 +1,115 @@
+import {remote} from 'electron'
+import React, {Component, PropTypes} from 'react'
+import {RetinaImg} from 'nylas-component-kit'
+import {sendMailMerge} from './mail-merge-utils'
+import ListensToMailMergeSession from './listens-to-mail-merge-session'
+
+
+class MailMergeSendButton extends Component {
+  static displayName = 'MailMergeSendButton'
+
+  static containerRequired = false
+
+  static propTypes = {
+    draft: PropTypes.object,
+    session: PropTypes.object,
+    sessionState: PropTypes.object,
+    isValidDraft: PropTypes.func,
+    fallback: PropTypes.func,
+  }
+
+  constructor(props) {
+    super(props)
+    this.state = {
+      sending: false,
+    }
+  }
+
+  onClick = () => {
+    const {sending} = this.state
+    if (sending) { return }
+
+    const {draft, isValidDraft} = this.props
+    if (draft.to.length === 0) {
+      const dialog = remote.dialog;
+      dialog.showMessageBox(remote.getCurrentWindow(), {
+        type: 'warning',
+        buttons: ['Edit Message', 'Cancel'],
+        message: 'Cannot Send',
+        detail: "Before sending, you need to drag the header cell of the column of emails to the To field in Recipients",
+      });
+    } else {
+      if (isValidDraft()) {
+        this.setState({sending: true})
+        try {
+          sendMailMerge(draft.clientId)
+        } catch (e) {
+          this.setState({sending: false})
+          NylasEnv.showErrorDialog(e.message)
+        }
+      }
+    }
+  }
+
+  primarySend() {
+    // Primary click is called when mod+enter is pressed.
+    // If mail merge is not open, we should revert to default behavior
+    const {isWorkspaceOpen} = this.props.sessionState
+    if (!isWorkspaceOpen && this.refs.fallbackButton) {
+      this.refs.fallbackButton.primarySend()
+    } else {
+      this.onClick()
+    }
+  }
+
+  render() {
+    const {sending} = this.state
+    const {isWorkspaceOpen, tableDataSource} = this.props.sessionState
+    if (!isWorkspaceOpen) {
+      const Fallback = this.props.fallback
+      return 
+    }
+
+    const count = tableDataSource.rows().length
+    const action = sending ? 'Sending' : 'Send'
+    const sendLabel = count > 1 ? `${action} ${count} messages` : `${action} ${count} message`;
+    let classes = "btn btn-toolbar btn-normal btn-emphasis btn-text btn-send"
+    if (sending) {
+      classes += " btn-disabled"
+    }
+    return (
+      
+    );
+  }
+}
+
+// TODO this is a hack so that the mail merge send button can still expose
+// the `primarySend` method required by the ComposerView. Ideally, this
+// decorator mechanism should expose whatever instance methods are exposed
+// by the component its wrapping.
+// However, I think the better fix will happen when mail merge lives in its
+// own window and doesn't need to override the Composer's send button, which
+// is already a bit of a hack.
+const EnhancedMailMergeSendButton = ListensToMailMergeSession(MailMergeSendButton)
+Object.assign(EnhancedMailMergeSendButton.prototype, {
+  primarySend() {
+    if (this.refs.composed) {
+      this.refs.composed.primarySend()
+    }
+  },
+})
+
+export default EnhancedMailMergeSendButton
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-subject-text-field.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-subject-text-field.jsx
new file mode 100644
index 000000000..2389187e9
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-subject-text-field.jsx
@@ -0,0 +1,132 @@
+/* eslint react/no-danger: 0 */
+import React, {Component, PropTypes} from 'react'
+import {findDOMNode} from 'react-dom'
+import {EditorAPI} from 'nylas-exports'
+import {OverlaidComponents, DropZone} from 'nylas-component-kit'
+import ListensToMailMergeSession from './listens-to-mail-merge-session'
+import * as Handlers from './mail-merge-token-dnd-handlers'
+
+
+class MailMergeSubjectTextField extends Component {
+  static displayName = 'MailMergeSubjectTextField'
+
+  static containerRequired = false
+
+  static propTypes = {
+    value: PropTypes.string,
+    fallback: PropTypes.func,
+    draft: PropTypes.object,
+    session: PropTypes.object,
+    sessionState: PropTypes.object,
+    draftClientId: PropTypes.string,
+    onSubjectChange: PropTypes.func.isRequired,
+  }
+
+  componentDidMount() {
+    const {isWorkspaceOpen} = this.props.sessionState
+
+    this.savedSelection = null
+    if (isWorkspaceOpen) {
+      this.editor = new EditorAPI(findDOMNode(this.refs.contenteditable))
+    }
+  }
+
+  shouldComponentUpdate(nextProps) {
+    return (
+      this.props.draftClientId !== nextProps.draftClientId ||
+      this.props.value !== nextProps.value ||
+      this.props.sessionState.isWorkspaceOpen !== nextProps.sessionState.isWorkspaceOpen
+    )
+  }
+
+  componentDidUpdate() {
+    const {isWorkspaceOpen} = this.props.sessionState
+
+    if (isWorkspaceOpen) {
+      this.editor = new EditorAPI(findDOMNode(this.refs.contenteditable))
+      if (this.savedSelection && this.savedSelection.rawSelection.anchorNode) {
+        this.editor.select(this.savedSelection)
+        this.savedSelection = null
+      }
+    }
+  }
+
+  onInputChange = (event) => {
+    const value = event.target.innerHTML
+    this.savedSelection = this.editor.currentSelection().exportSelection()
+    this.props.onSubjectChange(value)
+  }
+
+  onInputKeyDown = (event) => {
+    if (['Enter', 'Return'].includes(event.key)) {
+      event.stopPropagation()
+      event.preventDefault()
+    }
+  }
+
+  onDrop = (event) => {
+    Handlers.onDrop('subject', {editor: this.editor, event})
+  }
+
+  onDragOver = (event) => {
+    Handlers.onDragOver({editor: this.editor, event})
+  }
+
+  shouldAcceptDrop = (event) => {
+    return Handlers.shouldAcceptDrop({event})
+  }
+
+  focus() {
+    const {isWorkspaceOpen} = this.props.sessionState
+
+    if (isWorkspaceOpen) {
+      findDOMNode(this.refs.contenteditable).focus()
+    } else {
+      this.refs.fallback.focus()
+    }
+  }
+
+  renderContenteditable() {
+    const {value} = this.props
+    return (
+      
+        

+      
+    )
+  }
+
+  render() {
+    const {isWorkspaceOpen} = this.props.sessionState
+    if (!isWorkspaceOpen) {
+      const Fallback = this.props.fallback
+      return 
+    }
+
+    const {draft, session} = this.props
+    const exposedProps = {draft, session}
+    return (
+      
+        {this.renderContenteditable()}
+      
+    )
+  }
+}
+
+export default ListensToMailMergeSession(MailMergeSubjectTextField)
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-table.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-table.jsx
new file mode 100644
index 000000000..2647b705c
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-table.jsx
@@ -0,0 +1,43 @@
+import React, {PropTypes} from 'react'
+import {EditableTable} from 'nylas-component-kit'
+import {pickHTMLProps} from 'pick-react-known-prop'
+import MailMergeHeaderInput from './mail-merge-header-input'
+
+
+function InputRenderer(props) {
+  const {isHeader, draftClientId} = props;
+  if (!isHeader) {
+    return 
+  }
+  return 
+}
+InputRenderer.propTypes = {
+  isHeader: PropTypes.bool,
+  defaultValue: PropTypes.string,
+  draftClientId: PropTypes.string,
+}
+
+function MailMergeTable(props) {
+  const {draftClientId} = props
+  return (
+    

+      
+    

+  )
+}
+MailMergeTable.propTypes = {
+  tableDataSource: EditableTable.propTypes.tableDataSource,
+  selection: PropTypes.object,
+  draftClientId: PropTypes.string,
+  onShiftSelection: PropTypes.func,
+}
+
+export default MailMergeTable
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-token-dnd-handlers.es6 b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-token-dnd-handlers.es6
new file mode 100644
index 000000000..3900fc479
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-token-dnd-handlers.es6
@@ -0,0 +1,51 @@
+import {Utils} from 'nylas-exports'
+import {mailMergeSessionForDraft} from './mail-merge-draft-editing-session'
+import {DataTransferTypes, DragBehaviors} from './mail-merge-constants'
+
+
+function updateCursorPosition({editor, event}) {
+  const {clientX, clientY} = event
+  const range = document.caretRangeFromPoint(clientX, clientY);
+  range.collapse()
+  editor.select(range)
+  return range
+}
+
+export function shouldAcceptDrop({event}) {
+  const {dataTransfer} = event;
+  return !!dataTransfer.getData(DataTransferTypes.ColIdx);
+}
+
+export function onDragOver({editor, event}) {
+  updateCursorPosition({editor, event})
+}
+
+export function onDrop(field, {editor, event}) {
+  const {dataTransfer} = event
+  const colIdx = dataTransfer.getData(DataTransferTypes.ColIdx)
+  const colName = dataTransfer.getData(DataTransferTypes.ColName)
+  const dragBehavior = dataTransfer.getData(DataTransferTypes.DragBehavior)
+  const draftClientId = dataTransfer.getData(DataTransferTypes.DraftId)
+  const mailMergeSession = mailMergeSessionForDraft(draftClientId)
+  if (!mailMergeSession) {
+    return
+  }
+
+  if (dragBehavior === DragBehaviors.Move) {
+    const {tokenDataSource} = mailMergeSession.state
+    const {tokenId} = tokenDataSource.findTokens(field, {colName, colIdx}).pop() || {}
+    editor.removeCustomComponentByAnchorId(tokenId)
+  }
+
+  updateCursorPosition({editor, event})
+  const tokenId = Utils.generateTempId()
+  editor.insertCustomComponent('MailMergeBodyToken', {
+    field,
+    colIdx,
+    colName,
+    tokenId,
+    draftClientId,
+    anchorId: tokenId,
+    className: 'mail-merge-token-wrap',
+  })
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-token.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-token.jsx
new file mode 100644
index 000000000..43eb63615
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-token.jsx
@@ -0,0 +1,46 @@
+import React, {PropTypes} from 'react'
+import classnames from 'classnames'
+import {RetinaImg} from 'nylas-component-kit'
+import {DataTransferTypes, DragBehaviors} from './mail-merge-constants'
+
+
+function onDragStart(event, {draftClientId, colIdx, colName, dragBehavior}) {
+  const {dataTransfer} = event
+  dataTransfer.effectAllowed = 'move'
+  dataTransfer.setData(DataTransferTypes.DraftId, draftClientId)
+  dataTransfer.setData(DataTransferTypes.ColIdx, colIdx)
+  dataTransfer.setData(DataTransferTypes.ColName, colName)
+  dataTransfer.setData(DataTransferTypes.DragBehavior, dragBehavior)
+}
+
+function MailMergeToken(props) {
+  const {draftClientId, colIdx, colName, children, draggable, dragBehavior} = props
+  const classes = classnames({
+    'mail-merge-token': true,
+    [`token-color-${colIdx % 5}`]: true,
+  })
+  const _onDragStart = event => onDragStart(event, {draftClientId, colIdx, colName, dragBehavior})
+  const dragHandle = draggable ?  : null;
+
+  return (
+    
+      {dragHandle}
+      {children}
+    
+  )
+}
+MailMergeToken.propTypes = {
+  draftClientId: PropTypes.string,
+  colIdx: PropTypes.oneOfType([PropTypes.string, PropTypes.number]),
+  colName: PropTypes.string,
+  children: PropTypes.node,
+  draggable: PropTypes.bool,
+  dragBehavior: PropTypes.string,
+}
+
+MailMergeToken.defaultProps = {
+  draggable: false,
+  dragBehavior: DragBehaviors.Copy,
+}
+
+export default MailMergeToken
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-utils.es6 b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-utils.es6
new file mode 100644
index 000000000..208954f81
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-utils.es6
@@ -0,0 +1,204 @@
+import Papa from 'papaparse'
+import {
+  Utils,
+  Actions,
+  Contact,
+  RegExpUtils,
+  DraftHelpers,
+  DatabaseStore,
+  SoundRegistry,
+} from 'nylas-exports'
+
+import {PLUGIN_ID, MAX_ROWS, DataTransferTypes, ParticipantFields} from './mail-merge-constants'
+import {mailMergeSessionForDraft} from './mail-merge-draft-editing-session'
+import SendManyDraftsTask from './send-many-drafts-task'
+
+
+export function contactFromColIdx(colIdx, email) {
+  return new Contact({
+    name: email || '',
+    email: email || 'No value selected',
+    clientId: `${DataTransferTypes.ColIdx}:${colIdx}`,
+  })
+}
+
+export function colIdxFromContact(contact) {
+  const {clientId} = contact
+  if (!clientId.startsWith(DataTransferTypes.ColIdx)) {
+    return null
+  }
+  return contact.clientId.split(':')[2]
+}
+
+export function tokenQuerySelector(tokenId) {
+  if (!tokenId) {
+    return `img.mail-merge-token-wrap`
+  }
+  return `img.mail-merge-token-wrap[data-overlay-id="${tokenId}"]`
+}
+
+export function tokenRegex(tokenId) {
+  if (!tokenId) {
+    // https://regex101.com/r/sU7sO6/1
+    return /]*?class="[^>]*?mail-merge-token-wrap[^>]*?"[^>]*?>/gim
+  }
+  // https://regex101.com/r/fJ5eN6/5
+  const reStr = `]*?class="[^>]*?mail-merge-token-wrap[^>]*?" [^>]*?data-overlay-id="${tokenId}"[^>]*?>`
+  return new RegExp(reStr, 'gim')
+}
+
+function replaceContenteditableTokens(html, {field, tableDataSource, tokenDataSource, rowIdx}) {
+  const replaced = tokenDataSource.tokensForField(field)
+  .reduce((currentHtml, {colIdx, tokenId}) => {
+    const fieldValue = tableDataSource.cellAt({rowIdx, colIdx}) || ""
+    const markup = `${fieldValue}`
+    return currentHtml.replace(tokenRegex(tokenId), markup)
+  }, html)
+  if (tokenRegex().test(replaced)) {
+    throw new Error(`Field ${field} still contains tokens after attempting to replace for table values`)
+  }
+  return replaced
+}
+
+export function buildDraft(baseDraft, {tableDataSource, tokenDataSource, rowIdx}) {
+  if (tableDataSource.isEmpty({rowIdx})) {
+    return null
+  }
+  const draftToSend = baseDraft.clone()
+  draftToSend.clientId = Utils.generateTempId()
+
+  // Clear any previous mail merge metadata on the draft we are going to send
+  // and add rowIdx
+  draftToSend.applyPluginMetadata(PLUGIN_ID, {rowIdx})
+
+  // Replace tokens inside subject with values from table data
+  const draftSubject = replaceContenteditableTokens(draftToSend.subject, {
+    field: 'subject',
+    rowIdx,
+    tokenDataSource,
+    tableDataSource,
+  })
+  draftToSend.subject = Utils.extractTextFromHtml(draftSubject)
+
+  // Replace tokens inside body with values from table data
+  draftToSend.body = replaceContenteditableTokens(draftToSend.body, {
+    field: 'body',
+    rowIdx,
+    tokenDataSource,
+    tableDataSource,
+  })
+
+  // Update participant values
+  ParticipantFields.forEach((field) => {
+    draftToSend[field] = tokenDataSource.tokensForField(field).map(({colIdx}) => {
+      const column = tableDataSource.colAt(colIdx)
+      const value = (tableDataSource.cellAt({rowIdx, colIdx}) || "").trim()
+      const contact = new Contact({accountId: baseDraft.accountId, name: value, email: value})
+      if (!contact.isValid()) {
+        throw new Error(`Can't send messages:\nThe column ${column} contains an invalid email address at row ${rowIdx + 1}: "${value}"`)
+      }
+      return contact
+    })
+  })
+  return draftToSend
+}
+
+export function sendManyDrafts(mailMergeSession, recipientDrafts) {
+  const transformedDrafts = [];
+
+  return mailMergeSession.draftSession().ensureCorrectAccount({noSyncback: true})
+  .then(() => {
+    const baseDraft = mailMergeSession.draft();
+    return Promise.each(recipientDrafts, (recipientDraft) => {
+      recipientDraft.accountId = baseDraft.accountId;
+      recipientDraft.serverId = null;
+      return DraftHelpers.applyExtensionTransforms(recipientDraft).then((transformed) =>
+        transformedDrafts.push(transformed)
+      );
+    });
+  })
+  .then(() =>
+    DatabaseStore.inTransaction(t => t.persistModels(transformedDrafts))
+  )
+  .then(async () => {
+    const baseDraft = mailMergeSession.draft();
+
+    if (baseDraft.uploads.length > 0) {
+      recipientDrafts.forEach(async (d) => {
+        await DraftHelpers.removeStaleUploads(d);
+      })
+    }
+
+    const recipientClientIds = recipientDrafts.map(d => d.clientId)
+
+    Actions.queueTask(new SendManyDraftsTask(baseDraft.clientId, recipientClientIds))
+
+    if (NylasEnv.config.get("core.sending.sounds")) {
+      SoundRegistry.playSound('hit-send');
+    }
+    NylasEnv.close();
+  })
+}
+
+export function sendMailMerge(draftClientId) {
+  const mailMergeSession = mailMergeSessionForDraft(draftClientId)
+  if (!mailMergeSession) { return }
+
+  const baseDraft = mailMergeSession.draft()
+  const {tableDataSource, tokenDataSource} = mailMergeSession.state
+
+  const recipientDrafts = tableDataSource.rows()
+    .map((row, rowIdx) => (
+      buildDraft(baseDraft, {tableDataSource, tokenDataSource, rowIdx})
+    ))
+    .filter((draft) => draft != null)
+
+  if (recipientDrafts.length === 0) {
+    NylasEnv.showErrorDialog(`There are no drafts to send! Add add some data to the table below`)
+    return
+  }
+  sendManyDrafts(mailMergeSession, recipientDrafts)
+}
+
+export function parseCSV(file, maxRows = MAX_ROWS) {
+  return new Promise((resolve, reject) => {
+    Papa.parse(file, {
+      skipEmptyLines: true,
+      complete: ({data}) => {
+        if (data.length === 0) {
+          NylasEnv.showErrorDialog(
+            `The csv file you are trying to import contains no rows. Please select another file.`
+          );
+          resolve(null)
+          return;
+        }
+
+        // If a cell in the first row contains a valid email address, assume that
+        // the table has no headers. We need row[0] to be field names, so make some up!
+        const emailRegexp = RegExpUtils.emailRegex();
+        const emailInFirstRow = data[0].find((val) => emailRegexp.test(val));
+        if (emailInFirstRow) {
+          const headers = data[0].map((val, idx) => {
+            return emailInFirstRow === val ? 'Email Address' : `Column ${idx}`
+          })
+          data.unshift(headers);
+        }
+
+        const columns = data[0].slice()
+        const rows = data.slice(1)
+        if (rows.length > maxRows) {
+          NylasEnv.showErrorDialog(
+            `The csv file you are trying to import contains more than the max allowed number of rows (${maxRows}).\nWe have only imported the first ${maxRows} rows`
+          );
+          resolve({columns, rows: rows.slice(0, maxRows)})
+          return
+        }
+        resolve({columns, rows})
+      },
+      error: (error) => {
+        NylasEnv.showErrorDialog(`Sorry, we were unable to parse the file: ${file.name}\n${error.message}`);
+        reject(error)
+      },
+    })
+  })
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/mail-merge-workspace.jsx b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-workspace.jsx
new file mode 100644
index 000000000..38c85d59e
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/mail-merge-workspace.jsx
@@ -0,0 +1,159 @@
+import React, {Component, PropTypes} from 'react'
+import {RetinaImg, DropZone} from 'nylas-component-kit'
+import fs from 'fs';
+
+import {parseCSV} from './mail-merge-utils'
+import MailMergeTable from './mail-merge-table'
+
+
+class MailMergeWorkspace extends Component {
+  static displayName = 'MailMergeWorkspace'
+
+  static propTypes = {
+    isWorkspaceOpen: PropTypes.bool,
+    tableDataSource: MailMergeTable.propTypes.tableDataSource,
+    selection: PropTypes.object,
+    draftClientId: PropTypes.string,
+    session: PropTypes.object,
+  }
+
+  constructor() {
+    super()
+    this.state = {isDropping: false}
+  }
+
+  onDragStateChange = ({isDropping}) => {
+    this.setState({isDropping})
+  }
+
+  onChooseCSV = () => {
+    NylasEnv.showOpenDialog({
+      properties: ['openFile'],
+      filters: [
+        { name: 'CSV Files', extensions: ['csv', 'txt'] },
+      ],
+    }, (pathsToOpen) => {
+      if (!pathsToOpen || pathsToOpen.length === 0) {
+        return;
+      }
+
+      fs.readFile(pathsToOpen[0], (err, contents) => {
+        parseCSV(contents.toString()).then((tableData) => {
+          this.loadCSV(tableData)
+        });
+      });
+    });
+  }
+
+  onDropCSV = (event) => {
+    event.stopPropagation()
+    const {dataTransfer} = event
+    const file = dataTransfer.files[0]
+    parseCSV(file)
+    .then(tableData => this.loadCSV(tableData))
+  }
+
+  loadCSV(newTableData) {
+    const {tableDataSource, session} = this.props
+    // TODO We need to reset the table values first because `EditableTable` does
+    // not support controlled inputs, i.e. the inputs just use the
+    // defaultValue props which will only apply when the input is empty
+    session.clearTableData()
+    session.loadTableData({newTableData, prevColumns: tableDataSource.columns()})
+  }
+
+  shouldAcceptDrop = (event) => {
+    event.stopPropagation()
+    const {dataTransfer} = event
+    if (dataTransfer.files.length === 1) {
+      const file = dataTransfer.files[0]
+      if (file.type === 'text/csv') {
+        return true
+      }
+    }
+    return false
+  }
+
+  renderSelectionControls() {
+    const {selection, tableDataSource, session} = this.props
+    const rows = tableDataSource.rows()
+    return (
+      

+        

+          
 session.shiftSelection({row: -1})}
+          >
+            
+          

+          
 session.shiftSelection({row: 1})}
+          >
+            
+          

+        

+        Recipient {selection.rowIdx + 1} of {rows.length}
+        
+        

+          Import CSV
+        

+      

+    )
+  }
+
+  renderDropCover() {
+    const {isDropping} = this.state
+    const display = isDropping ? 'block' : 'none';
+    return (
+      

+        

+          Drop to Import CSV
+        

+      

+    )
+  }
+
+  render() {
+    const {session, draftClientId, isWorkspaceOpen, tableDataSource, selection, ...otherProps} = this.props
+    if (!isWorkspaceOpen) {
+      return false
+    }
+
+    return (
+      
+        
+        {this.renderDropCover()}
+        {this.renderSelectionControls()}
+        
+      
+    )
+  }
+}
+
+export default MailMergeWorkspace
diff --git a/packages/local-private/packages/composer-mail-merge/lib/main.es6 b/packages/local-private/packages/composer-mail-merge/lib/main.es6
new file mode 100644
index 000000000..8451d805d
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/main.es6
@@ -0,0 +1,53 @@
+import {
+  TaskRegistry,
+  ExtensionRegistry,
+  ComponentRegistry,
+  CustomContenteditableComponents,
+} from 'nylas-exports'
+
+import MailMergeButton from './mail-merge-button'
+import MailMergeContainer from './mail-merge-container'
+import SendManyDraftsTask from './send-many-drafts-task'
+import MailMergeSendButton from './mail-merge-send-button'
+import * as ComposerExtension from './mail-merge-composer-extension'
+import MailMergeSubjectTextField from './mail-merge-subject-text-field'
+import MailMergeBodyToken from './mail-merge-body-token'
+import MailMergeParticipantsTextField from './mail-merge-participants-text-field'
+
+export function activate() {
+  TaskRegistry.register('SendManyDraftsTask', () => SendManyDraftsTask)
+
+  ComponentRegistry.register(MailMergeContainer,
+    {role: 'Composer:ActionBarWorkspace'});
+
+  ComponentRegistry.register(MailMergeButton,
+    {role: 'Composer:ActionButton'});
+
+  ComponentRegistry.register(MailMergeSendButton,
+    {role: 'Composer:SendActionButton'});
+
+  ComponentRegistry.register(MailMergeParticipantsTextField,
+    {role: 'Composer:ParticipantsTextField'});
+
+  ComponentRegistry.register(MailMergeSubjectTextField,
+    {role: 'Composer:SubjectTextField'});
+
+  CustomContenteditableComponents.register('MailMergeBodyToken', MailMergeBodyToken)
+
+  ExtensionRegistry.Composer.register(ComposerExtension)
+}
+
+export function deactivate() {
+  TaskRegistry.unregister('SendManyDraftsTask')
+  ComponentRegistry.unregister(MailMergeContainer)
+  ComponentRegistry.unregister(MailMergeButton)
+  ComponentRegistry.unregister(MailMergeSendButton)
+  ComponentRegistry.unregister(MailMergeParticipantsTextField)
+  ComponentRegistry.unregister(MailMergeSubjectTextField)
+  CustomContenteditableComponents.unregister('MailMergeBodyToken');
+  ExtensionRegistry.Composer.unregister(ComposerExtension)
+}
+
+export function serialize() {
+
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/selection-state-reducers.es6 b/packages/local-private/packages/composer-mail-merge/lib/selection-state-reducers.es6
new file mode 100644
index 000000000..43834f554
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/selection-state-reducers.es6
@@ -0,0 +1,122 @@
+import _ from 'underscore'
+import {MAX_ROWS} from './mail-merge-constants'
+
+
+export function initialState(savedState) {
+  if (savedState && savedState.tableDataSource) {
+    return {
+      selection: {
+        rowIdx: 0,
+        colIdx: 0,
+        key: null,
+      },
+    }
+  }
+  return {
+    selection: {
+      rowIdx: 0,
+      colIdx: 0,
+      key: 'Enter',
+    },
+  }
+}
+
+export function clearTableData() {
+  return {
+    selection: {
+      rowIdx: 0,
+      colIdx: 0,
+      key: null,
+    },
+  }
+}
+
+export function loadTableData() {
+  return {
+    selection: {
+      rowIdx: 0,
+      colIdx: 0,
+      key: null,
+    },
+  }
+}
+
+export function addColumn({selection, tableDataSource}) {
+  const columns = tableDataSource.columns()
+  return {
+    selection: {
+      ...selection,
+      rowIdx: null,
+      colIdx: columns.length,
+      key: 'Enter',
+    },
+  }
+}
+
+export function removeLastColumn({selection, tableDataSource}) {
+  const columns = tableDataSource.columns()
+  const nextSelection = {...selection, key: null}
+  if (nextSelection.colIdx === columns.length - 1) {
+    nextSelection.colIdx--
+  }
+
+  return {selection: nextSelection}
+}
+
+export function addRow({selection, tableDataSource}, {maxRows = MAX_ROWS} = {}) {
+  const rows = tableDataSource.rows()
+  if (rows.length === maxRows) {
+    return {selection}
+  }
+
+  return {
+    selection: {
+      ...selection,
+      rowIdx: rows.length,
+      key: 'Enter',
+    },
+  }
+}
+
+export function removeRow({selection, tableDataSource}) {
+  const rows = tableDataSource.rows()
+  const nextSelection = {...selection, key: null}
+  if (nextSelection.rowIdx === rows.length - 1) {
+    nextSelection.rowIdx--
+  }
+
+  return {selection: nextSelection}
+}
+
+export function updateCell({selection}) {
+  return {
+    selection: {...selection, key: null},
+  }
+}
+
+export function setSelection({selection}, nextSelection) {
+  if (_.isEqual(selection, nextSelection)) {
+    return {selection}
+  }
+  return {
+    selection: {...nextSelection},
+  }
+}
+
+function shift(len, idx, delta = 0) {
+  const idxVal = idx != null ? idx : -1
+  return Math.min(len - 1, Math.max(0, idxVal + delta))
+}
+
+export function shiftSelection({tableDataSource, selection}, deltas) {
+  const rowLen = tableDataSource.rows().length
+  const colLen = tableDataSource.columns().length
+
+  const nextSelection = {
+    rowIdx: shift(rowLen, selection.rowIdx, deltas.row),
+    colIdx: shift(colLen, selection.colIdx, deltas.col),
+    key: deltas.key,
+  }
+
+  return setSelection({selection}, nextSelection)
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/send-many-drafts-task.es6 b/packages/local-private/packages/composer-mail-merge/lib/send-many-drafts-task.es6
new file mode 100644
index 000000000..fa0108084
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/send-many-drafts-task.es6
@@ -0,0 +1,171 @@
+import {
+  Task,
+  Actions,
+  Message,
+  TaskQueue,
+  DraftStore,
+  BaseDraftTask,
+  SendDraftTask,
+  SoundRegistry,
+  DatabaseStore,
+  TaskQueueStatusStore,
+} from 'nylas-exports'
+import {PLUGIN_ID} from './mail-merge-constants'
+
+
+const SEND_DRAFT_THROTTLE = 500
+
+export default class SendManyDraftsTask extends Task {
+
+  constructor(baseDraftClientId, draftIdsToSend = []) {
+    super()
+    this.baseDraftClientId = baseDraftClientId
+    this.draftIdsToSend = draftIdsToSend
+
+    this.queuedDraftIds = new Set()
+    this.failedDraftIds = []
+  }
+
+  label() {
+    return `Sending ${this.draftIdsToSend.length} messages...`
+  }
+
+  shouldDequeueOtherTask(other) {
+    return other instanceof SendManyDraftsTask && other.draftClientId === this.baseDraftClientId;
+  }
+
+  isDependentOnTask(other) {
+    const isSameDraft = other.draftClientId === this.baseDraftClientId;
+    const isSaveOrSend = other instanceof BaseDraftTask;
+    return isSameDraft && isSaveOrSend
+  }
+
+  performLocal() {
+    if (!this.baseDraftClientId) {
+      const errMsg = `Attempt to call SendManyDraftsTask.performLocal without a baseDraftClientId`;
+      return Promise.reject(new Error(errMsg));
+    }
+    if (this.draftIdsToSend.length === 0) {
+      const errMsg = `Attempt to call SendManyDraftsTask.performLocal without draftIdsToSend`;
+      return Promise.reject(new Error(errMsg));
+    }
+
+    return Promise.resolve();
+  }
+
+  performRemote() {
+    const unqueuedDraftIds = this.draftIdsToSend.filter(id => !this.queuedDraftIds.has(id))
+
+    if (unqueuedDraftIds.length > 0) {
+      return (
+        DatabaseStore.modelify(Message, unqueuedDraftIds)
+        .then((draftsToSend) => this.queueSendTasks(draftsToSend))
+        .then(() => this.waitForSendTasks())
+        .then(() => this.onTasksProcessed())
+        .catch((error) => this.handleError(error))
+      )
+    }
+    return (
+      this.waitForSendTasks()
+      .then(() => this.onTasksProcessed())
+      .catch((error) => this.handleError(error))
+    )
+  }
+
+  queueSendTasks(draftsToSend, throttle = SEND_DRAFT_THROTTLE) {
+    return Promise.each(draftsToSend, (draft) => {
+      return new Promise((resolve) => {
+        const task = new SendDraftTask(draft.clientId, {
+          playSound: false,
+          emitError: false,
+          allowMultiSend: false,
+        })
+        Actions.queueTask(task)
+        this.queuedDraftIds.add(draft.clientId)
+        setTimeout(resolve, throttle)
+      })
+    })
+  }
+
+  waitForSendTasks() {
+    const waitForTaskPromises = Array.from(this.queuedDraftIds).map((draftClientId) => {
+      const tasks = TaskQueue.allTasks()
+      const task = tasks.find((t) => t instanceof SendDraftTask && t.draftClientId === draftClientId)
+      if (!task) {
+        console.warn(`SendManyDraftsTask: Can't find queued SendDraftTask for draft id: ${draftClientId}`)
+        this.queuedDraftIds.delete(draftClientId)
+        return Promise.resolve()
+      }
+
+      return TaskQueueStatusStore.waitForPerformRemote(task)
+      .then((completedTask) => {
+        if (!this.queuedDraftIds.has(completedTask.draftClientId)) { return }
+
+        const {status} = completedTask.queueState
+        if (status === Task.Status.Failed) {
+          this.failedDraftIds.push(completedTask.draftClientId)
+        }
+
+        this.queuedDraftIds.delete(completedTask.draftClientId)
+      })
+    })
+    return Promise.all(waitForTaskPromises)
+  }
+
+  onTasksProcessed() {
+    if (this.failedDraftIds.length > 0) {
+      const error = new Error(
+        `Sorry, some of your messages failed to send.
+This could be due to sending limits imposed by your mail provider.
+Please try again after a while. Also make sure your messages are addressed correctly and are not too large.`,
+      )
+      return this.handleError(error)
+    }
+
+    Actions.recordUserEvent("Mail Merge Sent", {
+      numItems: this.draftIdsToSend.length,
+      numFailedItems: this.failedDraftIds.length,
+    })
+
+    if (NylasEnv.config.get("core.sending.sounds")) {
+      SoundRegistry.playSound('send');
+    }
+    return Promise.resolve(Task.Status.Success)
+  }
+
+  handleError(error) {
+    return (
+      DraftStore.sessionForClientId(this.baseDraftClientId)
+      .then((session) => {
+        return DatabaseStore.modelify(Message, this.failedDraftIds)
+        .then((failedDrafts) => {
+          const failedDraftRowIdxs = failedDrafts.map((draft) => draft.metadataForPluginId(PLUGIN_ID).rowIdx)
+          const currentMetadata = session.draft().metadataForPluginId(PLUGIN_ID)
+          const nextMetadata = {
+            ...currentMetadata,
+            failedDraftRowIdxs,
+          }
+          session.changes.addPluginMetadata(PLUGIN_ID, nextMetadata)
+          return session.changes.commit()
+        })
+      })
+      .then(() => {
+        this.failedDraftIds.forEach((id) => Actions.destroyDraft(id))
+        Actions.composePopoutDraft(this.baseDraftClientId, {errorMessage: error.message})
+        return Promise.resolve([Task.Status.Failed, error])
+      })
+    )
+  }
+
+  toJSON() {
+    const json = {...super.toJSON()}
+    json.queuedDraftIds = Array.from(json.queuedDraftIds)
+    return json
+  }
+
+  fromJSON(json) {
+    const result = super.fromJSON(json)
+    result.queuedDraftIds = new Set(result.queuedDraftIds)
+    return result
+  }
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/table-state-reducers.es6 b/packages/local-private/packages/composer-mail-merge/lib/table-state-reducers.es6
new file mode 100644
index 000000000..6d46aa51e
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/table-state-reducers.es6
@@ -0,0 +1,95 @@
+import {Table} from 'nylas-component-kit'
+import {MAX_ROWS} from './mail-merge-constants'
+
+const {TableDataSource} = Table
+
+
+export function toJSON({tableDataSource}) {
+  return {
+    tableDataSource: tableDataSource.toJSON(),
+  }
+}
+
+export function fromJSON({tableDataSource}) {
+  return {
+    tableDataSource: new TableDataSource(tableDataSource),
+  }
+}
+
+export function initialState(savedState) {
+  if (savedState && savedState.tableDataSource instanceof TableDataSource) {
+    if (savedState.failedDraftRowIdxs) {
+      const failedRowIdxs = new Set(savedState.failedDraftRowIdxs)
+      const dataSource = (
+        savedState.tableDataSource
+        .filterRows((row, idx) => failedRowIdxs.has(idx))
+      )
+      return {
+        tableDataSource: dataSource,
+      }
+    }
+    return {
+      tableDataSource: savedState.tableDataSource,
+    }
+  }
+  return {
+    tableDataSource: new TableDataSource({
+      columns: ['email'],
+      rows: [
+        [null],
+      ],
+    }),
+  }
+}
+
+export function clearTableData({tableDataSource}) {
+  return {
+    tableDataSource: tableDataSource.clear(),
+  }
+}
+
+export function loadTableData({tableDataSource}, {newTableData}) {
+  const newRows = newTableData.rows
+  const newCols = newTableData.columns
+  if (newRows.length === 0 || newCols.length === 0) {
+    return initialState()
+  }
+  return {
+    tableDataSource: new TableDataSource(newTableData),
+  }
+}
+
+export function addColumn({tableDataSource}) {
+  return {
+    tableDataSource: tableDataSource.addColumn(),
+  }
+}
+
+export function removeLastColumn({tableDataSource}) {
+  return {
+    tableDataSource: tableDataSource.removeLastColumn(),
+  }
+}
+
+export function addRow({tableDataSource}, {maxRows = MAX_ROWS} = {}) {
+  const rows = tableDataSource.rows()
+  if (rows.length === maxRows) {
+    return {tableDataSource}
+  }
+
+  return {
+    tableDataSource: tableDataSource.addRow(),
+  }
+}
+
+export function removeRow({tableDataSource}) {
+  return {
+    tableDataSource: tableDataSource.removeRow(),
+  }
+}
+
+export function updateCell({tableDataSource}, {rowIdx, colIdx, isHeader, value}) {
+  return {
+    tableDataSource: tableDataSource.updateCell({rowIdx, colIdx, isHeader, value}),
+  }
+}
diff --git a/packages/local-private/packages/composer-mail-merge/lib/token-data-source.es6 b/packages/local-private/packages/composer-mail-merge/lib/token-data-source.es6
new file mode 100644
index 000000000..f46eeed62
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/token-data-source.es6
@@ -0,0 +1,114 @@
+import _ from 'underscore'
+import {Utils} from 'nylas-exports'
+
+
+class FieldTokens {
+
+  constructor(field, tokens = {}) {
+    this._field = field
+    this._tokens = tokens
+  }
+
+  linkToken(colProps) {
+    const tokenId = colProps.tokenId ? colProps.tokenId : Utils.generateTempId()
+    return new FieldTokens(this._field, {
+      ...this._tokens,
+      [tokenId]: {...colProps, field: this._field, tokenId},
+    })
+  }
+
+  unlinkToken(tokenId) {
+    const nextTokens = {...this._tokens}
+    delete nextTokens[tokenId]
+    return new FieldTokens(this._field, nextTokens)
+  }
+
+  updateToken(tokenId, props) {
+    const token = this._tokens[tokenId]
+    return new FieldTokens(this._field, {
+      ...this._tokens,
+      [tokenId]: {...token, ...props},
+    })
+  }
+
+  tokens() {
+    return _.values(this._tokens)
+  }
+
+  findTokens(matcher) {
+    return _.where(this.tokens(), matcher)
+  }
+
+  getToken(tokenId) {
+    return this._tokens[tokenId]
+  }
+}
+
+class TokenDataSource {
+
+  static fromJSON(json) {
+    return json.reduce((dataSource, token) => {
+      const {field, ...props} = token
+      return dataSource.linkToken(field, props)
+    }, new TokenDataSource())
+  }
+
+  constructor(linkedTokensByField = {}) {
+    this._linkedTokensByField = linkedTokensByField
+  }
+
+  findTokens(field, matcher) {
+    if (!this._linkedTokensByField[field]) { return [] }
+    return this._linkedTokensByField[field].findTokens(matcher)
+  }
+
+  tokensForField(field) {
+    if (!this._linkedTokensByField[field]) { return [] }
+    return this._linkedTokensByField[field].tokens()
+  }
+
+  getToken(field, tokenId) {
+    if (!this._linkedTokensByField[field]) { return null }
+    return this._linkedTokensByField[field].getToken(tokenId)
+  }
+
+  linkToken(field, props) {
+    if (!this._linkedTokensByField[field]) {
+      this._linkedTokensByField[field] = new FieldTokens(field)
+    }
+
+    const current = this._linkedTokensByField[field]
+    return new TokenDataSource({
+      ...this._linkedTokensByField,
+      [field]: current.linkToken(props),
+    })
+  }
+
+  unlinkToken(field, tokenId) {
+    if (!this._linkedTokensByField[field]) { return this }
+
+    const current = this._linkedTokensByField[field]
+    return new TokenDataSource({
+      ...this._linkedTokensByField,
+      [field]: current.unlinkToken(tokenId),
+    })
+  }
+
+  updateToken(field, tokenId, props) {
+    if (!this._linkedTokensByField[field]) { return this }
+
+    const current = this._linkedTokensByField[field]
+    return new TokenDataSource({
+      ...this._linkedTokensByField,
+      [field]: current.updateToken(tokenId, props),
+    })
+  }
+
+  toJSON() {
+    return Object.keys(this._linkedTokensByField)
+    .map((field) => this._linkedTokensByField[field])
+    .reduce((prevTokens, dataSource) => prevTokens.concat(dataSource.tokens()), [])
+  }
+}
+
+export default TokenDataSource
diff --git a/packages/local-private/packages/composer-mail-merge/lib/token-state-reducers.es6 b/packages/local-private/packages/composer-mail-merge/lib/token-state-reducers.es6
new file mode 100644
index 000000000..b7226cadd
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/token-state-reducers.es6
@@ -0,0 +1,122 @@
+import {contactFromColIdx} from './mail-merge-utils'
+import TokenDataSource from './token-data-source'
+import {LinkableFields, ContenteditableFields, ParticipantFields} from './mail-merge-constants'
+
+
+export function toDraftChanges(draft, {tableDataSource, selection, tokenDataSource}) {
+  // Save the participant fields to fake Contacts
+  const participantChanges = {}
+  ParticipantFields.forEach((field) => (
+    participantChanges[field] = tokenDataSource.tokensForField(field).map(({colIdx}) => {
+      const selectionValue = tableDataSource.cellAt({rowIdx: selection.rowIdx, colIdx}) || ""
+      return contactFromColIdx(colIdx, selectionValue.trim())
+    })
+  ))
+
+  // Save the body and subject if they haven't been saved yet
+  // This is necessary because new tokens wont be saved to the contenteditable
+  // unless the user directly mutates the body or subject
+  const contenteditableChanges = {}
+  ContenteditableFields.forEach((field) => {
+    const node = document.querySelector(`.${field}-field [contenteditable]`)
+    if (node) {
+      const latestValue = node.innerHTML
+      if (draft[field] !== latestValue) {
+        contenteditableChanges[field] = latestValue
+      }
+    }
+  })
+
+  return {...participantChanges, ...contenteditableChanges}
+}
+
+export function toJSON({tokenDataSource}) {
+  return {tokenDataSource: tokenDataSource.toJSON()}
+}
+
+export function fromJSON({tokenDataSource}) {
+  return {tokenDataSource: TokenDataSource.fromJSON(tokenDataSource)}
+}
+
+export function initialState(savedData) {
+  if (savedData && savedData.tokenDataSource) {
+    return {
+      tokenDataSource: savedData.tokenDataSource,
+    }
+  }
+  const tokenDataSource = new TokenDataSource()
+  return { tokenDataSource }
+}
+
+export function loadTableData({tokenDataSource}, {newTableData}) {
+  const nextColumns = newTableData.columns
+  let nextTokenDataSource = new TokenDataSource()
+
+  // When loading table data, if the new table data contains columns with the same
+  // name, make sure to keep those tokens in our state with the updated position
+  // of the column
+  LinkableFields.forEach((field) => {
+    const currentTokens = tokenDataSource.tokensForField(field)
+    currentTokens.forEach((link) => {
+      const {colName, ...props} = link
+      const newColIdx = nextColumns.indexOf(colName)
+      if (newColIdx !== -1) {
+        nextTokenDataSource = nextTokenDataSource.linkToken(field, {
+          ...props,
+          colName,
+          colIdx: newColIdx,
+        })
+      }
+    })
+  })
+  return {tokenDataSource: nextTokenDataSource}
+}
+
+export function linkToDraft({tokenDataSource}, args) {
+  const {colIdx, colName, field, ...props} = args
+  if (!field) { throw new Error('MailMerge: Must provide `field` to `linkToDraft`') }
+  if (!colIdx) { throw new Error('MailMerge: Must provide `colIdx` to `linkToDraft`') }
+  if (colName == null) { throw new Error('MailMerge: Must provide `colName` to `linkToDraft`') }
+  return {
+    tokenDataSource: tokenDataSource.linkToken(field, {colIdx, colName, ...props}),
+  }
+}
+
+export function unlinkFromDraft({tokenDataSource}, {field, tokenId}) {
+  if (!field) { throw new Error('MailMerge: Must provide `field` to `linkToDraft`') }
+  if (!tokenId) { throw new Error('MailMerge: Must provide `tokenId` to `linkToDraft`') }
+  return {
+    tokenDataSource: tokenDataSource.unlinkToken(field, tokenId),
+  }
+}
+
+export function removeLastColumn({tokenDataSource, tableDataSource}) {
+  const colIdx = tableDataSource.columns().length - 1
+  const colName = tableDataSource.colAt(colIdx)
+  let nextTokenDataSource = tokenDataSource
+
+  // Unlink any fields that where linked to the column that is being removed
+  LinkableFields.forEach((field) => {
+    const tokensToRemove = tokenDataSource.findTokens(field, {colName})
+    nextTokenDataSource = tokensToRemove.reduce((prevTokenDataSource, {tokenId}) => {
+      return prevTokenDataSource.unlinkToken(field, tokenId)
+    }, nextTokenDataSource)
+  })
+  return {tokenDataSource: nextTokenDataSource}
+}
+
+export function updateCell({tokenDataSource, tableDataSource}, {colIdx, isHeader, value}) {
+  if (!isHeader) { return {tokenDataSource} }
+  const currentColName = tableDataSource.colAt(colIdx)
+  let nextTokenDataSource = tokenDataSource
+
+  // Update any tokens that referenced the column name that is being updated
+  LinkableFields.forEach((field) => {
+    const tokens = tokenDataSource.findTokens(field, {colName: currentColName})
+    tokens.forEach(({tokenId}) => {
+      nextTokenDataSource = nextTokenDataSource.updateToken(field, tokenId, {colName: value})
+    })
+  })
+  return {tokenDataSource: nextTokenDataSource}
+}
+
diff --git a/packages/local-private/packages/composer-mail-merge/lib/workspace-state-reducers.es6 b/packages/local-private/packages/composer-mail-merge/lib/workspace-state-reducers.es6
new file mode 100644
index 000000000..536d49144
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/lib/workspace-state-reducers.es6
@@ -0,0 +1,14 @@
+export function initialState(savedData) {
+  if (savedData && savedData.tokenDataSource && savedData.tableDataSource) {
+    return {
+      isWorkspaceOpen: true,
+    }
+  }
+  return {
+    isWorkspaceOpen: false,
+  }
+}
+
+export function toggleWorkspace({isWorkspaceOpen}) {
+  return {isWorkspaceOpen: !isWorkspaceOpen}
+}
diff --git a/packages/local-private/packages/composer-mail-merge/package.json b/packages/local-private/packages/composer-mail-merge/package.json
new file mode 100644
index 000000000..a0284d439
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/package.json
@@ -0,0 +1,23 @@
+{
+  "name": "composer-mail-merge",
+  "title":"Mail Merge",
+  "description": "Send personalized emails at scale using CSV-formatted data.",
+  "main": "./lib/main",
+  "version": "0.1.0",
+  "engines": {
+    "nylas": "*"
+  },
+  "icon": "./icon.png",
+  "isOptional": true,
+  "supportedEnvs": ["production", "staging"],
+  "windowTypes": {
+    "default": true,
+    "composer": true,
+    "work": true,
+    "thread-popout": true
+  },
+  "license": "GPL-3.0",
+  "dependencies": {
+    "papaparse": "^4.1.2"
+  }
+}
diff --git a/packages/local-private/packages/composer-mail-merge/spec/fixtures.es6 b/packages/local-private/packages/composer-mail-merge/spec/fixtures.es6
new file mode 100644
index 000000000..0a65f8857
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/fixtures.es6
@@ -0,0 +1,38 @@
+import {Table} from 'nylas-component-kit'
+import TokenDataSource from '../lib/token-data-source'
+
+const {TableDataSource} = Table
+
+export const testData = {
+  columns: ['name', 'email'],
+  rows: [
+    ['donald', 'donald@nylas.com'],
+    ['hilary', 'hilary@nylas.com'],
+  ],
+}
+
+export const testDataSource = new TableDataSource(testData)
+
+export const testSelection = {rowIdx: 1, colIdx: 0, key: 'Enter'}
+
+export const testTokenDataSource =
+  new TokenDataSource()
+  .linkToken('to', {colName: 'name', colIdx: 0, tokenId: 'name-0'})
+  .linkToken('bcc', {colName: 'email', colIdx: 1, tokenId: 'email-1'})
+
+export const testState = {
+  isWorkspaceOpen: true,
+  selection: testSelection,
+  tableDataSource: testDataSource,
+  tokenDataSource: testTokenDataSource,
+}
+
+export const testAnchorMarkup = (tokenId) => {
+  return ``
+}
+
+export const testContenteditableContent = () => {
+  const nameSpan = testAnchorMarkup('name-anchor')
+  const emailSpan = testAnchorMarkup('email-anchor')
+  return `
${nameSpan}
stuff${emailSpan}
`
+}
diff --git a/packages/local-private/packages/composer-mail-merge/spec/mail-merge-draft-editing-session-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/mail-merge-draft-editing-session-spec.es6
new file mode 100644
index 000000000..5a6b50a6c
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/mail-merge-draft-editing-session-spec.es6
@@ -0,0 +1,108 @@
+import {Message} from 'nylas-exports'
+import {MailMergeDraftEditingSession} from '../lib/mail-merge-draft-editing-session'
+
+
+const testReducers = [
+  {testAction: state => ({...state, val1: 'reducer1'})},
+  {testAction: state => ({...state, val2: 'reducer2'})},
+]
+const draftModel = new Message()
+const draftSess = {
+  draft() { return draftModel },
+}
+
+describe('MailMergeDraftEditingSession', function describeBlock() {
+  let mailMergeSess;
+  beforeEach(() => {
+    mailMergeSess = new MailMergeDraftEditingSession(draftSess, testReducers)
+  });
+
+  describe('dispatch', () => {
+    it('computes next state correctly based on registered reducers', () => {
+      const nextState = mailMergeSess.dispatch({name: 'testAction'}, {})
+      expect(nextState).toEqual({
+        val1: 'reducer1',
+        val2: 'reducer2',
+      })
+    });
+
+    it('computes state value for key correctly when 2 reducers ', () => {
+      const reducers = testReducers.concat([
+        {testAction: state => ({...state, val2: 'reducer3'})},
+      ])
+      mailMergeSess = new MailMergeDraftEditingSession(draftSess, reducers)
+
+      const nextState = mailMergeSess.dispatch({name: 'testAction'}, {})
+      expect(nextState).toEqual({
+        val1: 'reducer1',
+        val2: 'reducer3',
+      })
+    });
+
+    it('passes arguments correctly to reducers', () => {
+      const args = ['arg1']
+      const reducers = testReducers.concat([
+        {testAction: (state, arg) => ({...state, val3: arg})},
+      ])
+      mailMergeSess = new MailMergeDraftEditingSession(draftSess, reducers)
+
+      const nextState = mailMergeSess.dispatch({name: 'testAction', args}, {})
+      expect(nextState).toEqual({
+        val1: 'reducer1',
+        val2: 'reducer2',
+        val3: 'arg1',
+      })
+    });
+  });
+
+  describe('initializeState', () => {
+    it('loads any saved metadata on the draft', () => {
+      const savedMetadata = {
+        tableDataSource: {},
+        tokenDataSource: {},
+      }
+      const nextState = {next: 'state'}
+      spyOn(draftModel, 'metadataForPluginId').andReturn(savedMetadata)
+      spyOn(mailMergeSess, 'dispatch').andReturn(nextState)
+
+      mailMergeSess.initializeState(draftModel)
+      expect(mailMergeSess.dispatch.calls.length).toBe(2)
+      const args1 = mailMergeSess.dispatch.calls[0].args
+      const args2 = mailMergeSess.dispatch.calls[1].args
+
+      expect(args1).toEqual([{name: 'fromJSON'}, savedMetadata])
+      expect(args2).toEqual([{name: 'initialState'}, nextState])
+      expect(mailMergeSess._state).toEqual(nextState)
+    });
+
+    it('does not laod saved metadata if saved metadata is incorrect', () => {
+      const savedMetadata = {
+        tableDataSource: {},
+      }
+      const nextState = {next: 'state'}
+      spyOn(draftModel, 'metadataForPluginId').andReturn(savedMetadata)
+      spyOn(mailMergeSess, 'dispatch').andReturn(nextState)
+
+      mailMergeSess.initializeState(draftModel)
+      expect(mailMergeSess.dispatch.calls.length).toBe(1)
+      const {args} = mailMergeSess.dispatch.calls[0]
+
+      expect(args).toEqual([{name: 'initialState'}])
+      expect(mailMergeSess._state).toEqual(nextState)
+    });
+
+    it('just loads initial state if no metadata is saved on the draft', () => {
+      const savedMetadata = {}
+      const nextState = {next: 'state'}
+      spyOn(draftModel, 'metadataForPluginId').andReturn(savedMetadata)
+      spyOn(mailMergeSess, 'dispatch').andReturn(nextState)
+
+      mailMergeSess.initializeState(draftModel)
+      expect(mailMergeSess.dispatch.calls.length).toBe(1)
+      const {args} = mailMergeSess.dispatch.calls[0]
+
+      expect(args).toEqual([{name: 'initialState'}])
+      expect(mailMergeSess._state).toEqual(nextState)
+    });
+  });
+});
diff --git a/packages/local-private/packages/composer-mail-merge/spec/mail-merge-utils-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/mail-merge-utils-spec.es6
new file mode 100644
index 000000000..bd1c4621c
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/mail-merge-utils-spec.es6
@@ -0,0 +1,229 @@
+import Papa from 'papaparse'
+import {
+  Message,
+  Contact,
+  DraftHelpers,
+  Actions,
+  DatabaseTransaction,
+} from 'nylas-exports';
+
+import {DataTransferTypes} from '../lib/mail-merge-constants'
+import SendManyDraftsTask from '../lib/send-many-drafts-task'
+import {
+  parseCSV,
+  buildDraft,
+  sendManyDrafts,
+  contactFromColIdx,
+} from '../lib/mail-merge-utils'
+import {
+  testData,
+  testDataSource,
+  testAnchorMarkup,
+  testContenteditableContent,
+} from './fixtures'
+import TokenDataSource from '../lib/token-data-source'
+
+
+describe('MailMergeUtils', function describeBlock() {
+  describe('contactFromColIdx', () => {
+    it('creates a contact with the correct values', () => {
+      const email = 'email@email.com'
+      const contact = contactFromColIdx(0, email)
+      expect(contact instanceof Contact).toBe(true)
+      expect(contact.email).toBe(email)
+      expect(contact.name).toBe(email)
+      expect(contact.clientId).toBe(`${DataTransferTypes.ColIdx}:0`)
+    });
+  });
+
+  describe('buildDraft', () => {
+    beforeEach(() => {
+      this.baseDraft = new Message({
+        draft: true,
+        clientId: 'd1',
+        subject: `
Your email is: ${testAnchorMarkup('subject-email-anchor')}`,
+        body: testContenteditableContent(),
+      })
+
+      this.tokenDataSource = new TokenDataSource()
+      .linkToken('to', {colName: 'email', colIdx: 1, tokenId: 'email-0'})
+      .linkToken('bcc', {colName: 'email', colIdx: 1, tokenId: 'email-1'})
+      .linkToken('body', {colName: 'name', colIdx: 0, tokenId: 'name-anchor'})
+      .linkToken('body', {colName: 'email', colIdx: 1, tokenId: 'email-anchor'})
+      .linkToken('subject', {colName: 'email', colIdx: 1, tokenId: 'subject-email-anchor'})
+    });
+
+    it('creates a draft with the correct subject based on linked columns and rowIdx', () => {
+      const draft = buildDraft(this.baseDraft, {
+        rowIdx: 1,
+        tableDataSource: testDataSource,
+        tokenDataSource: this.tokenDataSource,
+      })
+      expect(draft.subject).toEqual('Your email is: hilary@nylas.com')
+    });
+
+    it('creates a draft with the correct body based on linked columns and rowIdx', () => {
+      const draft = buildDraft(this.baseDraft, {
+        rowIdx: 1,
+        tableDataSource: testDataSource,
+        tokenDataSource: this.tokenDataSource,
+      })
+      expect(draft.body).toEqual('
hilary
stuffhilary@nylas.com
')
+    });
+
+    it('creates a draft with the correct participants based on linked columns and rowIdx', () => {
+      const draft = buildDraft(this.baseDraft, {
+        rowIdx: 1,
+        tableDataSource: testDataSource,
+        tokenDataSource: this.tokenDataSource,
+      })
+      expect(draft.to[0].email).toEqual('hilary@nylas.com')
+      expect(draft.bcc[0].email).toEqual('hilary@nylas.com')
+    });
+
+    it('throws error if value for participant field in invalid email address', () => {
+      this.tokenDataSource = this.tokenDataSource.updateToken('to', 'email-0', {colName: 'name', colIdx: 0})
+      expect(() => {
+        buildDraft(this.baseDraft, {
+          rowIdx: 1,
+          tableDataSource: testDataSource,
+          tokenDataSource: this.tokenDataSource,
+        })
+      }).toThrow()
+    });
+  });
+
+  describe('sendManyDrafts', () => {
+    beforeEach(() => {
+      this.baseDraft = new Message({
+        draft: true,
+        accountId: '123',
+        serverId: '111',
+        clientId: 'local-111',
+      })
+      this.drafts = [
+        new Message({draft: true, clientId: 'local-d1'}),
+        new Message({draft: true, clientId: 'local-d2'}),
+        new Message({draft: true, clientId: 'local-d3'}),
+      ]
+      this.draftSession = {
+        ensureCorrectAccount: jasmine.createSpy('ensureCorrectAccount').andCallFake(() => {
+          return Promise.resolve()
+        }),
+      }
+      this.session = {
+        draftSession: () => this.draftSession,
+        draft: () => this.baseDraft,
+      }
+
+      spyOn(DraftHelpers, 'applyExtensionTransforms').andCallFake((d) => {
+        const transformed = d.clone()
+        transformed.body = 'transformed'
+        return Promise.resolve(transformed)
+      })
+      spyOn(DatabaseTransaction.prototype, 'persistModels').andReturn(Promise.resolve())
+      spyOn(Actions, 'queueTask')
+      spyOn(Actions, 'queueTasks')
+      spyOn(NylasEnv.config, 'get').andReturn(false)
+      spyOn(NylasEnv, 'close')
+    })
+
+    it('ensures account is correct', () => {
+      waitsForPromise(() => {
+        return sendManyDrafts(this.session, this.drafts)
+        .then(() => {
+          expect(this.draftSession.ensureCorrectAccount).toHaveBeenCalled()
+        })
+      })
+    });
+
+    it('applies extension transforms to each draft and saves them', () => {
+      waitsForPromise(() => {
+        return sendManyDrafts(this.session, this.drafts)
+        .then(() => {
+          const transformedDrafts = DatabaseTransaction.prototype.persistModels.calls[0].args[0]
+          expect(transformedDrafts.length).toBe(3)
+          transformedDrafts.forEach((d) => {
+            expect(d.body).toBe('transformed')
+            expect(d.accountId).toBe('123')
+            expect(d.serverId).toBe(null)
+          })
+        })
+      })
+    });
+
+    it('queues the correct task', () => {
+      waitsForPromise(() => {
+        return sendManyDrafts(this.session, this.drafts)
+        .then(() => {
+          const task = Actions.queueTask.calls[0].args[0]
+          expect(task instanceof SendManyDraftsTask).toBe(true)
+          expect(task.baseDraftClientId).toBe('local-111')
+          expect(task.draftIdsToSend).toEqual(['local-d1', 'local-d2', 'local-d3'])
+        })
+      })
+    });
+  });
+
+  describe('parseCSV', () => {
+    beforeEach(() => {
+      spyOn(NylasEnv, 'showErrorDialog')
+    });
+
+    it('shows error when csv file is empty', () => {
+      spyOn(Papa, 'parse').andCallFake((file, {complete}) => {
+        complete({data: []})
+      })
+      waitsForPromise(() => {
+        return parseCSV()
+        .then((data) => {
+          expect(NylasEnv.showErrorDialog).toHaveBeenCalled()
+          expect(data).toBe(null)
+        })
+      })
+    });
+
+    it('returns the correct table data', () => {
+      spyOn(Papa, 'parse').andCallFake((file, {complete}) => {
+        complete({data: [testData.columns].concat(testData.rows)})
+      })
+      waitsForPromise(() => {
+        return parseCSV()
+        .then((data) => {
+          expect(data).toEqual(testData)
+        })
+      })
+    });
+
+    it('adds a header row if the first row contains a value that resembles an email', () => {
+      spyOn(Papa, 'parse').andCallFake((file, {complete}) => {
+        complete({data: [...testData.rows]})
+      })
+      waitsForPromise(() => {
+        return parseCSV()
+        .then((data) => {
+          expect(data).toEqual({
+            columns: ['Column 0', 'Email Address'],
+            rows: testData.rows,
+          })
+        })
+      })
+    });
+
+    it('only imports MAX_ROWS number of rows', () => {
+      spyOn(Papa, 'parse').andCallFake((file, {complete}) => {
+        complete({
+          data: [testData.columns].concat([...testData.rows, ['extra', 'col@email.com']]),
+        })
+      })
+      waitsForPromise(() => {
+        return parseCSV(null, 2)
+        .then((data) => {
+          expect(data.rows.length).toBe(2)
+          expect(data).toEqual(testData)
+          expect(NylasEnv.showErrorDialog).toHaveBeenCalled()
+        })
+      })
+    });
+  });
+});
diff --git a/packages/local-private/packages/composer-mail-merge/spec/selection-state-reducers-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/selection-state-reducers-spec.es6
new file mode 100644
index 000000000..eaf02d1ce
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/selection-state-reducers-spec.es6
@@ -0,0 +1,182 @@
+import {
+  clearTableData,
+  loadTableData,
+  addColumn,
+  removeLastColumn,
+  addRow,
+  removeRow,
+  updateCell,
+  setSelection,
+  shiftSelection,
+} from '../lib/selection-state-reducers'
+import {testState, testSelection} from './fixtures'
+
+
+describe('SelectionStateReducers', function describeBlock() {
+  describe('clearTableData', () => {
+    it('sets selection correctly', () => {
+      const {selection} = clearTableData()
+      expect(selection).toEqual({
+        rowIdx: 0,
+        colIdx: 0,
+        key: null,
+      })
+    });
+  });
+
+  describe('loadTableData', () => {
+    it('sets selection correctly', () => {
+      const {selection} = loadTableData()
+      expect(selection).toEqual({
+        rowIdx: 0,
+        colIdx: 0,
+        key: null,
+      })
+    });
+  });
+
+  describe('addColumn', () => {
+    it('sets selection to the header and last column', () => {
+      const {selection} = addColumn(testState)
+      expect(selection).toEqual({rowIdx: null, colIdx: 2, key: 'Enter'})
+    });
+  });
+
+  describe('removeLastColumn', () => {
+    it('only sets key to null if selection is not in last column', () => {
+      const {selection} = removeLastColumn(testState)
+      expect(selection).toEqual({...testSelection, key: null})
+    });
+
+    it('decreases col selection by 1 if selection is currently in last column', () => {
+      const {selection} = removeLastColumn({...testState, selection: {rowIdx: 1, colIdx: 1, key: 'Enter'}})
+      expect(selection).toEqual({rowIdx: 1, colIdx: 0, key: null})
+    });
+  });
+
+  describe('addRow', () => {
+    it('does nothing if MAX_ROWS reached', () => {
+      const {selection} = addRow(testState, {maxRows: 2})
+      expect(selection).toBe(testSelection)
+    });
+
+    it('sets selection to last row', () => {
+      const {selection} = addRow(testState, {maxRows: 3})
+      expect(selection).toEqual({rowIdx: 2, colIdx: 0, key: 'Enter'})
+    });
+  });
+
+  describe('removeRow', () => {
+    it('only sets key to null if selection is not in last row', () => {
+      const {selection} = removeRow(testState)
+      expect(selection).toEqual({...testSelection, rowIdx: 0, key: null})
+    });
+
+    it('decreases row selection by 1 if selection is currently in last row', () => {
+      const {selection} = removeRow({...testState, selection: {rowIdx: 1, colIdx: 1, key: 'Enter'}})
+      expect(selection).toEqual({rowIdx: 0, colIdx: 1, key: null})
+    });
+  });
+
+  describe('updateCell', () => {
+    it('sets selection key to null (wont make input focus)', () => {
+      const {selection} = updateCell(testState)
+      expect(selection.key).toBe(null)
+    });
+  });
+
+  describe('setSelection', () => {
+    it('sets the selection to the given selection if selection has changed', () => {
+      const {selection} = setSelection(testState, {rowIdx: 1, colIdx: 1, key: null})
+      expect(selection).toEqual({rowIdx: 1, colIdx: 1, key: null})
+    });
+
+    it('returns same selection otherwise', () => {
+      const {selection} = setSelection(testState, {...testSelection})
+      expect(selection).toBe(testSelection)
+    });
+  });
+
+  describe('shiftSelection', () => {
+    it('sets the given key', () => {
+      const {selection} = shiftSelection(testState, {row: 0, col: 0, key: null})
+      expect(selection.key).toBe(null)
+    });
+
+    it('shifts row selection correctly when rowIdx is null (header)', () => {
+      let nextSelection = shiftSelection({
+        ...testState,
+        selection: {rowIdx: null, col: 0},
+      }, {row: 1}).selection
+      expect(nextSelection.rowIdx).toBe(0)
+
+      nextSelection = shiftSelection({
+        ...testState,
+        selection: {rowIdx: null, col: 0},
+      }, {row: 2}).selection
+      expect(nextSelection.rowIdx).toBe(1)
+
+      nextSelection = shiftSelection({
+        ...testState,
+        selection: {rowIdx: null, col: 0},
+      }, {row: -1}).selection
+      expect(nextSelection.rowIdx).toBe(0)
+    });
+
+    it('shifts row selection by correct value', () => {
+      let nextState = shiftSelection(
+        testState,
+        {row: -1}
+      )
+      expect(nextState.selection.rowIdx).toBe(0)
+
+      nextState = shiftSelection(
+        {...testState, selection: {rowIdx: 0, colIdx: 0, key: 'Enter'}},
+        {row: 1}
+      )
+      expect(nextState.selection.rowIdx).toBe(1)
+    });
+
+    it('does not shift row selection when at the edges', () => {
+      let nextState = shiftSelection(
+        testState,
+        {row: 2}
+      )
+      expect(nextState.selection.rowIdx).toBe(1)
+
+      nextState = shiftSelection(
+        {...testState, selection: {rowIdx: 0, colIdx: 0, key: 'Enter'}},
+        {row: -2}
+      )
+      expect(nextState.selection.rowIdx).toBe(0)
+    });
+
+    it('shifts col selection by correct value', () => {
+      let nextState = shiftSelection(
+        testState,
+        {col: 1}
+      )
+      expect(nextState.selection.colIdx).toBe(1)
+
+      nextState = shiftSelection(
+        {...testState, selection: {rowIdx: 0, colIdx: 1, key: 'Enter'}},
+        {col: -1}
+      )
+      expect(nextState.selection.colIdx).toBe(0)
+    });
+
+    it('does not shift col selection when at the edges', () => {
+      let nextState = shiftSelection(
+        testState,
+        {col: -2}
+      )
+      expect(nextState.selection.colIdx).toBe(0)
+
+      nextState = shiftSelection(
+        {...testState, selection: {rowIdx: 0, colIdx: 1, key: 'Enter'}},
+        {col: 2}
+      )
+      expect(nextState.selection.colIdx).toBe(1)
+    });
+  });
+});
diff --git a/packages/local-private/packages/composer-mail-merge/spec/send-many-drafts-task-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/send-many-drafts-task-spec.es6
new file mode 100644
index 000000000..69bc92a17
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/send-many-drafts-task-spec.es6
@@ -0,0 +1,251 @@
+import {
+  Task,
+  Actions,
+  Message,
+  TaskQueue,
+  DraftStore,
+  DatabaseStore,
+  SendDraftTask,
+  TaskQueueStatusStore,
+} from 'nylas-exports'
+import SendManyDraftsTask from '../lib/send-many-drafts-task'
+import {PLUGIN_ID} from '../lib/mail-merge-constants'
+
+
+describe('SendManyDraftsTask', function describeBlock() {
+  beforeEach(() => {
+    this.baseDraft = new Message({
+      clientId: 'baseId',
+      files: ['f1', 'f2'],
+      uploads: [],
+    })
+    this.d1 = new Message({
+      clientId: 'd1',
+      uploads: ['u1'],
+    })
+    this.d2 = new Message({
+      clientId: 'd2',
+    })
+
+    this.task = new SendManyDraftsTask('baseId', ['d1', 'd2'])
+
+    spyOn(DatabaseStore, 'modelify').andReturn(Promise.resolve([this.baseDraft, this.d1, this.d2]))
+    spyOn(DatabaseStore, 'inTransaction').andCallFake((cb) => {
+      return cb({persistModels() { return Promise.resolve() }})
+    })
+  });
+
+  describe('performRemote', () => {
+    beforeEach(() => {
+      spyOn(this.task, 'prepareDraftsToSend').andCallFake((baseId, draftIds) => {
+        return Promise.resolve(draftIds.map(id => this[id]))
+      })
+      spyOn(this.task, 'queueSendTasks').andReturn(Promise.resolve())
+      spyOn(this.task, 'waitForSendTasks').andReturn(Promise.resolve())
+      spyOn(this.task, 'onTasksProcessed')
+      spyOn(this.task, 'handleError').andCallFake((error) =>
+        Promise.resolve([Task.Status.Failed, error])
+      )
+    });
+
+    it('queues all drafts for sending when no tasks have been queued yet', () => {
+      waitsForPromise(() => {
+        return this.task.performRemote()
+        .then(() => {
+          expect(this.task.prepareDraftsToSend).toHaveBeenCalledWith('baseId', ['d1', 'd2'])
+          expect(this.task.queueSendTasks).toHaveBeenCalledWith([this.d1, this.d2])
+          expect(this.task.waitForSendTasks).toHaveBeenCalled()
+        })
+      })
+    });
+
+    it('only queues drafts that have not been queued for sending', () => {
+      this.task.queuedDraftIds = new Set(['d1'])
+      waitsForPromise(() => {
+        return this.task.performRemote()
+        .then(() => {
+          expect(this.task.prepareDraftsToSend).toHaveBeenCalledWith('baseId', ['d2'])
+          expect(this.task.queueSendTasks).toHaveBeenCalledWith([this.d2])
+          expect(this.task.waitForSendTasks).toHaveBeenCalled()
+        })
+      })
+    });
+
+    it('only waits for tasks to complete when all drafts have been queued for sending', () => {
+      this.task.queuedDraftIds = new Set(['d1', 'd2'])
+      waitsForPromise(() => {
+        return this.task.performRemote()
+        .then(() => {
+          expect(this.task.prepareDraftsToSend).not.toHaveBeenCalled()
+          expect(this.task.queueSendTasks).not.toHaveBeenCalled()
+          expect(this.task.waitForSendTasks).toHaveBeenCalled()
+        })
+      })
+    });
+
+    it('handles errors', () => {
+      jasmine.unspy(this.task, 'onTasksProcessed')
+      spyOn(this.task, 'onTasksProcessed').andReturn(Promise.reject(new Error('Oh no!')))
+      this.task.queuedDraftIds = new Set(['d1', 'd2'])
+      waitsForPromise(() => {
+        return this.task.performRemote()
+        .then(() => {
+          expect(this.task.handleError).toHaveBeenCalled()
+        })
+      })
+    });
+  });
+
+  describe('prepareDraftsToSend', () => {
+    it('updates the files and uploads on each draft to send', () => {
+      waitsForPromise(() => {
+        return this.task.prepareDraftsToSend('baseId', ['d1', 'd2'])
+        .then((draftsToSend) => {
+          expect(DatabaseStore.modelify).toHaveBeenCalledWith(Message, ['baseId', 'd1', 'd2'])
+          expect(draftsToSend.length).toBe(2)
+          expect(draftsToSend[0].files).toEqual(this.baseDraft.files)
+          expect(draftsToSend[0].uploads).toEqual([])
+          expect(draftsToSend[1].files).toEqual(this.baseDraft.files)
+          expect(draftsToSend[1].uploads).toEqual([])
+        })
+      })
+    });
+  });
+
+  describe('queueSendTasks', () => {
+    beforeEach(() => {
+      spyOn(Actions, 'queueTask')
+    });
+
+    it('queues SendDraftTask for all passed in drafts', () => {
+      waitsForPromise(() => {
+        const promise = this.task.queueSendTasks([this.d1, this.d2], 0)
+        advanceClock(1)
+        advanceClock(1)
+        return promise.then(() => {
+          expect(Actions.queueTask.calls.length).toBe(2)
+          expect(Array.from(this.task.queuedDraftIds)).toEqual(['d1', 'd2'])
+          Actions.queueTask.calls.forEach(({args}, idx) => {
+            const task = args[0]
+            expect(task instanceof SendDraftTask).toBe(true)
+            expect(task.draftClientId).toEqual(`d${idx + 1}`)
+          })
+        })
+      })
+    });
+  });
+
+  describe('waitForSendTasks', () => {
+    it('it updates queuedDraftIds and warns if there are no tasks matching the draft client id', () => {
+      this.task.queuedDraftIds = new Set(['d2'])
+      spyOn(TaskQueue, 'allTasks').andReturn([])
+      spyOn(console, 'warn')
+      waitsForPromise(() => {
+        return this.task.waitForSendTasks()
+        .then(() => {
+          expect(this.task.queuedDraftIds.size).toBe(0)
+          expect(console.warn).toHaveBeenCalled()
+        })
+      })
+    });
+
+    it('resolves when all queued tasks complete', () => {
+      this.task.queuedDraftIds = new Set(['d2'])
+      spyOn(TaskQueue, 'allTasks').andReturn([new SendDraftTask('d2')])
+      spyOn(TaskQueueStatusStore, 'waitForPerformRemote').andCallFake((task) => {
+        task.queueState.status = Task.Status.Success
+        return Promise.resolve(task)
+      })
+
+      waitsForPromise(() => {
+        return this.task.waitForSendTasks()
+        .then(() => {
+          expect(Array.from(this.task.queuedDraftIds)).toEqual([])
+          expect(this.task.failedDraftIds).toEqual([])
+        })
+      })
+    });
+
+    it('saves any draft ids of drafts that failed to send', () => {
+      this.task.queuedDraftIds = new Set(['d1', 'd2'])
+      spyOn(TaskQueue, 'allTasks').andReturn([new SendDraftTask('d1'), new SendDraftTask('d2')])
+      spyOn(TaskQueueStatusStore, 'waitForPerformRemote').andCallFake((task) => {
+        if (task.draftClientId === 'd1') {
+          task.queueState.status = Task.Status.Failed
+        } else {
+          task.queueState.status = Task.Status.Success
+        }
+        return Promise.resolve(task)
+      })
+
+      waitsForPromise(() => {
+        return this.task.waitForSendTasks()
+        .then(() => {
+          expect(Array.from(this.task.queuedDraftIds)).toEqual([])
+          expect(this.task.failedDraftIds).toEqual(['d1'])
+        })
+      })
+    });
+  });
+
+  describe('handleError', () => {
+    beforeEach(() => {
+      this.baseDraft.applyPluginMetadata(PLUGIN_ID, {tableDataSource: {}})
+      this.d1.applyPluginMetadata(PLUGIN_ID, {rowIdx: 0})
+      this.d2.applyPluginMetadata(PLUGIN_ID, {rowIdx: 1})
+      this.baseSession = {
+        draft: () => { return this.baseDraft },
+        changes: {
+          addPluginMetadata: jasmine.createSpy('addPluginMetadata'),
+          commit() { return Promise.resolve() },
+        },
+      }
+
+      this.task.failedDraftIds = ['d1', 'd2']
+      spyOn(Actions, 'destroyDraft')
+      spyOn(Actions, 'composePopoutDraft')
+      spyOn(DraftStore, 'sessionForClientId').andReturn(Promise.resolve(this.baseSession))
+
+      jasmine.unspy(DatabaseStore, 'modelify')
+      spyOn(DatabaseStore, 'modelify').andReturn(Promise.resolve([this.d1, this.d2]))
+    });
+
+    it('correctly saves the failed rowIdxs to the base draft metadata', () => {
+      waitsForPromise(() => {
+        return this.task.handleError({message: 'Error!'})
+        .then((status) => {
+          expect(status[0]).toBe(Task.Status.Failed)
+          expect(DatabaseStore.modelify).toHaveBeenCalledWith(Message, this.task.failedDraftIds)
+          expect(this.baseSession.changes.addPluginMetadata).toHaveBeenCalledWith(PLUGIN_ID, {
+            tableDataSource: {},
+            failedDraftRowIdxs: [0, 1],
+          })
+        })
+      })
+    });
+
+    it('correctly destroys failed drafts', () => {
+      waitsForPromise(() => {
+        return this.task.handleError({message: 'Error!'})
+        .then((status) => {
+          expect(status[0]).toBe(Task.Status.Failed)
+          expect(Actions.destroyDraft.calls.length).toBe(2)
+          expect(Actions.destroyDraft.calls[0].args).toEqual(['d1'])
+          expect(Actions.destroyDraft.calls[1].args).toEqual(['d2'])
+        })
+      })
+    });
+
+    it('correctly pops out base composer with error msg', () => {
+      waitsForPromise(() => {
+        return this.task.handleError({message: 'Error!'})
+        .then((status) => {
+          expect(status[0]).toBe(Task.Status.Failed)
+          expect(Actions.composePopoutDraft).toHaveBeenCalledWith('baseId', {
+            errorMessage: 'Error!',
+          })
+        })
+      })
+    });
+  });
+});
diff --git a/packages/local-private/packages/composer-mail-merge/spec/table-state-reducers-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/table-state-reducers-spec.es6
new file mode 100644
index 000000000..186727b99
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/table-state-reducers-spec.es6
@@ -0,0 +1,142 @@
+import {
+  initialState,
+  fromJSON,
+  toJSON,
+  clearTableData,
+  loadTableData,
+  addColumn,
+  removeLastColumn,
+  addRow,
+  removeRow,
+  updateCell,
+} from '../lib/table-state-reducers'
+import {testData, testDataSource} from './fixtures'
+
+
+describe('TableStateReducers', function describeBlock() {
+  describe('initialState', () => {
+    it('returns correct initial state when there is saved state', () => {
+      const savedState = {tableDataSource: testDataSource}
+      expect(initialState(savedState)).toEqual(savedState)
+    });
+
+    it('keeps only rowIdxs that failed if failedRowIdxs present in saved state', () => {
+      const savedState = {tableDataSource: testDataSource, failedDraftRowIdxs: [1]}
+      const {tableDataSource} = initialState(savedState)
+      expect(tableDataSource.rows()).toEqual([testDataSource.rowAt(1)])
+    });
+  });
+
+  describe('fromJSON', () => {
+    it('returns correct data source from json table data', () => {
+      const {tableDataSource} = fromJSON({tableDataSource: testData})
+      expect(tableDataSource.toJSON()).toEqual(testData)
+    });
+  });
+
+  describe('toJSON', () => {
+    it('returns correct json object from data source', () => {
+      const {tableDataSource} = toJSON({tableDataSource: testDataSource})
+      expect(tableDataSource).toEqual(testData)
+    });
+  });
+
+  describe('clearTableData', () => {
+    it('clears all data correcltly', () => {
+      const {tableDataSource} = clearTableData({tableDataSource: testDataSource})
+      expect(tableDataSource.toJSON()).toEqual({
+        columns: [],
+        rows: [[]],
+      })
+    });
+  });
+
+  describe('loadTableData', () => {
+    it('loads table data correctly', () => {
+      const newTableData = {
+        columns: ['my-col'],
+        rows: [['my-val']],
+      }
+      const {tableDataSource} = loadTableData({tableDataSource: testDataSource}, {newTableData})
+      expect(tableDataSource.toJSON()).toEqual(newTableData)
+    });
+
+    it('returns initial state if new table data is empty', () => {
+      const newTableData = {
+        columns: [],
+        rows: [[]],
+      }
+      const {tableDataSource} = loadTableData({tableDataSource: testDataSource}, {newTableData})
+      expect(tableDataSource.toJSON()).toEqual(initialState().tableDataSource.toJSON())
+    });
+  });
+
+  describe('addColumn', () => {
+    it('pushes a new column to the data source\'s columns', () => {
+      const {tableDataSource} = addColumn({tableDataSource: testDataSource})
+      expect(tableDataSource.columns()).toEqual(['name', 'email', null])
+    });
+
+    it('pushes a new column to every row', () => {
+      const {tableDataSource} = addColumn({tableDataSource: testDataSource})
+      expect(tableDataSource.rows()).toEqual([
+        ['donald', 'donald@nylas.com', null],
+        ['hilary', 'hilary@nylas.com', null],
+      ])
+    });
+  });
+
+  describe('removeLastColumn', () => {
+    it('removes last column from the data source\'s columns', () => {
+      const {tableDataSource} = removeLastColumn({tableDataSource: testDataSource})
+      expect(tableDataSource.columns()).toEqual(['name'])
+    });
+
+    it('removes last column from every row', () => {
+      const {tableDataSource} = removeLastColumn({tableDataSource: testDataSource})
+      expect(tableDataSource.rows()).toEqual([['donald'], ['hilary']])
+    });
+  });
+
+  describe('addRow', () => {
+    it('does nothing if MAX_ROWS reached', () => {
+      const {tableDataSource} = addRow({tableDataSource: testDataSource}, {maxRows: 2})
+      expect(tableDataSource).toBe(testDataSource)
+    });
+
+    it('pushes an empty row with correct number of columns', () => {
+      const {tableDataSource} = addRow({tableDataSource: testDataSource}, {maxRows: 3})
+      expect(tableDataSource.rows()).toEqual([
+        ['donald', 'donald@nylas.com'],
+        ['hilary', 'hilary@nylas.com'],
+        [null, null],
+      ])
+    });
+  });
+
+  describe('removeRow', () => {
+    it('removes last row', () => {
+      const {tableDataSource} = removeRow({tableDataSource: testDataSource})
+      expect(tableDataSource.rows()).toEqual([['donald', 'donald@nylas.com']])
+    });
+  });
+
+  describe('updateCell', () => {
+    it('updates cell value correctly when updating a cell that is /not/ a header', () => {
+      const {tableDataSource} = updateCell({tableDataSource: testDataSource}, {
+        rowIdx: 0, colIdx: 0, isHeader: false, value: 'new-val',
+      })
+      expect(tableDataSource.rows()).toEqual([
+        ['new-val', 'donald@nylas.com'],
+        ['hilary', 'hilary@nylas.com'],
+      ])
+    });
+
+    it('updates cell value correctly when updating a cell that /is/ a header', () => {
+      const {tableDataSource} = updateCell({tableDataSource: testDataSource}, {
+        rowIdx: null, colIdx: 0, isHeader: true, value: 'new-val',
+      })
+      expect(tableDataSource.columns()).toEqual(['new-val', 'email'])
+    });
+  });
+});
diff --git a/packages/local-private/packages/composer-mail-merge/spec/token-state-reducers-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/token-state-reducers-spec.es6
new file mode 100644
index 000000000..717bff2de
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/token-state-reducers-spec.es6
@@ -0,0 +1,155 @@
+import {Contact} from 'nylas-exports'
+import {
+  toDraftChanges,
+  toJSON,
+  initialState,
+  loadTableData,
+  linkToDraft,
+  unlinkFromDraft,
+  removeLastColumn,
+  updateCell,
+} from '../lib/token-state-reducers'
+import {testState, testTokenDataSource, testData} from './fixtures'
+
+
+describe('WorkspaceStateReducers', function describeBlock() {
+  describe('toDraftChanges', () => {
+    it('returns an object with participant fields populated with the correct Contact objects', () => {
+      const {to, bcc} = toDraftChanges({}, testState)
+      expect(to.length).toBe(1)
+      expect(bcc.length).toBe(1)
+
+      const toContact = to[0]
+      const bccContact = bcc[0]
+      expect(toContact instanceof Contact).toBe(true)
+      expect(toContact.email).toEqual('hilary')
+      expect(bccContact instanceof Contact).toEqual(true)
+      expect(bccContact.email).toEqual('hilary@nylas.com')
+    });
+  });
+
+  describe('toJSON', () => {
+    it('only saves linked fields to json', () => {
+      expect(toJSON(testState)).toEqual({
+        tokenDataSource: [
+          {field: 'to', colName: 'name', colIdx: 0, tokenId: 'name-0'},
+          {field: 'bcc', colName: 'email', colIdx: 1, tokenId: 'email-1'},
+        ],
+      })
+    });
+  });
+
+  describe('initialState', () => {
+    it('loads saved linked fields correctly when provided', () => {
+      expect(initialState({tokenDataSource: testTokenDataSource})).toEqual({
+        tokenDataSource: testTokenDataSource,
+      })
+    });
+  });
+
+  describe('loadTableData', () => {
+    describe('when newTableData contains columns that have already been linked in the prev tableData', () => {
+      it(`preserves the linked fields for the old columns that are still present
+         and update the index to the new value in newTableData`, () => {
+        const newTableData = {
+          columns: ['email', 'other'],
+          rows: [
+            ['donald@nylas.com', 'd'],
+            ['john@gmail.com', 'j'],
+          ],
+        }
+
+        const nextState = loadTableData(testState, {newTableData, prevColumns: testData.columns})
+        expect(nextState.tokenDataSource.toJSON()).toEqual([
+          {field: 'bcc', colName: 'email', colIdx: 0, tokenId: 'email-1'},
+        ])
+      });
+    });
+
+    describe('when newTableData only contains new columns', () => {
+      it('unlinks all fields that are no longer present ', () => {
+        const newTableData = {
+          columns: ['other1'],
+          rows: [
+            ['donald@nylas.com'],
+            ['john@gmail.com'],
+          ],
+        }
+
+        const nextState = loadTableData(testState, {newTableData, prevColumns: testData.columns})
+        expect(nextState.tokenDataSource.toJSON()).toEqual([])
+      });
+    });
+  });
+
+  describe('linkToDraft', () => {
+    it('adds the new field correctly to tokenDataSource state', () => {
+      const nextState = linkToDraft(testState, {
+        colIdx: 1,
+        colName: 'email',
+        field: 'body',
+        name: 'some',
+        tokenId: 'email-2',
+      })
+      expect(nextState.tokenDataSource.toJSON()).toEqual([
+        {field: 'to', colName: 'name', colIdx: 0, tokenId: 'name-0'},
+        {field: 'bcc', colName: 'email', colIdx: 1, tokenId: 'email-1'},
+        {field: 'body', colName: 'email', colIdx: 1, tokenId: 'email-2', name: 'some'},
+      ])
+
+      // Check that object ref is updated
+      expect(testTokenDataSource).not.toBe(nextState.tokenDataSource)
+    });
+
+    it('adds a new link if column has already been linked to that field', () => {
+      const nextState = linkToDraft(testState, {
+        colIdx: 1,
+        colName: 'email',
+        field: 'bcc',
+        name: 'some',
+        tokenId: 'email-2',
+      })
+      expect(nextState.tokenDataSource.toJSON()).toEqual([
+        {field: 'to', colName: 'name', colIdx: 0, tokenId: 'name-0'},
+        {field: 'bcc', colName: 'email', colIdx: 1, tokenId: 'email-1'},
+        {field: 'bcc', colName: 'email', colIdx: 1, tokenId: 'email-2', name: 'some'},
+      ])
+    });
+  });
+
+  describe('unlinkFromDraft', () => {
+    it('removes field correctly from tokenDataSource state', () => {
+      const nextState = unlinkFromDraft(testState, {field: 'bcc', tokenId: 'email-1'})
+      expect(nextState.tokenDataSource.toJSON()).toEqual([
+        {field: 'to', colName: 'name', colIdx: 0, tokenId: 'name-0'},
+      ])
+      // Check that object ref is updated
+      expect(testTokenDataSource).not.toBe(nextState.tokenDataSource)
+    });
+  });
+
+  describe('removeLastColumn', () => {
+    it('removes any tokenDataSource that were associated with the removed column', () => {
+      const nextState = removeLastColumn(testState)
+      expect(nextState.tokenDataSource.toJSON()).toEqual([
+        {field: 'to', colName: 'name', colIdx: 0, tokenId: 'name-0'},
+      ])
+    });
+  });
+
+  describe('updateCell', () => {
+    it('updates tokenDataSource when a column name (header cell) is updated', () => {
+      const nextState = updateCell(testState, {colIdx: 0, isHeader: true, value: 'nombre'})
+      expect(nextState.tokenDataSource.toJSON()).toEqual([
+        {field: 'to', colName: 'nombre', colIdx: 0, tokenId: 'name-0'},
+        {field: 'bcc', colName: 'email', colIdx: 1, tokenId: 'email-1'},
+      ])
+    });
+
+    it('does not update tokens state otherwise', () => {
+      const nextState = updateCell(testState, {colIdx: 0, isHeader: false, value: 'nombre'})
+      expect(nextState.tokenDataSource).toBe(testTokenDataSource)
+    });
+  });
+});
+
diff --git a/packages/local-private/packages/composer-mail-merge/spec/workspace-state-reducers-spec.es6 b/packages/local-private/packages/composer-mail-merge/spec/workspace-state-reducers-spec.es6
new file mode 100644
index 000000000..9df908595
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/spec/workspace-state-reducers-spec.es6
@@ -0,0 +1,29 @@
+import {
+  initialState,
+  toggleWorkspace,
+} from '../lib/workspace-state-reducers'
+import {testState} from './fixtures'
+
+
+describe('WorkspaceStateReducers', function describeBlock() {
+  describe('initialState', () => {
+    it('always opens the workspace if there is saved data', () => {
+      expect(initialState(testState)).toEqual({
+        isWorkspaceOpen: true,
+      })
+    });
+
+    it('defaults to closed', () => {
+      expect(initialState()).toEqual({
+        isWorkspaceOpen: false,
+      })
+    });
+  });
+
+  describe('toggleWorkspace', () => {
+    it('toggles workspace worrectly', () => {
+      expect(toggleWorkspace({isWorkspaceOpen: false})).toEqual({isWorkspaceOpen: true})
+      expect(toggleWorkspace({isWorkspaceOpen: true})).toEqual({isWorkspaceOpen: false})
+    });
+  });
+});
diff --git a/packages/local-private/packages/composer-mail-merge/stylesheets/mail-merge.less b/packages/local-private/packages/composer-mail-merge/stylesheets/mail-merge.less
new file mode 100644
index 000000000..39a15b749
--- /dev/null
+++ b/packages/local-private/packages/composer-mail-merge/stylesheets/mail-merge.less
@@ -0,0 +1,218 @@
+@import 'ui-variables';
+
+.mail-merge-workspace {
+  width: 100%;
+  height: 250px;
+  z-index: 1;
+  border-top: 1px solid lightgrey;
+  padding: @padding-large-vertical * 1.2 @padding-large-horizontal * 1.2;
+
+  .selection-controls {
+    display: flex;
+    align-items: center;
+    color: @text-color-very-subtle;
+    margin-bottom: @padding-small-horizontal;
+
+    .btn.btn-group {
+      padding: 0;
+      &:active {
+        background: initial;
+      }
+    }
+
+    .btn {
+      display: flex;
+      align-items: center;
+      height: 1.5em;
+      margin-right: 5px;
+      color: @text-color-very-subtle;
+
+      &:hover {
+        img {
+          background-color: @text-color-subtle;
+        }
+      }
+
+      .btn-prev,.btn-next {
+        height: 100%;
+        width: 15px;
+        img {
+          background-color: @text-color-very-subtle;
+        }
+        &:active {
+          background: darken(@btn-default-bg-color, 9%);
+        }
+      }
+      .btn-prev img {
+        transform: rotate(90deg) translate(-9px, -8px);
+      }
+      .btn-next img {
+        transform: rotate(-90deg) translate(9px, 8px);
+      }
+    }
+  }
+
+  .mail-merge-table {
+    height: 90%;
+
+    .editable-table-container {
+      width: 100%;
+      &>.key-commands-region {
+        width: initial;
+      }
+    }
+
+    .nylas-table.editable-table {
+      max-width: 700px;
+      font-size: 0.9em;
+
+      .table-row-header {
+        &.selected, th, th.selected {
+          background: initial;
+          border: 1px solid lighten(@border-color-secondary, 5%);
+        }
+      }
+
+      .table-row.table-row-header {
+        height: 35px;
+      }
+
+      .table-row {
+        height: 30px;
+      }
+
+      .numbered-cell {
+        width: 30px;
+      }
+
+      th.table-cell {
+        .mail-merge-token {
+          input {
+            cursor: -webkit-grab;
+          }
+        }
+      }
+      td.table-cell input {
+        cursor: default;
+      }
+      .table-cell:not(.numbered-cell) {
+        width: 120px;
+      }
+
+      .header-cell {
+        display: flex;
+        align-items: center;
+        height: 28px;
+        padding-left: 6px;
+      }
+    }
+  }
+}
+
+.generate-token-colors(@n, @i: 0) when (@i =< @n) {
+  @base-color: hsla(197 + (@i * 20), 58%, 95%, 1);
+  @text-color: darken(desaturate(@base-color, 28%), 49%);
+  @border-color: darken(@base-color, 8%);
+
+  .token-color-@{i}.mail-merge-token {
+    background-color: @base-color;
+    color: @text-color;
+    border-color: darken(@base-color, 8%);
+
+    img {
+      background-color: @text-color;
+    }
+  }
+  .mail-merge-participants-text-field {
+    .token.token-color-@{i} {
+      &.selected,&.dragging {
+        border-color: @text-color;
+      }
+    }
+  }
+  .generate-token-colors(@n, (@i + 1));
+}
+
+.generate-token-colors(4);
+
+.mail-merge-token {
+  border: 1px solid;
+  border-radius: 5px;
+  padding-left: 7px;
+  cursor: -webkit-grab;
+}
+
+.mail-merge-token-wrap,
+.n1-overlaid-component-anchor-container.mail-merge-token-wrap {
+  margin: 0 1px;
+  vertical-align: bottom;
+
+  .mail-merge-token {
+    padding: @padding-small-vertical * 0.5 @padding-small-horizontal * 0.5;
+    padding-right: 2px;
+    img {
+      margin-right: 10px;
+    }
+  }
+}
+
+.header-cell .mail-merge-token {
+  display: flex;
+  align-items: center;
+  height: 22px;
+
+  input {
+    max-width: 90%;
+    text-align: left;
+    height: 100%;
+    line-height: 100%;
+    color: inherit;
+    padding-bottom: 2px;
+  }
+}
+
+.mail-merge-participants-text-field {
+  .token {
+    box-shadow: none;
+    padding: 0;
+    border-radius: 5px;
+
+    .mail-merge-token {
+      display: flex;
+      align-items: center;
+      padding: 0 @padding-base-vertical;
+      font-size: 0.9em;
+      img {
+        margin-right: 10px;
+      }
+    }
+
+    &.selected, &.dragging {
+      background: none;
+      border-radius: 5px;
+    }
+  }
+}
+
+.mail-merge-subject-overlaid {
+  .toggle-preview {
+    top: 2px;
+    right: 22px;
+  }
+}
+
+.mail-merge-subject-text-field {
+  div[contenteditable] {
+    line-height: 21px;
+    font-weight: 400;
+    padding: 13px 0 9px 0;
+    min-width: 5em;
+    background-color: transparent;
+    border: none;
+    margin: 0;
+    &:empty:before {
+      content: attr(placeholder);
+      color: @text-color-very-subtle;
+    }
+  }
+}
diff --git a/packages/local-private/packages/composer-scheduler/README.md b/packages/local-private/packages/composer-scheduler/README.md
new file mode 100644
index 000000000..bb29b3912
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/README.md
@@ -0,0 +1,27 @@
+# QuickSchedule
+
+Say goodbye to the hassle of scheduling! This new plugin lets you avoid
+the typical back-and-forth of picking a time to meet. Just select a few
+options, and your recipient confirms with one click. It's the best way to
+instantly schedule meetings.
+
+This plugin works by adding a small "Schedule" button next to the Send
+button in the composer. Clicking the button will prompt the creation of a
+quick event creator.
+
+You can even select a set of proposed times. When you do this a calendar
+pops up with your availability. You can then select some proposed times
+for the receipient to choose from.
+
+#### Enable this plugin
+
+1. Download and run N1
+
+2. Navigate to Preferences > Plugins and click "Enable" beside the plugin.
+
+#### Who is this for?
+
+Anyone who makes a lot of appointments! If you are a developer, this is
+also a great example of a more complicated plugin that requires a backend
+service, and demonstrates how arbitrary JavaScript can be inserted to
+create custom functionality.
diff --git a/packages/local-private/packages/composer-scheduler/assets/ic-composer-scheduler@1x.png b/packages/local-private/packages/composer-scheduler/assets/ic-composer-scheduler@1x.png
new file mode 100644
index 0000000000000000000000000000000000000000..b716570ada860b9d10bf426730d8dcab6bba7bca
GIT binary patch
literal 1197
zcmbVMPiPcZ7@xS>hShDAwuoNxI*~LM=l{;GGtC;aGuiG^Hx293Zc&J{GjFp)cIHiI
zUb>T1X!Q`RhvGrnf?gLAunB@5L`y{wENZl+VDT8G#FHn*gOI-2WMdCON(W}%d*Azh
z-=E+2ee>Pu$j6EJ?l?tJi9%j4k@dak?S6xN=XSP!Aj>X1T)~G?4L1!RQbQK1LQrsw
zaae+eHTC6haDbv>p2xx^4~V=VbG*b!JR?eKl2?-=SbJ#V&9`c5NzbkM
zBBu;p$JkRjuGws|O^HSRI43BI!to*}iVV?Uf+-gpA>#(U9Sb@POyBmfja(2}G^%J4
zXJ``XY6^~5EUp`NgEgWEWn5@@oWSy&<3xG2wF6v&ufo_+J19?ikSoCeP5LION3FL5
zCfvQ=QKU$sQ9t!&7*n)h
zN`Xd7;$g?mMtDsZw4rQjSQnMFAPmcjB&YXF$y82~G(DNlcCZCEz=mtWj&GazUc>6I
z#j2VQ4UGIULXA!rjMfoGK^=KO)7n)B{YAsH-AK4MD%5J;I`r+Ypq2BH1KLch_6AY>
zHwISm*8gRYBSCRdyZoztIwI1O(RO_p$YFh)pi5@RC&MCMc`!o;?RG&QDu)lNFIQFu
zW0$+$i80iz*7E#ab|LdgzGw7qvGMn}eD9s>=TEEO0D0@Fn|b`;Bd~b>kKZrF`>vnd
zqx76QcjCoc+wXNPy}N0Ki7oYm&$cY>J$NmC{(
zJaZG%Q-e|yQz{EjrrIztFjr)TM3hAM`dB6B=jtVb)aX^@765fKFxc2v6eK2Rr0UTq__OB&@Hb09I0xZL0)vRD^GUf^&XRs)DJWnQpRynYn_wrJkXw
zxw(nCj)IYap{c%svA(f^u92~oiGh`gkpdJb0c|TvNwW%aaf8|g1^l#~=$>Fbx5
zm+O@q>*W`v>l<2HTIw4Z=^Gj80#)c1SLT%@R_NvxE5l51Ni9w;$}A|!%+FH*nV6WA
zUs__T1av9H3%LcpzHo!{ilG4o3^M(S!~%UoJp=vRT#&!Os*6j4QW4I;s=7F&vLIDI
zD784hv?v)EA8E=k&A_n3ZxKi#&^1>6MVY`zNz8G{PcF?(%`5SAu~h=f=%r+)SXm|-
z8>FTrTI!manHuVvn46^PS|leM=q4vxq?uY8CmR^17{JWIrr*)b$;I5z*vZAn(%jI{
z6{yA3+``zz%+1)?*~P@%38vRGuec;JFF6%vcP7wos9s0Bdaay`QWHz^i$e1Ab6}|;
zAS1sdzc?emK*8A=9P0`izKO}1c_2YeP((re?UGuQS(cjOR+OKs01j!ZOf0S-E(9R@
zQ^*ZLeW0WCLCFOv`M`vLX%fVQX9ge#o}E(jfO)70m|5!eLbVwf7!P~8IEGZ*O8WEv
zzy09=1|J@079Pp=O$)dUjEo+I%!!RV&9!KGnc07}>rP2$4os8rsYjW^g-T-sGQijH_H
zoMX58Ari=UcIAr8%t6N+B}8T|DEY$b8O7jL-y+p#yzIk?kIFL{gO2kYnG^8KTEN=(
zWrmfbfia__@|B70jTJ@${~abpGcJ9s!C{+|R=`lGCvcmo&{$x%(Bzpp3%)p+xHlb$
z&(H{F+Ax!GQ##+2T*r`)EG%mResPQK%v<3)g&`(wPx~dCBUU2vI~-bjoTo`H4ti60DW%Iz;*o_fIS*oQMdQ(8_K
zOGryd2Q7a3VudriRj<-=Pp6_O%Whs=As-UhFE{tVi+zksezdhEFhFe(*m~?Gt?O5CJ?f!vf4TU9PDOVOY%P_D=Fw9%kba!q0
R^a-GX)6><@Wt~$(699gTe~DWM4f
D;Eg-m

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-link@2x.png b/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-link@2x.png
new file mode 100644
index 0000000000000000000000000000000000000000..91f5ccdfad3c2d8cdd0174416afa538a95a993ed
GIT binary patch
literal 788
zcmeAS@N?(olHy`uVBq!ia0vp^@<1%d!3-oH>zJ+sQY`6?zK#qG8~eHcB(ehe3dtTp
zz6=aiY77hwEes65fII$-)Y%=<)JUYUP
ztjhe%;&O$Lqt*kpGbVYvyD)UH%6b4foCO|{#S9E$svykh8Km+7s9>6>i(`n!`O?WZ
zlbQ?!T-Dj~A|`Pya$V}-D6(b6mH+>Ds72^%?By4*IAbT!+v#iZS3B({LyM{8@jmW~
zeJodQOU$d{aPqU6!+249Tf@)%H>_8RIll2p3hRcy^SXU*gWMxermZT!E*xw$_j<@@
zAilx;nEskK!j=>J&ngrz=rD^`FiUrwqWtFJCI|J*S5b=W<>+-k=am=z
zyuKZ<`n-4X`$WEj<)S)|wtoD0;MD@|6BZkl_pziMk^ZsV+RoXsgp`#3@>*1V=kL@e1&+~rSoqsR-ZH#Tv{X^cqj|=B*aMhc&p}Ty?W7ZUp;1kET
zyfJHwJb!DV(V{Dd%`dx_F4ktAd@h#v+m&5Agj|KLR^H=yaN~7#tL#yZEvo19dBc8h
zGcT%eF6LI!~qPWBs#9cihTTzx1D~*!}lL
aZ-ac{4jH}o`#X++(x<1ZpUXO@geCySF7|E!

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-location@2x.png b/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-location@2x.png
new file mode 100644
index 0000000000000000000000000000000000000000..eef9572778f3e474cd32f162ae4e4a37c7567e36
GIT binary patch
literal 1260
zcmeAS@N?(olHy`uVBq!ia0vp^Qb4T2!3-o9Jh`O^q*&4&eH|GXHuiJ>Nn{1`6_P!I
zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4Fdz?=>VS)SD=EkqhRQV
z05BenjyE*{sp%y_e!&c42{U(`Ik>FMKz_-uvmT7qKY#Bl4Yc%1dAD7JbLHRpa%@Gf
z@9HpbeD5xK{FQQ@Cdb}$>dxhhXS=@T5&H4RyP=-xXvFTzN~!B6R0%w~)6)9jHt*97
z(Z{&Ia~*Ix;o`gHmg@PBOM~;)b{E=y@K0RhX>y6B_Dx02r+-o!8RDzi{Z@s3GrpGH
zaVVGng-rOK(}KF@U-OOP+n-3B=>j^DG0EHAg`tC0)&t1lEbxddW?&Fg1z|?dAe9e5
z1-Ct2978nDmj*wN7Y-DNQ7~a<5ej5p*wPZf({#z2QDw!JfFo`hi#*ghpJ*&x9K<4;
z_%HVT-s%HSnLa-(d-m+zy*KaPcd#s4lw7j0@FQpEl7q~(LM)Slue2$e^159<{KdV0
z{)@8?ZqIs?Pdb(PtrNIrBmL&5lLc2m#FAfbAA>EtMSk&|yR@PC^t{KdQzw4f@`lTI
zg^#sXlq{O(w$MfXu!VN#
zj0Z(RhrdV$&q+UU_LO+d)8N*L8B+ImTxI*rbMb`Evl*vc#V_u8abp8R#Ty6F@a_wO
zx`~l5Pd}gjT`;1sU}IP2PrraD;m04RuCrz_O$^*umZY52Hv1UcPS06R*)3Ds1+=5n
zbTVbGOKh`zd9d%;tycG&4j<=jk$?7suR!BP`5%K7Yv;e%##L3ahR5@PibmS;g07Q&
zw|VvlET}SQF+OejU3tT@P3L|u5I_FbYm&LnmG_Z9EPs9$*Pa!&r}q-;?UV=4zDDm*
zz4<%6`J9*FjR_0*oxQrv^>055N-Y)iO24CgN%WDKTT3DX!uh1v7Z<`LNxs@n5Cy&jMk&ia(C~nm2wHlzql*!&%@FSX)D3ZyNj?-YJ@}l%E571c0zYR<>HXgy>GU?;?BS+MxAH?btXZ_JWaopR
z>DucyJ`9^~{jN};+$X6&SWAz2&5QVFpT5@>?u)U_`_BG>W&aG5+4pM8_W)hS;OXk;
Jvd$@?2>?Ge_BsFn

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-people@2x.png b/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-people@2x.png
new file mode 100644
index 0000000000000000000000000000000000000000..b1a9b4ee6e4b9826f4e51c11ce3336addc3b9437
GIT binary patch
literal 1095
zcmeAS@N?(olHy`uVBq!ia0vp^azL!a!3-qp!)CexDVB6cUq=Rpjs4tz5?O(Kg=CK)
zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a!@$6}B)})c6{z6sC>X>c
z0F0nro|+s$s;eZ(FPMQz)!IVNXz_>VlNA}8|J*-%;>Fj*rcV{(%H6;JMd^MCd?U2B
z
zRY}$-AuYs4hI#qx8LYzF-k#YsbJ6O9kJDe4aXNNg+uCzW%6F;AH^b}$vo~g%K2m@B
zQ1Bg}cEYb^Kvyv)dAqwXbg;^L06Clm9+AZi3}UJv%;*`U@&TwI%hSa%#Nu@8#Psk(
z1`@4{nY6+R^b4A~Iy$P~GhW#%-ch|!Deq47`}Jm85gyUCb}HY$SuTI>>*T>=xUrB+
zb)uu_dF)>#s<-Bt^)EN;Bmn<3<7e4}~$
zf~~r9ZYRD@DDhF><+RbagX!*zmw)+c|4JBin@F}CJ}G$aXmUnGj^|U31evA$$BqkW
z?$}uP>yB`nr_$jmSygUsXKy`!Ca!VrsOt9i!qU@mE(uS=V^;4k%AH$x>&6|aJImg@
zdFMMLJrsuus(sW={FnGH9xvX%{
zoPFluS-*@gw@o?mko(2l@}_BbH7>F8eC7Y}zAz#=rR&VQp9i>9=HFwfczes)OlP{~
zb6_kmFeZ7syD)UH%6b4foCO|{#S9E$svykh8Km+7DEQCQ#W6(V{M`vJgAN&pxGfIL
zaAk2luy(;B1&u{JvX5#lc(iZc_j-}Vxu&^?xhv{Y_B}~?R#)5_ASrlQLe}`0aMe^3
zgNTsAtO+x?9pc_p)=sLkd0=qqS4Xn_+ybe>(sO$r+^`TV
z_dCulzGI{A!T*J!ddcS=oC*#ze^zB5w5qM+^8aZCcdtyGs4LN>oyQbowS5w!=>LcB
z@}0V;*u3|C^7yj6&tm;K?(^oEUX5FDBP~F{>eTuV-}Js*v9sHJ>MM6_iGyvh_NIBj
PkYMn1^>bP0l+XkK>*SzW

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-time@2x.png b/packages/local-private/packages/composer-scheduler/assets/ic-eventcard-time@2x.png
new file mode 100644
index 0000000000000000000000000000000000000000..140c6d616705d1f2d70bb8481e9c6ed6bb296fcc
GIT binary patch
literal 1358
zcmeAS@N?(olHy`uVBq!ia0vp^3P3E+!3-ofcG&L&QY`6?zK%d%Klh(RRv=#?*(1o8
zfuTx`fuW&=f#DZWsNn?zL#Y7+!>a@a2CEqi4B`cIb_LonFfhpk_=LCu6`UQ!U>yu#
zx)CO`4p|B!h%5qR6C;SG28|0b1a2&vEE*SXBTyWTjm(9}qL~RX3Rwy+f-DHvMLPyC
zVN7JzI0K|Ml?3?(Gw=lTZa;tb^1kUYqS_0-{N7?OFyrsPBXz#|N;+;u+kPn+-TOI3
zS>(OH)92G#66^nDn15Zw&N54%u{v7#$r7Rc7Ny^hvB^8mIm>LSV&KgEFlk+*-n>Fp(5z5K3EJZke)fCw)
z*(;c=JIyO+^1fj8d{e$_6_;X_pG!?hMSEY+$DFX6s{2E|t_7aF|Fq@C;m(yn*E1%0
zySp%Su*!M>Ih+L^k;M!QVyYm_=ozH)0jS`jr;B5V#`)5qpUNSD5`PytcGt$8_>_{n
zQ1F!S&P1K$_wB-SC|L{m6Iatlx{wA*=Zt>zRvwF?nKiiKpHoAGVH#Ir)wYRmppJ<3*qGvDE`eH_e-s7f6g7qOM
z`el!Ivc$SS@|-bght<@cDIrtm*-c(1q#8TvT;5a3nHI})KI)X+yE(0Qj=;J+
zmx*3)d3<$am1C?s^F`h}B5R`8_yuea^eABuJGkrgJI+-+Cws57q#evk{C<=>_b7AL
zVdsODM{db;Z@P1DZl`+F=P4We{Cvw4`*>c6u`T)fWm9_XInQ6cn|j#{R#YEg@LC*+(+Ng-CIAHkmG->3%(u*m(G_6V$?+C
zomEybp_d>mQ}{}+?Dfo~Ss|!nd_py+iw46$O%S%?XqaIabF#XctKma-%^w|~Y1F*W
zu9;`{P@ZrJSId_+irk#WS(R+#V%F-@6gzTlQ5pmYax$Zi1_L39j@mV`xHPbpVN#=x
zN#wx}mxv0ogRG>CcdNWBGNh75-7-IAWd)LX4qk}{A6gBB>XQbR~)qS}x&
zJ|3hmj*F96kq^tf5K=2~nQEaqU0-aUBPfNQAkR#0~E<96DIXKip=EXW0OAN
zIx!GinvW&902ky!vIP9}8U4bwf-Fe2!XPP=(}N}kHSu_oxihXe7|fVjlHK(HK>}$;
zPpPsY%#jtGB-Dv4=dK5rGCtGQ%e+5ToPLp%nL*diMfpM8l$p3&tQF!FvVm)*OGJ*5
z1+h{P0*-i>mG>c3mz2c1NC0z
z84lIh$kZUDO!gIBA_N2xEQ@m(ZDgil$Sc-sbXiD}nGnmBxg20X$MZhgNBJ#=YPLvg
z;;eqH$wC>mR?fm|S-;t6wwSGkB9k=+(c^*L2ZVMBY@M=S5}|$f7#eaP5PG)AgR_na
zKpJKB_nIO@m;tFuc?ooA1Jt>oe`BZF9BxVf(Y}cQA3K%8j3{x+p0jKEHeK=6vij|8D`~@geuYb3|&n(496^JXB0Ico0eO%MnK8
zoHd{)?ryioCx?p9x=hKHYad!WW)T_f5Gz7vg1yA(1X^81hoi-}uom2iz)fjzTlo
z$Tl*4INhV1k8%O8iwApHQaaOYO8BHtWOSTAm?JciaEuoV`cse9GCm4!8Nl5NnY>-;
zU*_ayxX(uFitck@I2=v_(?Hr77ZMRHz_?%XVQ5y1kC3#NgzF)kz`Sb%ZCG>|sNg+v4kFfNz|(#E)uh+qN61=B#<
z7#9)|EWo&68b}-CLL!0%7#Bj0>iL
zv@tFuB3OWN!8DLI#)U)#3otI22GYj3kceOb#s$+r+87rS5iG#CU>Zmp<3b{W1sE4h
z18HMiNJOvz6GWC!=5kg>PxYKCo{{a$SM~lobmf-So0_(^wRt^H
z6}a;r{(Sbi-KM@3-1VJPl|5=&zN=$r!|QpEoHg!kU$?Jm+~G4FH_Qi{
zPM7DqA3LCG{^5yH9jeVa1*#{TRUh`;oWeAPW6F@1LVfR-L&#xwpJ~
zS^muK*3J{H&a(}Vw&!J^-aG7wYWuLjUu8>sZ#=ZgdHM9s-D`$lm^Aci;QY2MVdI$N
zdF@Ak|FUzz+!sE`>gkv_vFB)Gq^0VUD^r`b!xrxTDSyYwYiqm2maefEb{wjDchA)G
zo4?<&U_Ud2yyR|w=9d=n&#&HkWp&e-f@bR>W?W97{s1#VTHiH2`{~~IW^Z`@;*SR(
zKR$NC2+4c>@VR<##I49iTq)k5h#I>#e`K4NeQcNgw;2xxPJHwJkkOZZ9=p@`tvBb)
zt1CPw*IoZ~*=x}}%gCi~FIw2Wr*rCxzu&pApnFZ{*IRd1zWl{8=JuN(XAv8nFRd9z
Sw{8JtBFfyeT-&Ert^5!F(pppi

literal 0
HcmV?d00001

diff --git a/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-calendar-data-source.es6 b/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-calendar-data-source.es6
new file mode 100644
index 000000000..bca5155a8
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-calendar-data-source.es6
@@ -0,0 +1,16 @@
+import Rx from 'rx-lite'
+import {CalendarDataSource} from 'nylas-exports'
+import ProposedTimeCalendarStore from '../proposed-time-calendar-store'
+
+export default class ProposedTimeCalendarDataSource extends CalendarDataSource {
+  buildObservable({startTime, endTime, disabledCalendars}) {
+    this.observable = Rx.Observable.combineLatest([
+      super.buildObservable({startTime, endTime, disabledCalendars}),
+      Rx.Observable.fromStore(ProposedTimeCalendarStore).map((store) => store.proposalsAsEvents()),
+    ])
+    .map(([superResult, proposedTimes]) => {
+      return {events: superResult.events.concat(proposedTimes)}
+    })
+    return this.observable;
+  }
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-event.jsx b/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-event.jsx
new file mode 100644
index 000000000..92eb77c7f
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-event.jsx
@@ -0,0 +1,51 @@
+import React from 'react'
+import classnames from 'classnames'
+import SchedulerActions from '../scheduler-actions'
+import {CALENDAR_ID} from '../scheduler-constants'
+
+/**
+ * Gets rendered in a CalendarEvent
+ */
+export default class ProposedTimeEvent extends React.Component {
+  static displayName = "ProposedTimeEvent";
+
+  static propTypes = {
+    event: React.PropTypes.object,
+  }
+
+  // Since ProposedTimeEvent is part of an Injected Component set, by
+  // default it's placed in its own container that's rendered separately.
+  //
+  // This makes two separate React trees which cause the react event
+  // propagations to be separate. See:
+  // https://github.com/facebook/react/issues/1691
+  //
+  // Unfortunately, this means that `stopPropagation` doesn't work from
+  // within injected component sets unless the `containerRequired` is set
+  // to `false`
+  static containerRequired = false;
+
+  _onMouseDown(event) {
+    event.stopPropagation();
+    SchedulerActions.removeProposedTime(event.target.dataset);
+  }
+
+  render() {
+    const className = classnames({
+      "rm-time": true,
+      "proposal": this.props.event.proposalType === "proposal",
+      "availability": this.props.event.proposalType === "availability",
+    });
+    if (this.props.event.calendarId === CALENDAR_ID) {
+      return (
+        
×

+      )
+    }
+    return false
+  }
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-picker.jsx b/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-picker.jsx
new file mode 100644
index 000000000..a5430cfa1
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/calendar/proposed-time-picker.jsx
@@ -0,0 +1,170 @@
+import React from 'react'
+import {Utils} from 'nylas-exports'
+import {NylasCalendar} from 'nylas-component-kit'
+import SchedulerActions from '../scheduler-actions'
+import ProposedTimeCalendarStore from '../proposed-time-calendar-store'
+import ProposedTimeCalendarDataSource from './proposed-time-calendar-data-source'
+
+/**
+ * A an extended NylasCalendar that lets you pick proposed times.
+ */
+export default class ProposedTimePicker extends React.Component {
+  static displayName = "ProposedTimePicker";
+
+  static containerStyles = {
+    height: "100%",
+  }
+
+  constructor(props) {
+    super(props);
+    this.state = {
+      proposals: ProposedTimeCalendarStore.proposals(),
+      duration: ProposedTimeCalendarStore.currentDuration(),
+      pendingSave: ProposedTimeCalendarStore.pendingSave(),
+    }
+  }
+
+  componentDidMount() {
+    this._usub = ProposedTimeCalendarStore.listen(() => {
+      this.setState({
+        duration: ProposedTimeCalendarStore.currentDuration(),
+        proposals: ProposedTimeCalendarStore.proposals(),
+        pendingSave: ProposedTimeCalendarStore.pendingSave(),
+      });
+    })
+    NylasEnv.displayWindow()
+  }
+
+  shouldComponentUpdate(nextProps, nextState) {
+    return (!Utils.isEqualReact(nextProps, this.props) ||
+            !Utils.isEqualReact(nextState, this.state));
+  }
+
+  componentWillUnmount() {
+    this._usub()
+  }
+
+  _dataSource() {
+    return new ProposedTimeCalendarDataSource()
+  }
+
+  _bannerComponents = () => {
+    return {
+      week: "Click and drag to propose times.",
+    }
+  }
+
+  _footerComponents = () => {
+    return {
+      week: [this._leftFooterComponents(), this._rightFooterComponents()],
+    }
+  }
+
+  _renderClearButton() {
+    if (this.state.proposals.length === 0) {
+      return false
+    }
+    return (
+      
+    )
+  }
+
+  _onClearProposals = () => {
+    SchedulerActions.clearProposals()
+  }
+
+  _leftFooterComponents() {
+    const optComponents = ProposedTimeCalendarStore.DURATIONS.map((opt, i) =>
+      
+    )
+
+    const durationPicker = (
+      

+        
+        
+      

+    )
+
+    return ([durationPicker, this._renderClearButton()]);
+  }
+
+  _rightFooterComponents() {
+    return (
+      
+    );
+  }
+
+  _onChangeDuration = (event) => {
+    SchedulerActions.changeDuration(event.target.value.split("|"))
+  }
+
+  _onDone = () => {
+    const proposals = ProposedTimeCalendarStore.proposals();
+    // NOTE: This gets dispatched to the main window
+    const {draftClientId} = NylasEnv.getWindowProps()
+    SchedulerActions.confirmChoices({proposals, draftClientId});
+    // Make sure the action gets to the main window then close this one.
+    setTimeout(() => { NylasEnv.close() }, 10)
+  }
+
+  _onCalendarMouseUp({time, currentView}) {
+    if (currentView !== NylasCalendar.WEEK_VIEW) { return }
+    if (time) {
+      SchedulerActions.addToProposedTimeBlock(time);
+    }
+    SchedulerActions.endProposedTimeBlock();
+    return
+  }
+
+  _onCalendarMouseMove({time, mouseIsDown, currentView}) {
+    if (!time || !mouseIsDown || currentView !== NylasCalendar.WEEK_VIEW) { return }
+    SchedulerActions.addToProposedTimeBlock(time);
+    return
+  }
+
+  _onCalendarMouseDown({time, currentView}) {
+    if (!time || currentView !== NylasCalendar.WEEK_VIEW) { return }
+    SchedulerActions.startProposedTimeBlock(time);
+    return
+  }
+
+  render() {
+    return (
+      
+    )
+  }
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/composer/email-b64-images.es6 b/packages/local-private/packages/composer-scheduler/lib/composer/email-b64-images.es6
new file mode 100644
index 000000000..bb510b1a4
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/composer/email-b64-images.es6
@@ -0,0 +1,7 @@
+export const b64Images = {
+  location: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMDY3IDc5LjE1Nzc0NywgMjAxNS8wMy8zMC0yMzo0MDo0MiAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6OTVFNzg0RkNFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6OTVFNzg0RkRFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3MkU3QjA3RkVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3MkU3QjA4MEVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PkDmWKcAAAFpSURBVHjahNNPKERRFMfxNzNK+RMrFhYSpWzobbHQrFBiIwspkdEsRBaUHSsbUhbIgkgaWVkqlCSpiVmJmqJsiTJIGt+r36vrzZuc+tR999175tx73oSy2axjIplMOopqjKEZHwjDLNrFOr7MItd1fxcXOH+jBfu4wzLOUYJOTKMDA3j2NtgJarGHHUz4El9jBYfYRB8yjsrzYgppa3M5hjGEMjyhBw2IBlVgDrWocUS/2K7zt6n0B5yhFwf+Cl5xo3EcrahBvRLE9O4eFd4mO8EnSq1xSCJa9653xXgLOoIZN+IEa2rnpZKY9m1oXaXuKidBWr1f0vMMErqDlOYK1a1E0BFM/5tQ5WtfynqO65hXQQku8IgFJ38MYlvdyEnwgll0oTtgs7mHImzZk2HfomPMYxV11nxMH9SIqsybwMQcTnGk56g+qklrzgnqghff6FfJGfV/3OrOvwkc/Y1H1dpbXVxg/AgwALwFT41l/QPcAAAAAElFTkSuQmCC`,
+
+  description: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMDY3IDc5LjE1Nzc0NywgMjAxNS8wMy8zMC0yMzo0MDo0MiAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzJFN0IwN0RFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzJFN0IwN0VFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3MkU3QjA3QkVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3MkU3QjA3Q0VENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PpVUSsoAAABkSURBVHjaYvz//z8DJYBx1AAGliNHjsQAaW4g/kWiXk4gfsECJAKBWASIv5JogCAQXwYZEAXyChDj8wsjDrF/IAN+UhQGQIwvDBihcjuB+BYuA/CFARNU7hYuA0ZTIhUMAAgwABS7Llh6+cz5AAAAAElFTkSuQmCC`,
+
+  time: `data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMDY3IDc5LjE1Nzc0NywgMjAxNS8wMy8zMC0yMzo0MDo0MiAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzJFN0IwNzlFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzJFN0IwN0FFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3MkU3QjA3N0VENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3MkU3QjA3OEVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PmHXWPIAAAIoSURBVHjaXJM7iFNBFIbn3oRkQ3ZJIEUIKrgiaLEoFoLFilupIMq2iqBgI1jYKCJooQg+EGwtBbFRRNBKrES2UAgICoKgEbFIookkhJt34vcP58plB75M7syZ/zzmTFCtVt2mcRguBkGwDIvz+XwGTdY24Ba0ksZh4n8Ab+B5GIYRh29Op9M1Dq+nUqkXrB+CH3A6KZBOCH2BDAf3MNeazabr9XpLiH0vl8sf8/n87clkcpa9x7AF7iYjeAmLsJzJZGqdTse1Wi2H9wvD4fBovV7Xf0ckj7BZgzs2e4FVOAYrGMira7fbDiGH9wVmHE+cIvJ5BsFbpnvwJBa4Dk/hbyyAV3nT/pI3CkPX7XYdNZGAlq5YxKsS2Km8ZDQajVwURS6dTvuQGQXz6lEkJqDxHk5JIAeftTKbzdx4PI6N9LMNhvqW90aj4YVt/ydsj4s4jz2Z51igBzdgr6Lq9/s+FUtvHtdgALt1UEbFYtGHqgZi/Ty8gmuIrysK1UfpMrbCr9BCOSkBpcCdu1Kp5OuB0G8OPZAQ37VsNusKhYJPh7EfngW0slr3tdViIHWlol5QuBIdDAZOhyuVisvlchK/is0l7EuBvYV3di374looHXkSuhk60efO9womn+CE0ouLeMTa84PuXunoNjQrItXFbuK4Hb5vtfnfypH1gwr3DR7CQQR2CMTOMG9Yy6uJLm9+TBpdOGCv7Zy16oLt/THPu+Br8jX+E2AAonoDF+eSc+MAAAAASUVORK5CYII=`,
+};
diff --git a/packages/local-private/packages/composer-scheduler/lib/composer/email-images.json b/packages/local-private/packages/composer-scheduler/lib/composer/email-images.json
new file mode 100644
index 000000000..fe846cda9
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/composer/email-images.json
@@ -0,0 +1,5 @@
+{
+  "location": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMDY3IDc5LjE1Nzc0NywgMjAxNS8wMy8zMC0yMzo0MDo0MiAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6OTVFNzg0RkNFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6OTVFNzg0RkRFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3MkU3QjA3RkVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3MkU3QjA4MEVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PkDmWKcAAAFpSURBVHjahNNPKERRFMfxNzNK+RMrFhYSpWzobbHQrFBiIwspkdEsRBaUHSsbUhbIgkgaWVkqlCSpiVmJmqJsiTJIGt+r36vrzZuc+tR999175tx73oSy2axjIplMOopqjKEZHwjDLNrFOr7MItd1fxcXOH+jBfu4wzLOUYJOTKMDA3j2NtgJarGHHUz4El9jBYfYRB8yjsrzYgppa3M5hjGEMjyhBw2IBlVgDrWocUS/2K7zt6n0B5yhFwf+Cl5xo3EcrahBvRLE9O4eFd4mO8EnSq1xSCJa9653xXgLOoIZN+IEa2rnpZKY9m1oXaXuKidBWr1f0vMMErqDlOYK1a1E0BFM/5tQ5WtfynqO65hXQQku8IgFJ38MYlvdyEnwgll0oTtgs7mHImzZk2HfomPMYxV11nxMH9SIqsybwMQcTnGk56g+qklrzgnqghff6FfJGfV/3OrOvwkc/Y1H1dpbXVxg/AgwALwFT41l/QPcAAAAAElFTkSuQmCC",
+  "description": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMDY3IDc5LjE1Nzc0NywgMjAxNS8wMy8zMC0yMzo0MDo0MiAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzJFN0IwN0RFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzJFN0IwN0VFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3MkU3QjA3QkVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3MkU3QjA3Q0VENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PpVUSsoAAABkSURBVHjaYvz//z8DJYBx1AAGliNHjsQAaW4g/kWiXk4gfsECJAKBWASIv5JogCAQXwYZEAXyChDj8wsjDrF/IAN+UhQGQIwvDBihcjuB+BYuA/CFARNU7hYuA0ZTIhUMAAgwABS7Llh6+cz5AAAAAElFTkSuQmCC",
+  "time": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyhpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNi1jMDY3IDc5LjE1Nzc0NywgMjAxNS8wMy8zMC0yMzo0MDo0MiAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIDIwMTUgKE1hY2ludG9zaCkiIHhtcE1NOkluc3RhbmNlSUQ9InhtcC5paWQ6NzJFN0IwNzlFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiIHhtcE1NOkRvY3VtZW50SUQ9InhtcC5kaWQ6NzJFN0IwN0FFRDYzMTFFNTlEOUREQkE1NjlFNkRFRjMiPiA8eG1wTU06RGVyaXZlZEZyb20gc3RSZWY6aW5zdGFuY2VJRD0ieG1wLmlpZDo3MkU3QjA3N0VENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIgc3RSZWY6ZG9jdW1lbnRJRD0ieG1wLmRpZDo3MkU3QjA3OEVENjMxMUU1OUQ5RERCQTU2OUU2REVGMyIvPiA8L3JkZjpEZXNjcmlwdGlvbj4gPC9yZGY6UkRGPiA8L3g6eG1wbWV0YT4gPD94cGFja2V0IGVuZD0iciI/PmHXWPIAAAIoSURBVHjaXJM7iFNBFIbn3oRkQ3ZJIEUIKrgiaLEoFoLFilupIMq2iqBgI1jYKCJooQg+EGwtBbFRRNBKrES2UAgICoKgEbFIookkhJt34vcP58plB75M7syZ/zzmTFCtVt2mcRguBkGwDIvz+XwGTdY24Ba0ksZh4n8Ab+B5GIYRh29Op9M1Dq+nUqkXrB+CH3A6KZBOCH2BDAf3MNeazabr9XpLiH0vl8sf8/n87clkcpa9x7AF7iYjeAmLsJzJZGqdTse1Wi2H9wvD4fBovV7Xf0ckj7BZgzs2e4FVOAYrGMira7fbDiGH9wVmHE+cIvJ5BsFbpnvwJBa4Dk/hbyyAV3nT/pI3CkPX7XYdNZGAlq5YxKsS2Km8ZDQajVwURS6dTvuQGQXz6lEkJqDxHk5JIAeftTKbzdx4PI6N9LMNhvqW90aj4YVt/ydsj4s4jz2Z51igBzdgr6Lq9/s+FUtvHtdgALt1UEbFYtGHqgZi/Ty8gmuIrysK1UfpMrbCr9BCOSkBpcCdu1Kp5OuB0G8OPZAQ37VsNusKhYJPh7EfngW0slr3tdViIHWlol5QuBIdDAZOhyuVisvlchK/is0l7EuBvYV3di374looHXkSuhk60efO9womn+CE0ouLeMTa84PuXunoNjQrItXFbuK4Hb5vtfnfypH1gwr3DR7CQQR2CMTOMG9Yy6uJLm9+TBpdOGCv7Zy16oLt/THPu+Br8jX+E2AAonoDF+eSc+MAAAAASUVORK5CYII="
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/composer/event-datetime-input.jsx b/packages/local-private/packages/composer-scheduler/lib/composer/event-datetime-input.jsx
new file mode 100644
index 000000000..a113edfa1
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/composer/event-datetime-input.jsx
@@ -0,0 +1,68 @@
+import React from 'react';
+import moment from 'moment-timezone'
+import {DateUtils} from 'nylas-exports'
+
+function getDateFormat(type) {
+  if (type === "date") {
+    return "YYYY-MM-DD";
+  } else if (type === "time") {
+    return "HH:mm:ss"
+  }
+  return null
+}
+
+export default class EventDatetimeInput extends React.Component {
+  static displayName = "EventDatetimeInput";
+
+  static propTypes = {
+    name: React.PropTypes.string,
+    value: React.PropTypes.number.isRequired,
+    onChange: React.PropTypes.func.isRequired,
+    reversed: React.PropTypes.bool,
+  };
+
+  constructor(props) {
+    super(props);
+    this._datePartStrings = {time: "", date: ""};
+  }
+
+  _onDateChange() {
+    const {date, time} = this._datePartStrings;
+    const format = `${getDateFormat("date")} ${getDateFormat("time")}`;
+    const newDate = moment.tz(`${date} ${time}`, format, DateUtils.timeZone).unix();
+    this.props.onChange(newDate)
+  }
+
+  _renderInput(type) {
+    const unixDate = this.props.value;
+    const str = moment.unix(unixDate).tz(DateUtils.timeZone).format(getDateFormat(type))
+    this._datePartStrings[type] = unixDate != null ? str : null;
+    return (
+       {
+          this._datePartStrings[type] = e.target.value;
+          this._onDateChange()
+        }}
+      />
+    )
+  }
+
+  render() {
+    if (this.props.reversed) {
+      return (
+        
+          {this._renderInput("time")} on {this._renderInput("date")}
+        
+      )
+    }
+    return (
+      
+        {this._renderInput("date")} at {this._renderInput("time")}
+      
+    )
+  }
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/composer/event-prep-helper.es6 b/packages/local-private/packages/composer-scheduler/lib/composer/event-prep-helper.es6
new file mode 100644
index 000000000..44230d84e
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/composer/event-prep-helper.es6
@@ -0,0 +1,20 @@
+export const prepareEvent = (inEvent, draft, proposals = []) => {
+  const event = inEvent
+  if (!event.title) {
+    event.title = "";
+  }
+
+  event.participants = draft.participants().map((contact) => {
+    return {
+      name: contact.name,
+      email: contact.email,
+      status: "noreply",
+    }
+  })
+
+  if (proposals.length > 0) {
+    event.end = null
+    event.start = null
+  }
+  return event;
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/composer/new-event-card-container.jsx b/packages/local-private/packages/composer-scheduler/lib/composer/new-event-card-container.jsx
new file mode 100644
index 000000000..902bfccf3
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/composer/new-event-card-container.jsx
@@ -0,0 +1,136 @@
+import React, {Component, PropTypes} from 'react';
+import {Utils, Event} from 'nylas-exports';
+
+import SchedulerActions from '../scheduler-actions'
+import NewEventCard from './new-event-card'
+import NewEventPreview from './new-event-preview'
+import {PLUGIN_ID} from '../scheduler-constants'
+import NewEventHelper from './new-event-helper'
+import RemoveEventHelper from './remove-event-helper'
+
+/**
+ * When you're creating an event you can either be creating:
+ *
+ * 1. A Meeting Request with a specific start and end time
+ * 2. OR a `pendingEvent` template that has a set of proposed times.
+ *
+ * Both are represented by a `pendingEvent` object on the `metadata` that
+ * holds the JSONified representation of the `Event`
+ *
+ * #2 adds a set of `proposals` on the metadata object.
+ *
+ * This component is an OverlayedComponent.
+ *
+ * The SchedulerComposerExtension::_insertNewEventCard will call
+ * `EditorAPI::insert`. We pass `insert` a React element. Under the hood,
+ * the `` wrapper will actually place an "anchor" tag
+ * and absolutely position our element over that anchor tag.
+ *
+ * This component is also decorated with the `InflatesDraftClientId`
+ * decorator. The former is necessary for OverlaidComponents to work. The
+ * latter provides us with up-to-date `draft` and `session` props by
+ * inflating a `draftClientId`.
+ *
+ * If the Anchor is deleted, or cut, then the ``
+ * element will unmount the `NewEventCardContainer`.
+ *
+ * If the anchor re-appears (via paste or some other mechanism), then this
+ * component will be re-mounted.
+ *
+ * We use the mounting and unmounting of this component as signals to add or
+ * remove the metadata on the draft.
+ */
+export default class NewEventCardContainer extends Component {
+  static displayName = 'NewEventCardContainer';
+
+  static propTypes = {
+    draft: PropTypes.object.isRequired,
+    session: PropTypes.object.isRequired,
+    style: PropTypes.object,
+    isPreview: PropTypes.bool,
+  }
+
+  componentDidMount() {
+    this._unlisten = SchedulerActions.confirmChoices.listen(::this._onConfirmChoices);
+    NewEventHelper.restoreOrCreateEvent(this.props.session)
+  }
+
+  componentWillUnmount() {
+    if (this._unlisten) {
+      this._unlisten();
+    }
+    RemoveEventHelper.hideEventData(this.props.session)
+  }
+
+  _onConfirmChoices({proposals = [], draftClientId}) {
+    const {draft} = this.props;
+
+    if (draft.clientId !== draftClientId) {
+      return;
+    }
+
+    const metadata = draft.metadataForPluginId(PLUGIN_ID) || {};
+    if (proposals.length === 0) {
+      delete metadata.proposals;
+    } else {
+      metadata.proposals = proposals;
+    }
+    this.props.session.changes.addPluginMetadata(PLUGIN_ID, metadata);
+  }
+
+  _getEvent() {
+    const metadata = this.props.draft.metadataForPluginId(PLUGIN_ID);
+    if (metadata && metadata.pendingEvent) {
+      return new Event().fromJSON(metadata.pendingEvent || {});
+    }
+    return null
+  }
+
+  _updateEvent = (newData) => {
+    const {draft, session} = this.props;
+
+    const newEvent = Object.assign(this._getEvent().clone(), newData);
+    const newEventJSON = newEvent.toJSON();
+
+    const metadata = draft.metadataForPluginId(PLUGIN_ID);
+    if (!Utils.isEqual(metadata.pendingEvent, newEventJSON)) {
+      metadata.pendingEvent = newEventJSON;
+      session.changes.addPluginMetadata(PLUGIN_ID, metadata);
+    }
+  }
+
+  _removeEvent = () => {
+    // This will delete the metadata, but it won't remove the anchor from
+    // the contenteditable. We also need to remove the event card.
+    RemoveEventHelper.deleteEventData(this.props.session);
+    SchedulerActions.removeEventCard();
+  }
+
+  render() {
+    const {style, isPreview} = this.props;
+    const event = this._getEvent();
+    let card = false;
+
+    if (isPreview) {
+      return 
+    }
+
+    if (event) {
+      card = (
+         {}}
+        />
+      )
+    }
+    return (
+      

+        {card}
+      

+    )
+  }
+}
diff --git a/packages/local-private/packages/composer-scheduler/lib/composer/new-event-card.jsx b/packages/local-private/packages/composer-scheduler/lib/composer/new-event-card.jsx
new file mode 100644
index 000000000..94952c223
--- /dev/null
+++ b/packages/local-private/packages/composer-scheduler/lib/composer/new-event-card.jsx
@@ -0,0 +1,257 @@
+import React from 'react';
+import moment from 'moment-timezone'
+import {
+  RetinaImg,
+  DatePicker,
+  TimePicker,
+  TabGroupRegion,
+} from 'nylas-component-kit'
+
+import {
+  DateUtils,
+  Calendar,
+  AccountStore,
+  DatabaseStore} from 'nylas-exports';
+
+import {PLUGIN_ID} from '../scheduler-constants'
+import NewEventHelper from './new-event-helper'
+import ProposedTimeList from './proposed-time-list'
+
+export default class NewEventCard extends React.Component {
+  static displayName = 'NewEventCard';
+
+  static propTypes = {
+    event: React.PropTypes.object.isRequired,
+    draft: React.PropTypes.object.isRequired,
+    onChange: React.PropTypes.func.isRequired,
+    onRemove: React.PropTypes.func.isRequired,
+    onParticipantsClick: React.PropTypes.func.isRequired,
+  };
+
+  constructor(props) {
+    super(props);
+    this._mounted = false;
+    this.state = {
+      calendars: [],
+    };
+  }
+
+  componentDidMount() {
+    this._mounted = true;
+    const email = this.props.draft.from[0].email
+    this._loadCalendarsForEmail(email);
+  }
+
+  componentWillReceiveProps(newProps) {
+    const email = newProps.draft.from[0].email
+    this._loadCalendarsForEmail(email);
+  }
+
+  componentWillUnmount() {
+    this._mounted = false;
+  }
+
+  _loadCalendarsForEmail(email) {
+    if (this._lastEmail === email) {
+      return
+    }
+    this._lastEmail = email
+
+    const account = AccountStore.accountForEmail(email);
+    DatabaseStore.findAll(Calendar, {accountId: account.id})
+    .then((calendars) => {
+      if (!this._mounted || !calendars) { return }
+      this.setState({calendars: calendars.filter(c => !c.readOnly)})
+    });
+  }
+
+  _renderIcon(name) {
+    return (
+      
+    )
+  }
+
+  _renderParticipants() {
+    return this.props.draft.participants().map(r => r.displayName()).join(", ")
+  }
+
+  _renderCalendarPicker() {
+    if (this.state.calendars.length <= 1) {
+      return false;
+    }
+    const calOpts = this.state.calendars.map(cal =>
+      
+    );
+    const onChange = (e) => { this.props.onChange({calendarId: e.target.value}) }
+    return (
+      

+        {this._renderIcon("ic-eventcard-calendar@2x.png")}
+        
+      

+    )
+  }
+
+  _onProposeTimes = () => {
+    NewEventHelper.launchCalendarWindow(this.props.draft.clientId);
+  }
+
+  _eventStart() {
+    return moment.unix(this.props.event.start || moment().unix())
+  }
+
+  _eventEnd() {
+    return moment.unix(this.props.event.end || moment().unix())
+  }
+
+  _onChangeDay = (newTimestamp) => {
+    const newDay = moment(newTimestamp)
+    const start = this._eventStart()
+    const end = this._eventEnd()
+    start.year(newDay.year())
+    end.year(newDay.year())
+    start.dayOfYear(newDay.dayOfYear())
+    end.dayOfYear(newDay.dayOfYear())
+    this.props.onChange({start: start.unix(), end: end.unix()})
+  }
+
+  _onChangeStartTime = (newTimestamp) => {
+    const newTime = moment(newTimestamp)
+    const start = this._eventStart()
+    const end = this._eventEnd()
+    start.hour(newTime.hour())
+    start.minute(newTime.minute())
+    let newEnd = moment(end)
+    if (end.isSameOrBefore(start)) {
+      const leftInDay = moment(start).endOf('day').diff(start)
+      const move = Math.min(leftInDay, moment.duration(1, 'hour').asMilliseconds());
+      newEnd = moment(start).add(move, 'ms')
+    }
+    this.props.onChange({start: start.unix(), end: newEnd.unix()})
+  }
+
+  _onChangeEndTime = (newTimestamp) => {
+    const newTime = moment(newTimestamp)
+    const start = this._eventStart()
+    const end = this._eventEnd()
+    end.hour(newTime.hour())
+    end.minute(newTime.minute())
+    let newStart = moment(start)
+    if (start.isSameOrAfter(end)) {
+      const sinceDay = end.diff(moment(end).startOf('day'))
+      const move = Math.min(sinceDay, moment.duration(1, 'hour').asMilliseconds());
+      newStart = moment(end).subtract(move, 'ms');
+    }
+    this.props.onChange({end: end.unix(), start: newStart.unix()})
+  }
+
+  _renderTimePicker() {
+    const metadata = this.props.draft.metadataForPluginId(PLUGIN_ID);
+    if (metadata && metadata.proposals) {
+      return (
+        
+      )
+    }
+
+    const startVal = (this.props.event.start) * 1000;
+    const endVal = (this.props.event.end) * 1000;
+    return (
+      

+        {this._renderIcon("ic-eventcard-time@2x.png")}
+        
+          
+          to
+          
+          
+            {moment().tz(DateUtils.timeZone).format("z")}
+          
+           
+          on
+           
+          
+        
+      

+    )
+  }
+
+  _renderSuggestPrompt() {
+    const metadata = this.props.draft.metadataForPluginId(PLUGIN_ID);
+    if (metadata && metadata.proposals) {
+      return (
+        
+      )
+    }
+    return (
+      
+    )
+  }
+
+  render() {
+    return (
+      

+        
+          
✕

+          

+            {this._renderIcon("ic-eventcard-description@2x.png")}
+             this.props.onChange({title: e.target.value})}
+            />
+          

+
+          {this._renderTimePicker()}
+
+          {this._renderSuggestPrompt()}
+
+          {this._renderCalendarPicker()}
+
+          

+            {this._renderIcon("ic-eventcard-people@2x.png")}
+            
{this._renderParticipants()}

+          

+
+          

+            {this._renderIcon("ic-eventcard-location@2x.png")}
+             this.props.onChange({location: e.target.value})}
+            />
+          

+
+          

+            {this._renderIcon("ic-eventcard-notes@2x.png")}
+
+

In _data/apps.yml: