Initial commit

.eslintrc

@@ -0,0 +1,46 @@
+{
+  "parser": "babel-eslint",
+  "extends": "airbnb",
+  "globals": {
+    "NylasEnv": false,
+    "$n": false,
+    "waitsForPromise": false,
+    "advanceClock": false,
+    "TEST_ACCOUNT_ID": false,
+    "TEST_ACCOUNT_NAME": false,
+    "TEST_ACCOUNT_EMAIL": false,
+    "__base": false
+  },
+  "env": {
+    "browser": true,
+    "node": true,
+    "jasmine": true
+  },
+  "rules": {
+    "arrow-body-style": "off",
+    "prefer-arrow-callback": ["error", {"allowNamedFunctions": true}],
+    "eqeqeq": ["error", "smart"],
+    "id-length": "off",
+    "object-curly-spacing": "off",
+    "max-len": "off",
+    "new-cap": ["error", {"capIsNew": false}],
+    "no-console": "off",
+    "no-constant-condition": "off",
+    "no-loop-func": "off",
+    "no-shadow": "error",
+    "no-underscore-dangle": "off",
+    "object-shorthand": "off",
+    "quotes": "off",
+    "global-require": "off",
+    "quote-props": ["error", "consistent-as-needed", { "keywords": true }],
+    "no-param-reassign": ["error", { "props": false }],
+    "semi": "off",
+    "import/no-unresolved": ["error", {"ignore": ["nylas-exports", "nylas-component-kit", "electron", "nylas-store", "react-dom/server", "nylas-observables", "windows-shortcuts", "moment-round", "chrono-node", "event-kit", "enzyme"]}],
+    "react/no-multi-comp": "off",
+    "react/prop-types": ["error", {"ignore": ["children"]}],
+    "react/sort-comp": "error"
+  },
+  "settings": {
+    "import/resolver": {"node": {"extensions": [".es6", ".jsx", ".coffee", ".json", ".cjsx", ".js"]}}
+  }
+}

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+node_modules

api/app.js

@@ -0,0 +1,71 @@
+const Hapi = require('hapi');
+const HapiSwagger = require('hapi-swagger');
+const HapiBasicAuth = require('hapi-auth-basic');
+const Inert = require('inert');
+const Vision = require('vision');
+const Package = require('./package');
+const fs = require('fs');
+const path = require('path');
+global.__base = path.join(__dirname, '..')
+
+const server = new Hapi.Server();
+server.connection({ port: 3000 });
+
+const plugins = [Inert, Vision, HapiBasicAuth, {
+  register: HapiSwagger,
+  options: {
+    info: {
+      title: 'Nylas API Documentation',
+      version: Package.version,
+    },
+  },
+}];
+
+let sharedDb = null;
+const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`)
+DatabaseConnectionFactory.forShared().then((db) => {
+  sharedDb = db;
+});
+
+const validate = (request, username, password, callback) => {
+  const {AccountToken} = sharedDb;
+
+  AccountToken.find({
+    where: {
+      value: username,
+    },
+  }).then((token) => {
+    if (!token) {
+      callback(null, false, {});
+      return
+    }
+    token.getAccount().then((account) => {
+      callback(null, true, account);
+    });
+  });
+};
+
+const attach = (directory) => {
+  const routesDir = path.join(__dirname, directory)
+  fs.readdirSync(routesDir).forEach((filename) => {
+    if (filename.endsWith('.js')) {
+      const routeFactory = require(path.join(routesDir, filename));
+      routeFactory(server);
+    }
+  });
+}
+
+server.register(plugins, (err) => {
+  if (err) { throw err; }
+
+  attach('./routes/')
+  attach('./decorators/')
+
+  server.auth.strategy('api-consumer', 'basic', { validateFunc: validate });
+  server.auth.default('api-consumer');
+
+  server.start((startErr) => {
+    if (startErr) { throw startErr; }
+    console.log('Server running at:', server.info.uri);
+  });
+});

api/decorators/connections.js

@@ -0,0 +1,10 @@
+/* eslint func-names:0 */
+
+const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`);
+
+module.exports = (server) => {
+  server.decorate('request', 'getAccountDatabase', function () {
+    const account = this.auth.credentials;
+    return DatabaseConnectionFactory.forAccount(account.id);
+  });
+}

api/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "api",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "hapi": "^13.4.1",
+    "hapi-auth-basic": "^4.2.0",
+    "hapi-swagger": "^6.1.0",
+    "inert": "^4.0.0",
+    "joi": "^8.4.2",
+    "vision": "^4.1.0"
+  }
+}

api/routes/accounts.js

@@ -0,0 +1,24 @@
+const Serialization = require('../serialization');
+
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/account',
+    config: {
+      description: 'Returns the current account.',
+      notes: 'Notes go here',
+      tags: ['accounts'],
+      validate: {
+        params: {
+        },
+      },
+      response: {
+        schema: Serialization.jsonSchema('Account'),
+      },
+    },
+    handler: (request, reply) => {
+      const account = request.auth.credentials;
+      reply(Serialization.jsonStringify(account));
+    },
+  });
+};

api/routes/threads.js

@@ -0,0 +1,31 @@
+const Joi = require('joi');
+const Serialization = require('../serialization');
+
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/threads',
+    config: {
+      description: 'Returns threads',
+      notes: 'Notes go here',
+      tags: ['threads'],
+      validate: {
+        params: {
+        },
+      },
+      response: {
+        schema: Joi.array().items(
+          Serialization.jsonSchema('Account')
+        ),
+      },
+    },
+    handler: (request, reply) => {
+      request.getAccountDatabase().then((db) => {
+        const {Thread} = db;
+        Thread.findAll({limit: 50}).then((threads) => {
+          reply(Serialization.jsonStringify(threads));
+        })
+      })
+    },
+  });
+};

api/serialization.js

@@ -0,0 +1,25 @@
+const Joi = require('joi');
+
+function replacer(key, value) {
+  // force remove any disallowed keys here
+  return value;
+}
+
+function jsonSchema(modelName) {
+  if (modelName === 'Account') {
+    return Joi.object().keys({
+      id: Joi.number(),
+      email_address: Joi.string(),
+    })
+  }
+  return null;
+}
+
+function jsonStringify(models) {
+  return JSON.stringify(models, replacer, 2);
+}
+
+module.exports = {
+  jsonSchema,
+  jsonStringify,
+}

core/config/development.json

@@ -0,0 +1,11 @@
+{
+  "storage": {
+    "database": "account-$ACCOUNTID",
+    "username": null,
+    "password": null,
+    "options": {
+      "dialect": "sqlite",
+      "storage": "./account-$ACCOUNTID.sqlite"
+    }
+  }
+}

core/database-connection-factory.js

@@ -0,0 +1,78 @@
+const Sequelize = require('sequelize');
+const fs = require('fs');
+const path = require('path');
+
+const STORAGE_DIR = path.join(__base, 'storage');
+if (!fs.existsSync(STORAGE_DIR)) {
+  fs.mkdirSync(STORAGE_DIR);
+}
+
+class DatabaseConnectionFactory {
+  constructor() {
+    this._pools = {};
+  }
+
+  _readModelsInDirectory(sequelize, dirname) {
+    const db = {};
+    for (const filename of fs.readdirSync(dirname)) {
+      if (filename.endsWith('.js')) {
+        const model = sequelize.import(path.join(dirname, filename));
+        db[model.name] = model;
+      }
+    }
+    Object.keys(db).forEach((modelName) => {
+      if ("associate" in db[modelName]) {
+        db[modelName].associate(db);
+      }
+    });
+
+    return db;
+  }
+
+  _sequelizeForAccount(accountId) {
+    const sequelize = new Sequelize(accountId, '', '', {
+      storage: path.join(STORAGE_DIR, `a-${accountId}.sqlite`),
+      dialect: "sqlite",
+    });
+
+    const modelsPath = path.join(__dirname, 'models/account');
+    const db = this._readModelsInDirectory(sequelize, modelsPath)
+
+    db.sequelize = sequelize;
+    db.Sequelize = Sequelize;
+
+    return sequelize.authenticate().then(() =>
+      sequelize.sync()
+    ).thenReturn(db);
+  }
+
+  forAccount(accountId) {
+    this._pools[accountId] = this._pools[accountId] || this._sequelizeForAccount(accountId);
+    return this._pools[accountId];
+  }
+
+  _sequelizeForShared() {
+    const sequelize = new Sequelize('shared', '', '', {
+      storage: path.join(STORAGE_DIR, 'shared.sqlite'),
+      dialect: "sqlite",
+    });
+
+    const modelsPath = path.join(__dirname, 'models/shared');
+    const db = this._readModelsInDirectory(sequelize, modelsPath)
+
+    db.sequelize = sequelize;
+    db.Sequelize = Sequelize;
+
+    return sequelize.authenticate().then(() =>
+      sequelize.sync()
+    ).thenReturn(db);
+  }
+
+  forShared() {
+    this._pools.shared = this._pools.shared || this._sequelizeForShared();
+    return this._pools.shared;
+  }
+
+}
+
+module.exports = new DatabaseConnectionFactory()

core/migrations/20160617002207-create-user.js

@@ -0,0 +1,33 @@
+'use strict';
+module.exports = {
+  up: function(queryInterface, Sequelize) {
+    return queryInterface.createTable('Users', {
+      id: {
+        allowNull: false,
+        autoIncrement: true,
+        primaryKey: true,
+        type: Sequelize.INTEGER
+      },
+      first_name: {
+        type: Sequelize.STRING
+      },
+      last_name: {
+        type: Sequelize.STRING
+      },
+      bio: {
+        type: Sequelize.TEXT
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      }
+    });
+  },
+  down: function(queryInterface, Sequelize) {
+    return queryInterface.dropTable('Users');
+  }
+};

core/models/account/category.js

@@ -0,0 +1,24 @@
+module.exports = (sequelize, Sequelize) => {
+  const Category = sequelize.define('Category', {
+    name: Sequelize.STRING,
+    role: Sequelize.STRING,
+    syncState: {
+      type: Sequelize.STRING,
+      defaultValue: '{}',
+      get: function get() {
+        return JSON.parse(this.getDataValue('syncState'))
+      },
+      set: function set(val) {
+        this.setDataValue('syncState', JSON.stringify(val));
+      },
+    },
+  }, {
+    classMethods: {
+      associate: ({MessageUID}) => {
+        Category.hasMany(MessageUID)
+      },
+    },
+  });
+
+  return Category;
+};

core/models/account/message.js

@@ -0,0 +1,21 @@
+module.exports = (sequelize, Sequelize) => {
+  const Message = sequelize.define('Message', {
+    subject: Sequelize.STRING,
+    snippet: Sequelize.STRING,
+    body: Sequelize.STRING,
+    headers: Sequelize.STRING,
+    date: Sequelize.DATE,
+    unread: Sequelize.BOOLEAN,
+    starred: Sequelize.BOOLEAN,
+  }, {
+    classMethods: {
+      associate: ({MessageUID}) => {
+        // is this really a good idea?
+        // Message.hasMany(Contact, {as: 'from'})
+        Message.hasMany(MessageUID, {as: 'uids'})
+      },
+    },
+  });
+
+  return Message;
+};

core/models/account/message_uid.js

@@ -0,0 +1,14 @@
+module.exports = (sequelize, Sequelize) => {
+  const MessageUID = sequelize.define('MessageUID', {
+    uid: Sequelize.STRING,
+  }, {
+    classMethods: {
+      associate: ({Category, Message}) => {
+        MessageUID.belongsTo(Category)
+        MessageUID.belongsTo(Message)
+      },
+    },
+  });
+
+  return MessageUID;
+};

core/models/account/thread.js

@@ -0,0 +1,15 @@
+module.exports = (sequelize, Sequelize) => {
+  const Thread = sequelize.define('Thread', {
+    first_name: Sequelize.STRING,
+    last_name: Sequelize.STRING,
+    bio: Sequelize.TEXT,
+  }, {
+    classMethods: {
+      associate: (models) => {
+        // associations can be defined here
+      },
+    },
+  });
+
+  return Thread;
+};

core/models/shared/account-token.js

@@ -0,0 +1,18 @@
+module.exports = (sequelize, Sequelize) => {
+  const AccountToken = sequelize.define('AccountToken', {
+    value: Sequelize.STRING,
+  }, {
+    classMethods: {
+      associate: ({Account}) => {
+        AccountToken.belongsTo(Account, {
+          onDelete: "CASCADE",
+          foreignKey: {
+            allowNull: false,
+          },
+        });
+      },
+    },
+  });
+
+  return AccountToken;
+};

core/models/shared/account.js

@@ -0,0 +1,21 @@
+module.exports = (sequelize, Sequelize) => {
+  const Account = sequelize.define('Account', {
+    emailAddress: Sequelize.STRING,
+  }, {
+    classMethods: {
+      associate: ({AccountToken}) => {
+        Account.hasMany(AccountToken, {as: 'tokens'})
+      },
+    },
+    instanceMethods: {
+      toJSON: function toJSON() {
+        return {
+          id: this.id,
+          email_address: this.emailAddress,
+        }
+      },
+    },
+  });
+
+  return Account;
+};

core/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "core",
+  "version": "1.0.0",
+  "description": "",
+  "main": "database-connection-factory.js",
+  "dependencies": {
+    "mysql": "^2.10.2",
+    "sequelize": "^3.23.3",
+    "sqlite3": "^3.1.4"
+  },
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

process.json

@@ -0,0 +1,24 @@
+{
+  "apps": [
+    {
+      "name": "api",
+      "script": "api/app.js",
+      "watch": true,
+      "instances": "max",
+      "exec_mode": "cluster",
+      "env": {
+        "NODE_ENV": "development"
+      },
+      "env_production": {
+        "NODE_ENV": "production"
+      }
+    },
+    {
+      "name": "sync",
+      "script": "sync/app.js",
+      "watch": true,
+      "instances": "max",
+      "exec_mode": "cluster"
+    }
+  ]
+}

sync/app.js

@@ -0,0 +1,19 @@
+const path = require('path');
+
+global.__base = path.join(__dirname, '..')
+global.config = require(`${__base}/core/config/${process.env.ENV || 'development'}.json`);
+
+const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`)
+const SyncWorkerPool = require('./sync-worker-pool');
+const workerPool = new SyncWorkerPool();
+
+DatabaseConnectionFactory.forShared().then((db) => {
+  const {Account} = db
+  Account.findAll().then((accounts) => {
+    accounts.forEach((account) => {
+      workerPool.addWorkerForAccount(account);
+    });
+  });
+});
+
+global.workerPool = workerPool;

sync/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "imap-experiment",
+  "version": "1.0.0",
+  "description": "",
+  "main": "app.js",
+  "dependencies": {
+    "bluebird": "^3.4.1",
+    "imap": "^0.8.17"
+  },
+  "devDependencies": {},
+  "scripts": {
+    "start": "node app.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

sync/sync-worker-pool.js

@@ -0,0 +1,16 @@
+const SyncWorker = require('./sync-worker');
+const DatabaseConnectionFactory = require(`${__base}/core/database-connection-factory`)
+
+class SyncWorkerPool {
+  constructor() {
+    this._workers = {};
+  }
+
+  addWorkerForAccount(account) {
+    DatabaseConnectionFactory.forAccount(account.id).then((db) => {
+      this._workers[account.id] = new SyncWorker(account, db);
+    });
+  }
+}
+
+module.exports = SyncWorkerPool;

sync/sync-worker.js

@@ -0,0 +1,297 @@
+const {inspect} = require('util');
+const Promise = require('bluebird');
+const Imap = require('imap');
+
+const State = {
+  Closed: 'closed',
+  Connecting: 'connecting',
+  Open: 'open',
+}
+
+const Capabilities = {
+  Gmail: 'X-GM-EXT-1',
+  Quota: 'QUOTA',
+  UIDPlus: 'UIDPLUS',
+  Condstore: 'CONDSTORE',
+  Search: 'ESEARCH',
+  Sort: 'SORT',
+}
+
+class SyncIMAPConnection {
+  constructor(settings) {
+    this._queue = [];
+    this._current = null;
+    this._state = State.Connecting;
+    this._capabilities = [];
+
+    this._imap = Promise.promisifyAll(new Imap(settings));
+
+    this._imap.once('ready', () => {
+      this._state = State.Open;
+      for (const key of Object.keys(Capabilities)) {
+        const val = Capabilities[key];
+        if (this._imap.serverSupports(val)) {
+          this._capabilities.push(val);
+        }
+      }
+      this.processNextOperation();
+    });
+    this._imap.once('error', (err) => {
+      console.log(err);
+    });
+    this._imap.once('end', () => {
+      this._state = State.Closed;
+      console.log('Connection ended');
+    });
+    this._imap.connect();
+  }
+
+  queueOperation(op) {
+    this._queue.push(op);
+    if (this._state === State.Open && !this._current) {
+      this.processNextOperation();
+    }
+  }
+
+  processNextOperation() {
+    if (this._current) { return; }
+
+    this._current = this._queue.shift();
+
+    if (this._current) {
+      console.log(`Starting task ${this._current.constructor.name}`)
+
+      const result = this._current.run(this._imap);
+      if (result instanceof Promise === false) {
+        throw new Error(`processNextOperation: Expected ${this._current.constructor.name} to return promise.`);
+      }
+      result.catch((err) => {
+        this._current = null;
+        console.error(err);
+      });
+      result.then(() => {
+        console.log(`Finished task ${this._current.constructor.name}`)
+        this._current = null;
+        this.processNextOperation();
+      });
+    }
+  }
+}
+
+class SyncMailboxOperation {
+  constructor(db, {role} = {}) {
+    this._db = db;
+    this._category = null;
+    this._box = null;
+  }
+
+  _fetch(imap, range) {
+    return new Promise((resolve, reject) => {
+      const f = imap.fetch(range, {
+        bodies: ['HEADER', 'TEXT'],
+      });
+      f.on('message', (msg, uid) => this._receiveMessage(msg, uid));
+      f.once('error', reject);
+      f.once('end', resolve);
+    });
+  }
+
+  _unlinkAllMessages() {
+    const {MessageUID} = this._db;
+    return MessageUID.destroy({
+      where: {
+        categoryId: this._category.id,
+      },
+    })
+  }
+
+  _receiveMessage(msg, uid) {
+    let attributes = null;
+    let body = null;
+    let headers = null;
+
+    msg.on('attributes', (attrs) => {
+      attributes = attrs;
+    });
+    msg.on('body', (stream, type) => {
+      const chunks = [];
+      stream.on('data', (chunk) => {
+        chunks.push(chunk);
+      });
+      stream.once('end', () => {
+        const full = Buffer.concat(chunks).toString('utf8');
+        if (type === 'TEXT') {
+          body = full;
+        }
+        if (type === 'HEADERS') {
+          headers = full;
+        }
+      });
+    });
+    msg.once('end', () => {
+      this._processMessage(attributes, headers, body, uid);
+    });
+  }
+
+  _processMessage(attributes, headers, body) {
+    console.log(attributes);
+    const {Message, MessageUID} = this._db;
+
+    return Message.create({
+      unread: attributes.flags.includes('\\Unseen'),
+      starred: attributes.flags.includes('\\Flagged'),
+      date: attributes.date,
+      body: body,
+    }).then((model) => {
+      return MessageUID.create({
+        MessageId: model.id,
+        CategoryId: this._category.id,
+        uid: attributes.uid,
+      });
+    });
+  }
+
+  // _flushProcessedMessages() {
+  //   return sequelize.transaction((transaction) => {
+  //     return Promise.props({
+  //       msgs: Message.bulkCreate(this._processedMessages, {transaction})
+  //       uids: MessageUID.bulkCreate(this._processedMessageUIDs, {transaction})
+  //     })
+  //   }).then(() => {
+  //     this._processedMessages = [];
+  //     this._processedMessageUIDs = [];
+  //   });
+  // }
+
+  run(imap) {
+    const {Category} = this._db;
+
+    return Promise.props({
+      box: imap.openBoxAsync('INBOX', true),
+      category: Category.find({name: 'INBOX'}),
+    })
+    .then(({category, box}) => {
+      if (this.box.persistentUIDs === false) {
+        throw new Error("Mailbox does not support persistentUIDs.")
+      }
+
+      this._category = category;
+      this._box = box;
+
+      if (box.uidvalidity !== category.syncState.uidvalidity) {
+        return this._unlinkAllMessages();
+      }
+      return Promise.resolve();
+    })
+    .then(() => {
+      const lastUIDNext = this._category.syncState.uidnext;
+      const currentUIDNext = this._box.uidnext
+
+      if (lastUIDNext) {
+        if (lastUIDNext === currentUIDNext) {
+          return Promise.resolve();
+        }
+
+        // just request mail >= UIDNext
+        return this._fetch(imap, `${lastUIDNext}:*`);
+      }
+      return this._fetch(imap, `1:*`);
+    });
+  }
+}
+
+class RefreshMailboxesOperation {
+  constructor(db) {
+    this._db = db;
+  }
+
+  _roleForMailbox(box) {
+    for (const attrib of (box.attribs || [])) {
+      const role = {
+        '\\Sent': 'sent',
+        '\\Drafts': 'drafts',
+        '\\Junk': 'junk',
+        '\\Flagged': 'flagged',
+      }[attrib];
+      if (role) {
+        return role;
+      }
+    }
+    return null;
+  }
+
+  _updateCategoriesWithBoxes(categories, boxes) {
+    const {Category} = this._db;
+
+    const stack = [];
+    const created = [];
+    const next = [];
+
+    Object.keys(boxes).forEach((name) => {
+      stack.push([name, boxes[name]]);
+    });
+
+    while (stack.length > 0) {
+      const [name, box] = stack.pop();
+      if (box.children) {
+        Object.keys(box.children).forEach((subname) => {
+          stack.push([`${name}/${subname}`, box.children[subname]]);
+        });
+      }
+
+      let category = categories.find((cat) => cat.name === name);
+      if (!category) {
+        category = Category.build({
+          name: name,
+          role: this._roleForMailbox(box),
+        });
+        created.push(category);
+      }
+      next.push(category);
+    }
+
+    // Todo: decide whether these are renames or deletes
+    const deleted = categories.filter(cat => !next.includes(cat));
+
+    return {next, created, deleted};
+  }
+
+  run(imap) {
+    return imap.getBoxesAsync().then((boxes) => {
+      const {Category, sequelize} = this._db;
+
+      return sequelize.transaction((transaction) => {
+        return Category.findAll({transaction}).then((categories) => {
+          const {created, deleted} = this._updateCategoriesWithBoxes(categories, boxes);
+
+          let promises = [Promise.resolve()]
+          promises = promises.concat(created.map(cat => cat.save({transaction})))
+          promises = promises.concat(deleted.map(cat => cat.destroy({transaction})))
+          return Promise.all(promises)
+        });
+      });
+    });
+  }
+}
+
+class SyncWorker {
+  constructor(account, db) {
+    this._db = db
+    this._conns = []
+
+    const main = new SyncIMAPConnection({
+      user: 'inboxapptest1@fastmail.fm',
+      password: 'trar2e',
+      host: 'mail.messagingengine.com',
+      port: 993,
+      tls: true,
+    })
+    main.queueOperation(new RefreshMailboxesOperation(db));
+    main.queueOperation(new SyncMailboxOperation(db, {
+      role: 'inbox',
+    }));
+    this._conns.push(main);
+  }
+}
+
+module.exports = SyncWorker;