[local-sync] Implement /threads/search endpoint for Gmail

Summary: See title

Test Plan: Ran it locally

Reviewers: khamidou, juan, evan

Reviewed By: juan, evan

Differential Revision: https://phab.nylas.com/D3496

packages/isomorphic-core/src/models/account.js

@@ -104,6 +104,16 @@ module.exports = (sequelize, Sequelize) => {
         }
       },
 
+      bearerToken(xoauth2) {
+        // We have to unpack the access token from the entire XOAuth2
+        // token because it is re-packed during the SMTP connection login.
+        // https://github.com/nodemailer/smtp-connection/blob/master/lib/smtp-connection.js#L1418
+        const bearer = "Bearer ";
+        const decoded = atob(xoauth2);
+        const tokenIndex = decoded.indexOf(bearer) + bearer.length;
+        return decoded.substring(tokenIndex, decoded.length - 2);
+      },
+
       smtpConfig() {
         const {smtp_host, smtp_port, ssl_required} = this.connectionSettings;
         const config = {
@@ -118,15 +128,7 @@ module.exports = (sequelize, Sequelize) => {
         } else if (this.provider === 'gmail') {
           const {xoauth2} = this.decryptedCredentials();
           const {imap_username} = this.connectionSettings;
-
-          // We have to unpack the access token from the entire XOAuth2
-          // token because it is re-packed during the SMTP connection login.
-          // https://github.com/nodemailer/smtp-connection/blob/master/lib/smtp-connection.js#L1418
-          const bearer = "Bearer ";
-          const decoded = atob(xoauth2);
-          const tokenIndex = decoded.indexOf(bearer) + bearer.length;
-          const token = decoded.substring(tokenIndex, decoded.length - 2);
-
+          const token = this.bearerToken(xoauth2);
           config.auth = { user: imap_username, xoauth2: token }
         } else {
           throw new Error(`${this.provider} not yet supported`)

packages/local-sync/src/local-api/routes/threads.js

@@ -2,6 +2,7 @@ const Joi = require('joi');
 const _ = require('underscore');
 const Serialization = require('../serialization');
 const {createSyncbackRequest} = require('../route-helpers')
+const {searchClientForAccount} = require('../search');
 
 module.exports = (server) => {
   server.route({
@@ -50,6 +51,29 @@ module.exports = (server) => {
     },
   });
 
+  server.route({
+    method: 'GET',
+    path: '/threads/search/streaming',
+    config: {
+      description: 'Returns threads',
+      notes: 'Notes go here',
+      tags: ['threads'],
+      validate: {
+        query: {
+          limit: Joi.number().integer().min(1).max(2000).default(100),
+          q: Joi.string(),
+        },
+      },
+    },
+    handler: async (request, reply) => {
+      const account = request.auth.credentials;
+      const db = await request.getAccountDatabase();
+      const client = searchClientForAccount(account);
+      const threads = await client.searchThreads(db, request.query.q, request.query.limit);
+      reply(`${JSON.stringify(threads)}\n`);
+    },
+  });
+
   server.route({
     method: 'PUT',
     path: '/threads/{id}',

packages/local-sync/src/local-api/search.js

@@ -0,0 +1,153 @@
+const request = require('request');
+const _ = require('underscore');
+
+class GmailSearchClient {
+  constructor(accountToken) {
+    this.accountToken = accountToken;
+  }
+
+  // Note that the Gmail API returns message IDs in hex format. So for
+  // example the IMAP X-GM-MSGID 1438297078380071706 corresponds to
+  // 13f5db9286538b1a in API responses. Normally we could just use parseInt(id, 16),
+  // but many of the IDs returned are outside of the precise range of doubles,
+  // so this function accomplishes hex ID parsing using rudimentary arbitrary
+  // precision ints implemented using strings.
+  _parseHexId(hexId) {
+    const add = (a, b) => {
+      let carry = 0;
+      const x = a.split('').map(Number);
+      const y = b.split('').map(Number);
+      const result = [];
+      while (x.length || y.length) {
+        const sum = (x.pop() || 0) + (y.pop() || 0) + carry;
+        result.push(sum < 10 ? sum : sum - 10);
+        carry = sum < 10 ? 0 : 1;
+      }
+      if (carry) {
+        result.push(carry);
+      }
+      result.reverse();
+      return result.join('');
+    };
+
+    let value = '0';
+    for (const c of hexId) {
+      const digit = parseInt(c, 16);
+      for (let mask = 0x8; mask; mask >>= 1) {
+        value = add(value, value);
+        if (digit & mask) {
+          value = add(value, '1');
+        }
+      }
+    }
+    return value;
+  }
+
+  _search(query, limit) {
+    let results = [];
+    const params = {q: query, maxResults: limit};
+
+    return new Promise((resolve, reject) => {
+      const maxTries = 10;
+      const trySearch = (numTries) => {
+        if (numTries >= maxTries) {
+          // If we've been through the loop 10 times, it means we got a request
+          // a crazy-high offset --- raise an error.
+          console.error('Too many results:', results.length);
+          reject(new Error('Too many results'));
+          return;
+        }
+
+        request('https://www.googleapis.com/gmail/v1/users/me/messages', {
+          qs: params,
+          headers: {Authorization: `Bearer ${this.accountToken}`},
+        }, (error, response, body) => {
+          if (error) {
+            reject(new Error(`Error issuing search request: ${error}`));
+            return;
+          }
+
+          if (response.statusCode !== 200) {
+            reject(new Error(`Error issuing search request: ${response.statusMessage}`));
+            return;
+          }
+
+          let data = null;
+          try {
+            data = JSON.parse(body);
+          } catch (e) {
+            reject(new Error(`Error parsing response as JSON: ${e}`));
+            return;
+          }
+          if (!data.messages) {
+            resolve(results);
+            return;
+          }
+
+          // Note that the Gmail API returns message IDs in hex format. So for
+          // example the IMAP X-GM-MSGID 1438297078380071706 corresponds to
+          // 13f5db9286538b1a in the API response we have here.
+          results = results.concat(data.messages.map((m) => this._parseHexId(m.id)));
+
+          if (results.length >= limit) {
+            resolve(results.slice(0, limit));
+            return;
+          }
+
+          if (!data.nextPageToken) {
+            resolve(results);
+            return;
+          }
+          params.pageToken = data.nextPageToken;
+          trySearch(numTries + 1);
+        });
+      };
+      trySearch(0);
+    });
+  }
+
+  async searchThreads(db, query, limit) {
+    const messageIds = await this._search(query, limit);
+    if (!messageIds.length) {
+      return [];
+    }
+
+    const {Message, Folder, Label, Thread} = db;
+    const messages = await Message.findAll({
+      where: {gMsgId: {$in: messageIds}},
+    });
+
+    const threadIds = _.uniq(messages.map((m) => m.threadId));
+    const threads = await Thread.findAll({
+      where: {id: threadIds},
+      include: [
+        {model: Folder},
+        {model: Label},
+        {
+          model: Message,
+          as: 'messages',
+          attributes: _.without(Object.keys(Message.attributes), 'body'),
+          include: [
+            {model: Folder},
+          ],
+        },
+      ],
+      limit: limit,
+      order: [['lastMessageReceivedDate', 'DESC']],
+    });
+    return threads;
+  }
+}
+
+module.exports.searchClientForAccount = (account) => {
+  switch (account.provider) {
+    case 'gmail': {
+      const credentials = account.decryptedCredentials();
+      const accountToken = account.bearerToken(credentials.xoauth2);
+      return new GmailSearchClient(accountToken);
+    }
+    default: {
+      throw new Error(`Unsupported provider for search endpoint: ${account.provider}`);
+    }
+  }
+};

packages/local-sync/src/models/message.js

@@ -20,6 +20,7 @@ module.exports = (sequelize, Sequelize) => {
     accountId: { type: Sequelize.STRING, allowNull: false },
     version: Sequelize.INTEGER,
     headerMessageId: Sequelize.STRING,
+    gMsgId: { type: Sequelize.STRING, allowNull: true },
     body: Sequelize.TEXT('long'),
     headers: buildJSONColumnOptions('headers'),
     subject: Sequelize.STRING(500),

packages/local-sync/src/shared/message-factory.js

@@ -105,6 +105,7 @@ async function parseFromImap(imapMessage, desiredParts, {db, accountId, folder})
     labels: [],
     headers: parsedHeaders,
     headerMessageId: parsedHeaders['message-id'] ? parsedHeaders['message-id'][0] : '',
+    gMsgId: parsedHeaders['x-gm-msgid'],
     subject: parsedHeaders.subject[0],
   }