[local-sync] audit database indices

Summary:
Fixes T7398

We were create unnecessary and duplicate indices for the IDs of all of
our objects and increasing db write overhead.

We were not creating the correct reverse index for our join tables.

The search API'd db is already in scope of the accountId, this is an
unnecessary constraint on the query

Test Plan: manual

Reviewers: spang, juan

Reviewed By: juan

Maniphest Tasks: T7398

Differential Revision: https://phab.nylas.com/D3606

packages/local-sync/src/local-api/search.js

@@ -260,10 +260,7 @@ class ImapSearchClient {
     const {Message} = db;
     return (await this._search(db, query)).flatMap((uids) => {
       return Message.findAll({
-        where: {
-          accountId: this.account.id,
-          folderImapUID: uids,
-        },
+        where: {folderImapUID: uids},
       });
     }).flatMap((messages) => {
       return getThreadsForMessages(db, messages, limit);

packages/local-sync/src/local-sync-worker/syncback_tasks/ensure-message-in-sent-folder.imap.js

@@ -32,7 +32,7 @@ class EnsureMessageInSentFolderIMAP extends SyncbackTask {
     }
 
     const baseMessage = await Message.findById(messageId,
-      {include: [{model: db.Folder}, {model: db.Label}]});
+      {include: [{model: db.Folder}, {model: db.Label}, {model: db.File}]});
 
     if (!baseMessage) {
       throw new APIError(`Couldn't find message ${messageId} to stuff in sent folder`, 500)

packages/local-sync/src/models/contact.js

@@ -1,5 +1,14 @@
 const crypto = require('crypto')
 
+/**
+ * NOTE: SQLITE creates an index on the `primaryKey` (the ID) for you.
+ * This "Auto Index" is called `sqlite_autoindex_contacts_1`.
+ *
+ * If you run `EXPLAIN QUERY PLAN SELECT * FROM contacts WHERE id=1` you
+ * get:
+ *   SEARCH TABLE contacts USING INDEX sqlite_autoindex_contacts_1
+ * (id=?)
+ */
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('contact', {
     id: {type: Sequelize.STRING(65), primaryKey: true},
@@ -8,12 +17,6 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     email: Sequelize.STRING,
   }, {
-    indexes: [
-      {
-        unique: true,
-        fields: ['id'],
-      },
-    ],
     classMethods: {
       hash({email}) {
         return crypto.createHash('sha256').update(email, 'utf8').digest('hex');

packages/local-sync/src/models/file.js

@@ -13,17 +13,14 @@ module.exports = (sequelize, Sequelize) => {
     accountId: { type: Sequelize.STRING, allowNull: false },
     contentType: Sequelize.STRING(500),
   }, {
+    indexes: [
+      {fields: ['messageId']},
+    ],
     classMethods: {
       associate: ({File, Message}) => {
         File.belongsTo(Message)
       },
     },
-    indexes: [
-      {
-        unique: true,
-        fields: ['id'],
-      },
-    ],
     instanceMethods: {
       async fetch({account, db, logger}) {
         const settings = Object.assign({}, account.connectionSettings, account.decryptedCredentials())

packages/local-sync/src/models/folder.js

@@ -38,16 +38,6 @@ module.exports = (sequelize, Sequelize) => {
     */
     syncState: JSONColumn('syncState'),
   }, {
-    indexes: [
-      {
-        unique: true,
-        fields: ['role'],
-      },
-      {
-        unique: true,
-        fields: ['id'],
-      },
-    ],
     classMethods: {
       associate({Folder, Message, Thread}) {
         Folder.hasMany(Message)

packages/local-sync/src/models/label.js

@@ -9,16 +9,6 @@ module.exports = (sequelize, Sequelize) => {
     name: Sequelize.STRING,
     role: Sequelize.STRING,
   }, {
-    indexes: [
-      {
-        unique: true,
-        fields: ['role'],
-      },
-      {
-        unique: true,
-        fields: ['id'],
-      },
-    ],
     classMethods: {
       associate({Label, Message, MessageLabel, Thread, ThreadLabel}) {
         Label.belongsToMany(Message, {through: MessageLabel})

packages/local-sync/src/models/message.js

@@ -62,18 +62,10 @@ module.exports = (sequelize, Sequelize) => {
     }),
   }, {
     indexes: [
-      {
-        unique: true,
-        fields: ['id'],
-      },
-      {
-        unique: false,
-        fields: ['folderId'],
-      },
-      {
-        unique: false,
-        fields: ['threadId'],
-      },
+      {fields: ['folderId']},
+      {fields: ['threadId']},
+      {fields: ['gMsgId']}, // Use in `searchThreads`
+      {fields: ['folderImapUID']}, // Use in `searchThreads`
     ],
     hooks: {
       beforeUpdate(message) {

packages/local-sync/src/models/messageLabel.js

@@ -1,5 +1,25 @@
+/**
+ * CREATE TABLE IF NOT EXISTS `messageLabels` (
+ *   `createdAt` DATETIME NOT NULL,
+ *   `updatedAt` DATETIME NOT NULL,
+ *   `labelId` VARCHAR(65) NOT NULL REFERENCES `labels` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ *   `messageId` VARCHAR(65) NOT NULL REFERENCES `messages` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ *   PRIMARY KEY (`labelId`, `messageId`)
+ * );
+ *
+ * sqlite_autoindex_messageLabels_1 (labelId, messageId)
+ */
 module.exports = (sequelize) => {
   return sequelize.define('messageLabel', {
   }, {
+    indexes: [
+      // NOTE: When SQLite sets up this table, it creates an auto index in
+      // the order ['labelId', 'messageId']. This is the correct index we
+      // need for queries requesting Messages for a certain Label.
+      //
+      // We need to create one more index to allow queryes from the
+      // reverse direction requesting Labels for a certain Message.
+      {fields: ['messageId', 'labelId']},
+    ],
   });
 };

packages/local-sync/src/models/syncbackRequest.js

@@ -1,5 +1,17 @@
 const {DatabaseTypes: {JSONColumn}} = require('isomorphic-core');
 
+/**
+ * CREATE TABLE IF NOT EXISTS `syncbackRequests` (
+ *   `id` INTEGER PRIMARY KEY AUTOINCREMENT, `type` VARCHAR(255),
+ *   `status` TEXT NOT NULL DEFAULT 'NEW',
+ *   `error` TEXT,
+ *   `props` TEXT,
+ *   `responseJSON` TEXT,
+ *   `accountId` VARCHAR(255) NOT NULL,
+ *   `createdAt` DATETIME NOT NULL,
+ *   `updatedAt` DATETIME NOT NULL
+ * );
+ */
 module.exports = (sequelize, Sequelize) => {
   return sequelize.define('syncbackRequest', {
     type: Sequelize.STRING,
@@ -13,6 +25,9 @@ module.exports = (sequelize, Sequelize) => {
     responseJSON: JSONColumn('responseJSON'),
     accountId: { type: Sequelize.STRING, allowNull: false },
   }, {
+    indexes: [
+      {fields: ['status']},
+    ],
     instanceMethods: {
       toJSON() {
         return {

packages/local-sync/src/models/thread.js

@@ -23,7 +23,6 @@ module.exports = (sequelize, Sequelize) => {
     participants: JSONArrayColumn('participants'),
   }, {
     indexes: [
-      { fields: ['id'], unique: true },
       { fields: ['subject'] },
       { fields: ['remoteThreadId'] },
     ],

packages/local-sync/src/models/threadFolder.js

@@ -1,5 +1,25 @@
+/**
+ * CREATE TABLE IF NOT EXISTS `threadFolders` (
+ *   `createdAt` DATETIME NOT NULL,
+ *   `updatedAt` DATETIME NOT NULL,
+ *   `threadId` VARCHAR(65) NOT NULL REFERENCES `threads` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ *   `folderId` VARCHAR(65) NOT NULL REFERENCES `folders` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ *   PRIMARY KEY (`threadId`, `folderId`)
+ * );
+ *
+ * sqlite_autoindex_threadFolders_1 (threadId, folderId)
+ */
 module.exports = (sequelize) => {
   return sequelize.define('threadFolder', {
   }, {
+    indexes: [
+      // NOTE: When SQLite sets up this table, it creates an auto index in
+      // the order ['threadId', 'folderId']. This is the correct index we
+      // need for queries requesting Folders for a certain Thread.
+      //
+      // We need to create one more index to allow queryes from the
+      // reverse direction requesting Threads for a certain Folder.
+      {fields: ['folderId', 'threadId']},
+    ],
   });
 };

packages/local-sync/src/models/threadLabel.js

@@ -1,5 +1,25 @@
+/**
+ * CREATE TABLE IF NOT EXISTS `threadLabels` (
+ *   `createdAt` DATETIME NOT NULL,
+ *   `updatedAt` DATETIME NOT NULL,
+ *   `labelId` VARCHAR(65) NOT NULL REFERENCES `labels` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ *   `threadId` VARCHAR(65) NOT NULL REFERENCES `threads` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+ *   PRIMARY KEY (`labelId`, `threadId`)
+ * );
+ *
+ * sqlite_autoindex_threadLabels_1 labelId, threadId
+ */
 module.exports = (sequelize) => {
   return sequelize.define('threadLabel', {
   }, {
+    indexes: [
+      // NOTE: When SQLite sets up this table, it creates an auto index in
+      // the order ['labelId', 'threadId']. This is the correct index we
+      // need for queries requesting Threads for a certain Label.
+      //
+      // We need to create one more index to allow queryes from the
+      // reverse direction requesting Labels for a certain Thread.
+      {fields: ['threadId', 'labelId']},
+    ],
   });
 };

packages/local-sync/src/shared/local-database-connector.js

@@ -6,6 +6,8 @@ const TransactionConnector = require('./transaction-connector')
 
 require('./database-extensions'); // Extends Sequelize on require
 
+const ENABLE_SEQUELIZE_DEUBG_LOGGING = false
+
 class LocalDatabaseConnector {
   constructor() {
     this._cache = {};
@@ -16,7 +18,7 @@ class LocalDatabaseConnector {
     return new Sequelize(dbname, '', '', {
       storage: storage,
       dialect: "sqlite",
-      logging: false,
+      logging: ENABLE_SEQUELIZE_DEUBG_LOGGING ? console.log : false,
     })
   }