From 73555273978e499a8f5488ee032e853021e8df58 Mon Sep 17 00:00:00 2001
From: Halla Moore <halla@nylas.com>
Date: Fri, 24 Mar 2017 14:25:36 -0700
Subject: [PATCH] [client-app] Retry queries when database is locked

Summary:
We still don't know exactly what scenarios cause us to get 'database is
locked' errors, but generically this means that we have multiple
processes trying to access the database at the same time. In an attempt
to handle this gracefully, this diff makes it so that we retry the
queries in these cases. Theoretically, the database should free up once
the other process is done using it, and the erroring process just needs
to wait its turn. We still throw an error after 5 retries, so if there's
a larger issue, we'll still be able to tell in Sentry.

Addresses T7992

Test Plan:
I opened a transaction in the worker window and then tried to
do the same in the main window. If I didn't release the transaction in
the worker window, the main window eventually errored. If I did release
the transaction, the main window continued creating its own transaction.

Reviewers: mark, juan, evan

Reviewed By: juan, evan

Differential Revision: https://phab.nylas.com/D4254
---
 .../src/flux/stores/database-store.es6        | 47 ++++++++++++++-----
 1 file changed, 35 insertions(+), 12 deletions(-)

diff --git a/packages/client-app/src/flux/stores/database-store.es6 b/packages/client-app/src/flux/stores/database-store.es6
index fff54ae60..ec236d0eb 100644
--- a/packages/client-app/src/flux/stores/database-store.es6
+++ b/packages/client-app/src/flux/stores/database-store.es6
@@ -5,6 +5,7 @@ import childProcess from 'child_process';
 import PromiseQueue from 'promise-queue';
 import {remote, ipcRenderer} from 'electron';
 import LRU from "lru-cache";
+import {ExponentialBackoffScheduler} from 'isomorphic-core';
 
 import NylasStore from '../../global/nylas-store';
 import Utils from '../models/utils';
@@ -25,6 +26,9 @@ const DatabasePhase = {
 const DEBUG_TO_LOG = false;
 const DEBUG_QUERY_PLANS = NylasEnv.inDevMode();
 
+const BASE_RETRY_LOCK_DELAY = 100;
+const MAX_RETRY_LOCK_DELAY = 3 * 1000;
+
 let JSONBlob = null;
 
 /*
@@ -291,7 +295,7 @@ class DatabaseStore extends NylasStore {
   // If a query is made before the database has been opened, the query will be
   // held in a queue and run / resolved when the database is ready.
   _query(query, values = [], background = false, logQueryPlanDebugOutput = true) {
-    return new Promise((resolve, reject) => {
+    return new Promise(async (resolve, reject) => {
       if (!this._open) {
         this._waiting.push(() => this._query(query, values).then(resolve, reject));
         return;
@@ -325,7 +329,7 @@ class DatabaseStore extends NylasStore {
       const start = Date.now();
 
       if (!background) {
-        const results = this._executeLocally(query, values);
+        const results = await this._executeLocally(query, values);
         const msec = Date.now() - start;
         if ((msec > 100) || DEBUG_TO_LOG) {
           this._prettyConsoleLog(`${msec}msec: ${query}`);
@@ -350,16 +354,31 @@ class DatabaseStore extends NylasStore {
     });
   }
 
-  _executeLocally(query, values) {
+  async _executeLocally(query, values) {
     const fn = query.startsWith('SELECT') ? 'all' : 'run';
-    let tries = 0;
     let results = null;
+    const scheduler = new ExponentialBackoffScheduler({
+      baseDelay: BASE_RETRY_LOCK_DELAY,
+      maxDelay: MAX_RETRY_LOCK_DELAY,
+    })
+
+    const malformedStr = 'database disk image is malformed'
+    const schemaChangedStr = 'database schema has changed'
+
+    const retryableRegexp = new RegExp(
+      `(database is locked)||` +
+      `(${malformedStr})||` +
+      `(${schemaChangedStr})`,
+    'i')
 
     // Because other processes may be writing to the database and modifying the
     // schema (running ANALYZE, etc.), we may `prepare` a statement and then be
     // unable to execute it. Handle this case silently unless it's persistent.
     while (!results) {
       try {
+        // wait for the currentDelay before continuing
+        await new Promise((resolve) => setTimeout(resolve, scheduler.currentDelay()))
+
         let stmt = this._preparedStatementCache.get(query);
         if (!stmt) {
           stmt = this._db.prepare(query);
@@ -368,18 +387,22 @@ class DatabaseStore extends NylasStore {
         results = stmt[fn](values);
       } catch (err) {
         const errString = err.toString()
-        if (/database disk image is malformed/gi.test(errString)) {
-          // This is unrecoverable. We have to do a full database reset
-          NylasEnv.reportError(err)
-          Actions.resetEmailCache()
-        } else if (tries < 3 && /database schema has changed/gi.test(errString)) {
-          this._preparedStatementCache.del(query);
-          tries += 1;
-        } else {
+
+        if (scheduler.numTries() > 5 || !retryableRegexp.test(errString)) {
           // note: this function may throw a promise, which causes our Promise to reject
           throw new Error(`DatabaseStore: Query ${query}, ${JSON.stringify(values)} failed ${err.toString()}`);
         }
+
+        // Some errors require action before the query can be retried
+        if ((new RegExp(malformedStr, 'i')).test(errString)) {
+          // This is unrecoverable. We have to do a full database reset
+          NylasEnv.reportError(err)
+          Actions.resetEmailCache()
+        } else if ((new RegExp(schemaChangedStr, 'i')).test(errString)) {
+          this._preparedStatementCache.del(query);
+        }
       }
+      scheduler.nextDelay()
     }
 
     return results;