basic implementation of DB upgrades

src/app.py

@@ -14,6 +14,7 @@ from password_api import password_api
 from settings_api import settings_api
 from notes_history_api import notes_history_api
 from audit_api import audit_api
+from migration_api import migration_api, APP_DB_VERSION
 import config_provider
 import my_scrypt
 
@@ -37,6 +38,7 @@ app.register_blueprint(password_api)
 app.register_blueprint(settings_api)
 app.register_blueprint(notes_history_api)
 app.register_blueprint(audit_api)
+app.register_blueprint(migration_api)
 
 class User(UserMixin):
     pass
@@ -48,8 +50,18 @@ def login_form():
 @app.route('/app', methods=['GET'])
 @login_required
 def show_app():
+    db_version = int(getOption('db_version'))
+
+    if db_version != APP_DB_VERSION:
+        return redirect('migration')
+
     return render_template('app.html')
 
+@app.route('/migration', methods=['GET'])
+@login_required
+def show_migration():
+    return render_template('migration.html')
+
 @app.route('/logout', methods=['POST'])
 @login_required
 def logout():

src/backup.py

@@ -7,25 +7,29 @@ from shutil import copyfile
 import os
 import re
 
-def backup():
+def regular_backup():
     now = utils.nowTimestamp()
     last_backup_date = int(getOption('last_backup_date'))
 
     if now - last_backup_date > 43200:
-        config = config_provider.getConfig()
-
-        document_path = config['Document']['documentPath']
-        backup_directory = config['Backup']['backupDirectory']
-
-        date_str = datetime.utcnow().strftime("%Y-%m-%d %H:%M")
-
-        copyfile(document_path, backup_directory + "/" + "backup-" + date_str + ".db")
-
-        setOption('last_backup_date', now)
-        commit()
+        backup_now()
 
         cleanup_old_backups()
 
+def backup_now():
+    now = utils.nowTimestamp()
+
+    config = config_provider.getConfig()
+
+    document_path = config['Document']['documentPath']
+    backup_directory = config['Backup']['backupDirectory']
+
+    date_str = datetime.utcnow().strftime("%Y-%m-%d %H:%M")
+
+    copyfile(document_path, backup_directory + "/" + "backup-" + date_str + ".db")
+
+    setOption('last_backup_date', now)
+    commit()
 
 def cleanup_old_backups():
     now = datetime.utcnow()

src/migration_api.py

@@ -0,0 +1,72 @@
+import os
+import re
+
+import traceback
+
+from flask import Blueprint, jsonify
+from flask_login import login_required
+
+from sql import getOption, setOption, commit, execute_script
+
+import backup
+
+APP_DB_VERSION = 0
+
+MIGRATIONS_DIR = "src/migrations"
+
+migration_api = Blueprint('migration_api', __name__)
+
+@migration_api.route('/api/migration', methods = ['GET'])
+@login_required
+def getMigrationInfo():
+    return jsonify({
+        'db_version': int(getOption('db_version')),
+        'app_db_version': APP_DB_VERSION
+    })
+
+@migration_api.route('/api/migration', methods = ['POST'])
+@login_required
+def runMigration():
+    migrations = []
+
+    backup.backup_now()
+
+    current_db_version = int(getOption('db_version'))
+
+    for file in os.listdir(MIGRATIONS_DIR):
+        match = re.search(r"([0-9]{4})__([a-zA-Z0-9_ ]+)\.sql", file)
+
+        if match:
+            db_version = int(match.group(1))
+
+            if db_version > current_db_version:
+                name = match.group(2)
+
+                migration_record = {
+                    'db_version': db_version,
+                    'name': name
+                }
+
+                migrations.append(migration_record)
+
+                with open(MIGRATIONS_DIR + "/" + file, 'r') as sql_file:
+                    sql = sql_file.read()
+
+                    try:
+                        execute_script(sql)
+
+                        setOption('db_version', db_version)
+                        commit()
+
+                        migration_record['success'] = True
+                    except:
+                        migration_record['success'] = False
+                        migration_record['error'] = traceback.format_exc()
+
+                        break
+
+    migrations.sort(key=lambda x: x['db_version'])
+
+    return jsonify({
+        'migrations': migrations
+    })

src/sql.py

@@ -64,6 +64,11 @@ def execute(sql, params=[]):
     cursor.execute(sql, params)
     return cursor
 
+def execute_script(sql):
+    cursor = conn.cursor()
+    cursor.executescript(sql)
+    return cursor
+
 def getResults(sql, params=[]):
     cursor = conn.cursor()
     query = cursor.execute(sql, params)

src/templates/app.html

@@ -98,9 +98,9 @@
       <br/><br/>
 
       <p>
-        <button class="btn btn-sm" id="recentNotesJumpTo">Jump to</button>
+        <button class="btn btn-sm" id="recentNotesJumpTo">Jump to (enter)</button>
         &nbsp;
-        <button class="btn btn-sm" id="recentNotesAddLink">Add link</button>
+        <button class="btn btn-sm" id="recentNotesAddLink">Add link (l)</button>
       </p>
     </div>
 

src/templates/migration.html

@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Migration</title>
+  </head>
+  <body>
+    <div style="width: 800px; margin: auto;">
+        <h1>Migration</h1>
+
+        <div id="up-to-date" style="display:none;">
+          <p>Your database is up-to-date with the application.</p>
+        </div>
+
+        <div id="need-to-migrate" style="display:none;">
+          <p>Your database needs to be migrated to new version before you can use the application again.
+          Database will be backed up before migration in case of something going wrong.</p>
+
+          <table class="table table-bordered" style="width: 200px;">
+            <tr>
+              <th>Application version:</th>
+              <td id="app-db-version" style="text-align: right;"></td>
+            <tr>
+              <th>Database version:</th>
+              <td id="db-version" style="text-align: right;"></td>
+            </tr>
+          </table>
+
+          <button class="btn btn-warning" id="run-migration">Run migration</button>
+        </div>
+
+        <div id="migration-result" style="display:none;">
+          <h2>Migration result</h2>
+
+          <table id="migration-table" class="table">
+            <tr>
+              <th>Database version</th>
+              <th>Name</th>
+              <th>Success</th>
+              <th>Error</th>
+            </tr>
+          </table>
+        </div>
+    </div>
+
+    <script type="text/javascript">
+      const baseApiUrl = 'api/';
+    </script>
+
+    <script src="stat/lib/jquery.min.js"></script>
+
+    <link href="stat/lib/bootstrap/css/bootstrap.css" rel="stylesheet">
+    <script src="stat/lib/bootstrap/js/bootstrap.js"></script>
+
+    <script src="stat/js/migration.js"></script>
+  </body>
+</html>

src/tree_api.py

@@ -13,7 +13,7 @@ tree_api = Blueprint('tree_api', __name__)
 @tree_api.route('/api/tree', methods = ['GET'])
 @login_required
 def getTree():
-    backup.backup()
+    backup.regular_backup()
 
     notes = getResults("select "
                        "notes_tree.*, "

static/js/migration.js

@@ -0,0 +1,43 @@
+$(document).ready(() => {
+    $.get(baseApiUrl + 'migration').then(result => {
+        const appDbVersion = result.app_db_version;
+        const dbVersion = result.db_version;
+
+        if (appDbVersion === dbVersion) {
+            $("#up-to-date").show();
+        }
+        else {
+            $("#need-to-migrate").show();
+
+            $("#app-db-version").html(appDbVersion);
+            $("#db-version").html(dbVersion);
+        }
+    });
+});
+
+$("#run-migration").click(() => {
+    $("#run-migration").prop("disabled", true);
+
+    $("#migration-result").show();
+
+    $.ajax({
+        url: baseApiUrl + 'migration',
+        type: 'POST',
+        success: result => {
+            for (const migration of result.migrations) {
+                const row = $('<tr>')
+                        .append($('<td>').html(migration.db_version))
+                        .append($('<td>').html(migration.name))
+                        .append($('<td>').html(migration.success ? 'Yes' : 'No'))
+                        .append($('<td>').html(migration.success ? 'N/A' : migration.error));
+
+                if (!migration.success) {
+                    row.addClass("danger");
+                }
+
+                $("#migration-table").append(row);
+            }
+        },
+        error: () => alert("Migration failed with unknown error")
+    });
+});