generate anonymization script into a file

.gitpod.yml

@@ -2,7 +2,7 @@ image:
   file: .gitpod.dockerfile
 
 tasks:
-    - before: nvm install 16.13.1 && nvm use 16.13.1
+    - before: nvm install 16.13.2 && nvm use 16.13.2
       init: npm install
       command: npm run start-server
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16.13.1-alpine
+FROM node:16.13.2-alpine
 
 # Create app directory
 WORKDIR /usr/src/app

bin/build-linux-x64.sh

@@ -27,6 +27,8 @@ cp images/app-icons/png/128x128.png $BUILD_DIR/icon.png
 # removing software WebGL binaries because they are pretty huge and not necessary
 rm -r $BUILD_DIR/swiftshader
 
+cp bin/tpl/anonymize-database.sql $BUILD_DIR/
+
 cp bin/tpl/trilium-portable.sh $BUILD_DIR/
 chmod 755 $BUILD_DIR/trilium-portable.sh
 

bin/build-mac-x64.sh

@@ -23,6 +23,8 @@ rm -rf $BUILD_DIR
 # Mac build has by default useless directory level
 mv "./dist/Trilium Notes-darwin-x64" $BUILD_DIR
 
+cp bin/tpl/anonymize-database.sql $BUILD_DIR/
+
 echo "Zipping mac x64 electron distribution..."
 
 VERSION=`jq -r ".version" package.json`

bin/build-server.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 PKG_DIR=dist/trilium-linux-x64-server
-NODE_VERSION=16.13.1
+NODE_VERSION=16.13.2
 
 if [ "$1" != "DONTCOPY" ]
 then
@@ -20,12 +20,16 @@ rm -r $PKG_DIR/node/lib/node_modules/npm
 rm -r $PKG_DIR/node/include/node
 
 rm -r $PKG_DIR/node_modules/electron*
+rm -r $PKG_DIR/webpack*
+rm -r $PKG_DIR/electron.js
 
 cp -r bin/better-sqlite3/linux-server-better_sqlite3.node $PKG_DIR/node_modules/better-sqlite3/build/Release/better_sqlite3.node
 
 printf "#!/bin/sh\n./node/bin/node src/www" > $PKG_DIR/trilium.sh
 chmod 755 $PKG_DIR/trilium.sh
 
+cp bin/tpl/anonymize-database.sql $PKG_DIR/
+
 VERSION=`jq -r ".version" package.json`
 
 cd dist

bin/build-win-x64.sh

@@ -25,6 +25,8 @@ mv "./dist/Trilium Notes-win32-x64" $BUILD_DIR
 # removing software WebGL binaries because they are pretty huge and not necessary
 rm -r $BUILD_DIR/swiftshader
 
+cp bin/tpl/anonymize-database.sql $BUILD_DIR/
+
 cp bin/tpl/trilium-portable.bat $BUILD_DIR/
 cp bin/tpl/trilium-no-cert-check.bat $BUILD_DIR/
 cp bin/tpl/trilium-safe-mode.bat $BUILD_DIR/

bin/copy-trilium.sh

@@ -5,7 +5,7 @@ if [[ $# -eq 0 ]] ; then
     exit 1
 fi
 
-n exec 16.13.1 npm run webpack
+n exec 16.13.2 npm run webpack
 
 DIR=$1
 
@@ -27,7 +27,7 @@ cp -r electron.js $DIR/
 cp webpack-* $DIR/
 
 # run in subshell (so we return to original dir)
-(cd $DIR && n exec 16.13.1 npm install --only=prod)
+(cd $DIR && n exec 16.13.2 npm install --only=prod)
 
 # cleanup of useless files in dependencies
 rm -r $DIR/node_modules/image-q/demo

bin/create-anonymization-script.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+const anonymizationService = require('../src/services/anonymization');
+const fs = require('fs');
+const path = require('path');
+
+fs.writeFileSync(path.resolve(__dirname, 'tpl', 'anonymize-database.sql'), anonymizationService.getAnonymizationScript());

bin/generate-anonymization-script.js

@@ -1,31 +0,0 @@
-#!/usr/bin/env node
-
-const BUILTIN_ATTRIBUTES = require('../src/services/builtin_attributes');
-const fs = require('fs');
-const path = require('path');
-
-// we want to delete all non-builtin attributes because they can contain sensitive names and values
-// on the other hand builtin/system attrs should not contain any sensitive info
-const builtinAttrNames = BUILTIN_ATTRIBUTES
-    .map(attr => "'" + attr.name + "'").join(', ');
-
-const anonymizeScript = `
-UPDATE etapi_tokens SET tokenHash = 'API token hash value';
-UPDATE notes SET title = 'title';
-UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
-UPDATE note_revisions SET title = 'title';
-UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
-
-UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrNames});
-UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrNames});
-UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
-UPDATE options SET value = 'anonymized' WHERE 
-                    ('documentId', 'documentSecret', 'encryptedDataKey', 
-                     'passwordVerificationHash', 'passwordVerificationSalt', 
-                     'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
-                      AND value != '';
-
-VACUUM;
-`;
-
-fs.writeFileSync(path.resolve(__dirname, 'anonymize-database.sql'), anonymizeScript);

bin/tpl/anonymize-database.sql

@@ -8,7 +8,7 @@ UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
 UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
 UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN ('inbox', 'disableVersioning', 'calendarRoot', 'archived', 'excludeFromExport', 'disableInclusion', 'appCss', 'appTheme', 'hidePromotedAttributes', 'readOnly', 'autoReadOnlyDisabled', 'hoistedCssClass', 'cssClass', 'iconClass', 'keyboardShortcut', 'run', 'runOnInstance', 'runAtHour', 'customRequestHandler', 'customResourceProvider', 'widget', 'noteInfoWidgetDisabled', 'linkMapWidgetDisabled', 'noteRevisionsWidgetDisabled', 'whatLinksHereWidgetDisabled', 'similarNotesWidgetDisabled', 'workspace', 'workspaceIconClass', 'workspaceTabBackgroundColor', 'searchHome', 'hoistedInbox', 'hoistedSearchHome', 'sqlConsoleHome', 'datePattern', 'pageSize', 'viewType', 'mapRootNoteId', 'bookmarked', 'bookmarkFolder', 'sorted', 'top', 'fullContentWidth', 'shareHiddenFromTree', 'shareAlias', 'shareOmitDefaultCss', 'internalLink', 'imageLink', 'relationMapLink', 'includeMapLink', 'runOnNoteCreation', 'runOnNoteTitleChange', 'runOnNoteChange', 'runOnChildNoteCreation', 'runOnAttributeCreation', 'runOnAttributeChange', 'template', 'widget', 'renderNote', 'shareCss', 'shareJs', 'shareFavicon');
 UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
-UPDATE options SET value = 'anonymized' WHERE 
+UPDATE options SET value = 'anonymized' WHERE name IN
                     ('documentId', 'documentSecret', 'encryptedDataKey', 
                      'passwordVerificationHash', 'passwordVerificationSalt', 
                      'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "trilium",
-  "version": "0.49.4",
+  "version": "0.49.5",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "trilium",
-      "version": "0.49.4",
+      "version": "0.49.5",
       "license": "AGPL-3.0-only",
       "dependencies": {
         "@electron/remote": "2.0.1",
@@ -62,7 +62,7 @@
         "tmp": "0.2.1",
         "turndown": "7.1.1",
         "unescape": "1.0.1",
-        "ws": "8.4.0",
+        "ws": "8.4.1",
         "yauzl": "2.10.0"
       },
       "bin": {
@@ -11045,9 +11045,9 @@
       }
     },
     "node_modules/ws": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz",
-      "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==",
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.1.tgz",
+      "integrity": "sha512-6eqQ4yN2y2xv8b+BgbkUzPPyfo/PDl3VOWb06ZE0jIFYwuHMsMQN6F7o84yxJYCblfCRAxzpU59We4Rr4w0Luw==",
       "engines": {
         "node": ">=10.0.0"
       },
@@ -19881,9 +19881,9 @@
       }
     },
     "ws": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.0.tgz",
-      "integrity": "sha512-IHVsKe2pjajSUIl4KYMQOdlyliovpEPquKkqbwswulszzI7r0SfQrxnXdWAEqOlDCLrVSJzo+O1hAwdog2sKSQ==",
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.4.1.tgz",
+      "integrity": "sha512-6eqQ4yN2y2xv8b+BgbkUzPPyfo/PDl3VOWb06ZE0jIFYwuHMsMQN6F7o84yxJYCblfCRAxzpU59We4Rr4w0Luw==",
       "requires": {}
     },
     "xdg-basedir": {

src/becca/entities/etapi_token.js

@@ -2,14 +2,13 @@
 
 const dateUtils = require('../../services/date_utils');
 const AbstractEntity = require("./abstract_entity");
-const sql = require("../../services/sql.js");
 
 /**
  * EtapiToken is an entity representing token used to authenticate against Trilium REST API from client applications.
  * Used by:
  * - Trilium Sender
  * - ETAPI clients
- * 
+ *
  * The format user is presented with is "<etapiTokenId>_<tokenHash>". This is also called "authToken" to distinguish it
  * from tokenHash and token.
  */
@@ -42,10 +41,10 @@ class EtapiToken extends AbstractEntity {
         this.utcDateModified = row.utcDateModified || this.utcDateCreated;
         /** @type {boolean} */
         this.isDeleted = !!row.isDeleted;
-        
+
         this.becca.etapiTokens[this.etapiTokenId] = this;
     }
-    
+
     init() {
         if (this.etapiTokenId) {
             this.becca.etapiTokens[this.etapiTokenId] = this;

src/etapi/auth.js

@@ -1,7 +1,7 @@
 const becca = require("../becca/becca");
 const eu = require("./etapi_utils");
-const passwordEncryptionService = require("../services/password_encryption.js");
-const etapiTokenService = require("../services/etapi_tokens.js");
+const passwordEncryptionService = require("../services/password_encryption");
+const etapiTokenService = require("../services/etapi_tokens");
 
 function register(router) {
     eu.NOT_AUTHENTICATED_ROUTE(router, 'post', '/etapi/auth/login', (req, res, next) => {

src/etapi/etapi_utils.js

@@ -2,8 +2,8 @@ const cls = require("../services/cls");
 const sql = require("../services/sql");
 const log = require("../services/log");
 const becca = require("../becca/becca");
-const etapiTokenService = require("../services/etapi_tokens.js");
-const config = require("../services/config.js");
+const etapiTokenService = require("../services/etapi_tokens");
+const config = require("../services/config");
 const GENERIC_CODE = "GENERIC";
 
 const noAuthentication = config.General && config.General.noAuthentication === true;
@@ -11,7 +11,7 @@ const noAuthentication = config.General && config.General.noAuthentication === t
 class EtapiError extends Error {
     constructor(statusCode, code, message) {
         super();
-        
+
         this.statusCode = statusCode;
         this.code = code;
         this.message = message;
@@ -72,7 +72,7 @@ function NOT_AUTHENTICATED_ROUTE(router, method, path, routeHandler) {
 
 function getAndCheckNote(noteId) {
     const note = becca.getNote(noteId);
-    
+
     if (note) {
         return note;
     }
@@ -118,7 +118,7 @@ function validateAndPatch(target, source, allowedProperties) {
             }
         }
     }
-    
+
     // validation passed, let's patch
     for (const propName of Object.keys(source)) {
         target[propName] = source[propName];

src/routes/api/database.js

@@ -3,10 +3,11 @@
 const sql = require('../../services/sql');
 const log = require('../../services/log');
 const backupService = require('../../services/backup');
+const anonymizationService = require('../../services/anonymization');
 const consistencyChecksService = require('../../services/consistency_checks');
 
 async function anonymize() {
-    return await backupService.anonymize();
+    return await anonymizationService.createAnonymizedCopy();
 }
 
 async function backupDatabase() {

src/services/anonymization.js

@@ -0,0 +1,60 @@
+const BUILTIN_ATTRIBUTES = require("./builtin_attributes");
+const fs = require("fs-extra");
+const dataDir = require("./data_dir");
+const dateUtils = require("./date_utils");
+const Database = require("better-sqlite3");
+const sql = require("./sql");
+
+function getAnonymizationScript() {
+    // we want to delete all non-builtin attributes because they can contain sensitive names and values
+// on the other hand builtin/system attrs should not contain any sensitive info
+    const builtinAttrNames = BUILTIN_ATTRIBUTES
+        .map(attr => "'" + attr.name + "'").join(', ');
+
+    const anonymizeScript = `
+UPDATE etapi_tokens SET tokenHash = 'API token hash value';
+UPDATE notes SET title = 'title';
+UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL;
+UPDATE note_revisions SET title = 'title';
+UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL;
+
+UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrNames});
+UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrNames});
+UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL;
+UPDATE options SET value = 'anonymized' WHERE name IN
+                    ('documentId', 'documentSecret', 'encryptedDataKey', 
+                     'passwordVerificationHash', 'passwordVerificationSalt', 
+                     'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
+                      AND value != '';
+
+VACUUM;
+`;
+
+    return anonymizeScript;
+}
+
+async function createAnonymizedCopy() {
+    if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
+        fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
+    }
+
+    const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
+
+    await sql.copyDatabase(anonymizedFile);
+
+    const db = new Database(anonymizedFile);
+
+    db.exec(getAnonymizationScript());
+
+    db.close();
+
+    return {
+        success: true,
+        anonymizedFilePath: anonymizedFile
+    };
+}
+
+module.exports = {
+    getAnonymizationScript,
+    createAnonymizedCopy
+}

src/services/backup.js

@@ -6,9 +6,8 @@ const fs = require('fs-extra');
 const dataDir = require('./data_dir');
 const log = require('./log');
 const syncMutexService = require('./sync_mutex');
-const attributeService = require('./attributes');
 const cls = require('./cls');
-const Database = require('better-sqlite3');
+const sql = require('./sql');
 
 function regularBackup() {
     cls.init(() => {
@@ -41,23 +40,12 @@ function periodBackup(optionName, backupType, periodInSeconds) {
     }
 }
 
-async function copyFile(backupFile) {
-    const sql = require('./sql');
-
-    try {
-        fs.unlinkSync(backupFile);
-    } catch (e) {
-    } // unlink throws exception if the file did not exist
-
-    await sql.dbConnection.backup(backupFile);
-}
-
 async function backupNow(name) {
-    // we don't want to backup DB in the middle of sync with potentially inconsistent DB state
+    // we don't want to back up DB in the middle of sync with potentially inconsistent DB state
     return await syncMutexService.doExclusively(async () => {
         const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
 
-        await copyFile(backupFile);
+        await sql.copyDatabase(backupFile);
 
         log.info("Created backup at " + backupFile);
 
@@ -65,53 +53,11 @@ async function backupNow(name) {
     });
 }
 
-async function anonymize() {
-    if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
-        fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
-    }
-
-    const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
-
-    await copyFile(anonymizedFile);
-
-    const db = new Database(anonymizedFile);
-
-    db.prepare("UPDATE etapi_tokens SET tokenHash = 'API token hash value'").run();
-    db.prepare("UPDATE notes SET title = 'title'").run();
-    db.prepare("UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL").run();
-    db.prepare("UPDATE note_revisions SET title = 'title'").run();
-    db.prepare("UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL").run();
-
-    // we want to delete all non-builtin attributes because they can contain sensitive names and values
-    // on the other hand builtin/system attrs should not contain any sensitive info
-    const builtinAttrs = attributeService
-        .getBuiltinAttributeNames()
-        .map(name => "'" + name + "'").join(', ');
-
-    db.prepare(`UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label' AND name NOT IN(${builtinAttrs})`).run();
-    db.prepare(`UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name NOT IN (${builtinAttrs})`).run();
-    db.prepare("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL").run();
-    db.prepare(`UPDATE options SET value = 'anonymized' WHERE name IN 
-                    ('documentId', 'documentSecret', 'encryptedDataKey', 
-                     'passwordVerificationHash', 'passwordVerificationSalt', 
-                     'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy') 
-                      AND value != ''`).run();
-    db.prepare("VACUUM").run();
-
-    db.close();
-
-    return {
-        success: true,
-        anonymizedFilePath: anonymizedFile
-    };
-}
-
 if (!fs.existsSync(dataDir.BACKUP_DIR)) {
     fs.mkdirSync(dataDir.BACKUP_DIR, 0o700);
 }
 
 module.exports = {
     backupNow,
-    anonymize,
     regularBackup
 };

src/services/sql.js

@@ -8,6 +8,7 @@ const log = require('./log');
 const Database = require('better-sqlite3');
 const dataDir = require('./data_dir');
 const cls = require('./cls');
+const fs = require("fs-extra");
 
 const dbConnection = new Database(dataDir.DOCUMENT_PATH);
 dbConnection.pragma('journal_mode = WAL');
@@ -276,6 +277,15 @@ function fillParamList(paramIds, truncate = true) {
     s.run(paramIds);
 }
 
+async function copyDatabase(targetFilePath) {
+    try {
+        fs.unlinkSync(targetFilePath);
+    } catch (e) {
+    } // unlink throws exception if the file did not exist
+
+    await dbConnection.backup(targetFilePath);
+}
+
 module.exports = {
     dbConnection,
     insert,
@@ -347,5 +357,6 @@ module.exports = {
     executeScript,
     transactional,
     upsert,
-    fillParamList
+    fillParamList,
+    copyDatabase
 };