From 97fd550402165db626aba318c4505719da8e99e7 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 6 Feb 2022 13:49:33 +0100 Subject: [PATCH] add light anonymization option to the existing full anonymization --- bin/create-anonymization-script.js | 2 +- src/public/app/dialogs/options/advanced.js | 32 ++++++++++++++++++---- src/routes/api/database.js | 4 +-- src/routes/routes.js | 2 +- src/services/anonymization.js | 29 ++++++++++++++++---- 5 files changed, 55 insertions(+), 14 deletions(-) diff --git a/bin/create-anonymization-script.js b/bin/create-anonymization-script.js index 1517411b7..4c71d6071 100644 --- a/bin/create-anonymization-script.js +++ b/bin/create-anonymization-script.js @@ -4,4 +4,4 @@ const anonymizationService = require('../src/services/anonymization'); const fs = require('fs'); const path = require('path'); -fs.writeFileSync(path.resolve(__dirname, 'tpl', 'anonymize-database.sql'), anonymizationService.getAnonymizationScript()); +fs.writeFileSync(path.resolve(__dirname, 'tpl', 'anonymize-database.sql'), anonymizationService.getFullAnonymizationScript()); diff --git a/src/public/app/dialogs/options/advanced.js b/src/public/app/dialogs/options/advanced.js index 1c8c1b10e..322628300 100644 --- a/src/public/app/dialogs/options/advanced.js +++ b/src/public/app/dialogs/options/advanced.js @@ -25,10 +25,20 @@ const TPL = `

Anonymize database

+
Full anonymization
+

This action will create a new copy of the database and anonymize it (remove all note content and leave only structure and some non-sensitive metadata) for sharing online for debugging purposes without fear of leaking your personal data.

-

+

+ +
Light anonymization
+ +

This action will create a new copy of the database and do a light anonymization on it - specifically only content of all notes will be removed, but titles and attributes will remaing. Additionally, custom JS frontend/backend script notes and custom widgets will remain. This provides more context to debug the issues.

+ +

You can decide yourself if you want to provide fully or lightly anonymized database. Even fully anonymized DB is very useful, however in some cases lightly anonymized database can speed up the process of bug identification and fixing.

+ +

Vacuum database

@@ -42,7 +52,8 @@ export default class AdvancedOptions { this.$forceFullSyncButton = $("#force-full-sync-button"); this.$fillEntityChangesButton = $("#fill-entity-changes-button"); - this.$anonymizeButton = $("#anonymize-button"); + this.$anonymizeFullButton = $("#anonymize-full-button"); + this.$anonymizeLightButton = $("#anonymize-light-button"); this.$vacuumDatabaseButton = $("#vacuum-database-button"); this.$findAndFixConsistencyIssuesButton = $("#find-and-fix-consistency-issues-button"); this.$checkIntegrityButton = $("#check-integrity-button"); @@ -59,14 +70,25 @@ export default class AdvancedOptions { toastService.showMessage("Sync rows filled successfully"); }); - this.$anonymizeButton.on('click', async () => { - const resp = await server.post('database/anonymize'); + this.$anonymizeFullButton.on('click', async () => { + const resp = await server.post('database/anonymize/full'); if (!resp.success) { toastService.showError("Could not create anonymized database, check backend logs for details"); } else { - toastService.showMessage(`Created anonymized database in ${resp.anonymizedFilePath}`, 10000); + toastService.showMessage(`Created fully anonymized database in ${resp.anonymizedFilePath}`, 10000); + } + }); + + this.$anonymizeLightButton.on('click', async () => { + const resp = await server.post('database/anonymize/light'); + + if (!resp.success) { + toastService.showError("Could not create anonymized database, check backend logs for details"); + } + else { + toastService.showMessage(`Created lightly anonymized database in ${resp.anonymizedFilePath}`, 10000); } }); diff --git a/src/routes/api/database.js b/src/routes/api/database.js index 40b22f9cf..0f008b56c 100644 --- a/src/routes/api/database.js +++ b/src/routes/api/database.js @@ -6,8 +6,8 @@ const backupService = require('../../services/backup'); const anonymizationService = require('../../services/anonymization'); const consistencyChecksService = require('../../services/consistency_checks'); -async function anonymize() { - return await anonymizationService.createAnonymizedCopy(); +async function anonymize(req) { + return await anonymizationService.createAnonymizedCopy(req.params.type); } async function backupDatabase() { diff --git a/src/routes/routes.js b/src/routes/routes.js index 9681211e8..aee443e1a 100644 --- a/src/routes/routes.js +++ b/src/routes/routes.js @@ -324,7 +324,7 @@ function register(app) { apiRoute(GET, '/api/sql/schema', sqlRoute.getSchema); apiRoute(POST, '/api/sql/execute/:noteId', sqlRoute.execute); - route(POST, '/api/database/anonymize', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.anonymize, apiResultHandler, false); + route(POST, '/api/database/anonymize/:type', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.anonymize, apiResultHandler, false); // backup requires execution outside of transaction route(POST, '/api/database/backup-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.backupDatabase, apiResultHandler, false); diff --git a/src/services/anonymization.js b/src/services/anonymization.js index 5dcb501de..998a8ce86 100644 --- a/src/services/anonymization.js +++ b/src/services/anonymization.js @@ -5,7 +5,7 @@ const dateUtils = require("./date_utils"); const Database = require("better-sqlite3"); const sql = require("./sql"); -function getAnonymizationScript() { +function getFullAnonymizationScript() { // we want to delete all non-builtin attributes because they can contain sensitive names and values // on the other hand builtin/system attrs should not contain any sensitive info const builtinAttrNames = BUILTIN_ATTRIBUTES @@ -33,18 +33,37 @@ VACUUM; return anonymizeScript; } -async function createAnonymizedCopy() { +function getLightAnonymizationScript() { + return ` + UPDATE note_contents SET content = 'text' WHERE content IS NOT NULL AND noteId NOT IN ( + SELECT noteId FROM notes WHERE mime IN ('application/javascript;env=backend', 'application/javascript;env=frontend') + ); + UPDATE note_revision_contents SET content = 'text' WHERE content IS NOT NULL AND noteRevisionId NOT IN ( + SELECT noteRevisionId FROM note_revisions WHERE mime IN ('application/javascript;env=backend', 'application/javascript;env=frontend') + ); + `; +} + +async function createAnonymizedCopy(type) { + if (!['full', 'light'].includes(type)) { + throw new Error(`Unrecognized anonymization type '${type}'`); + } + if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) { fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700); } - const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db"; + const anonymizedFile = `${dataDir.ANONYMIZED_DB_DIR}/anonymized-${type}-${dateUtils.getDateTimeForFile()}.db`; await sql.copyDatabase(anonymizedFile); const db = new Database(anonymizedFile); - db.exec(getAnonymizationScript()); + const anonymizationScript = type === 'light' + ? getLightAnonymizationScript() + : getFullAnonymizationScript(); + + db.exec(anonymizationScript); db.close(); @@ -55,6 +74,6 @@ async function createAnonymizedCopy() { } module.exports = { - getAnonymizationScript, + getFullAnonymizationScript, createAnonymizedCopy }