2017-10-22 09:10:33 +08:00
|
|
|
"use strict";
|
|
|
|
|
2018-04-03 08:46:46 +08:00
|
|
|
const dateUtils = require('./date_utils');
|
2018-04-02 09:27:46 +08:00
|
|
|
const optionService = require('./options');
|
2017-10-15 11:31:44 +08:00
|
|
|
const fs = require('fs-extra');
|
2017-10-24 11:30:23 +08:00
|
|
|
const dataDir = require('./data_dir');
|
2017-10-25 10:17:48 +08:00
|
|
|
const log = require('./log');
|
2018-04-03 09:25:20 +08:00
|
|
|
const sqlInit = require('./sql_init');
|
2018-04-02 09:27:46 +08:00
|
|
|
const syncMutexService = require('./sync_mutex');
|
2018-03-29 11:41:22 +08:00
|
|
|
const cls = require('./cls');
|
2020-06-03 05:13:55 +08:00
|
|
|
const sqlite = require('sqlite');
|
|
|
|
const sqlite3 = require('sqlite3');
|
2017-10-15 11:31:44 +08:00
|
|
|
|
|
|
|
async function regularBackup() {
|
2018-07-30 22:40:50 +08:00
|
|
|
await periodBackup('lastDailyBackupDate', 'daily', 24 * 3600);
|
|
|
|
|
|
|
|
await periodBackup('lastWeeklyBackupDate', 'weekly', 7 * 24 * 3600);
|
|
|
|
|
|
|
|
await periodBackup('lastMonthlyBackupDate', 'monthly', 30 * 24 * 3600);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function periodBackup(optionName, fileName, periodInSeconds) {
|
2017-12-11 01:56:59 +08:00
|
|
|
const now = new Date();
|
2018-07-30 22:40:50 +08:00
|
|
|
const lastDailyBackupDate = dateUtils.parseDateTime(await optionService.getOption(optionName));
|
2017-10-15 11:31:44 +08:00
|
|
|
|
2018-07-30 22:40:50 +08:00
|
|
|
if (now.getTime() - lastDailyBackupDate.getTime() > periodInSeconds * 1000) {
|
|
|
|
await backupNow(fileName);
|
2017-12-11 01:56:59 +08:00
|
|
|
|
2019-03-14 05:43:59 +08:00
|
|
|
await optionService.setOption(optionName, dateUtils.utcNowDateTime());
|
2017-10-15 11:31:44 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-03 05:13:55 +08:00
|
|
|
const COPY_ATTEMPT_COUNT = 50;
|
2020-05-31 16:24:59 +08:00
|
|
|
|
2020-06-03 05:13:55 +08:00
|
|
|
async function copyFile(backupFile) {
|
2020-05-30 03:55:08 +08:00
|
|
|
const sql = require('./sql');
|
|
|
|
|
2020-06-03 05:13:55 +08:00
|
|
|
try {
|
|
|
|
fs.unlinkSync(backupFile);
|
|
|
|
} catch (e) {
|
|
|
|
} // unlink throws exception if the file did not exist
|
|
|
|
|
|
|
|
let success = false;
|
|
|
|
let attemptCount = 0
|
|
|
|
|
|
|
|
for (; attemptCount < COPY_ATTEMPT_COUNT && !success; attemptCount++) {
|
|
|
|
try {
|
|
|
|
await sql.executeNoWrap(`VACUUM INTO '${backupFile}'`);
|
|
|
|
|
|
|
|
success = true;
|
|
|
|
} catch (e) {
|
|
|
|
log.info(`Copy DB attempt ${attemptCount + 1} failed with "${e.message}", retrying...`);
|
|
|
|
}
|
|
|
|
// we re-try since VACUUM is very picky and it can't run if there's any other query currently running
|
|
|
|
// which is difficult to guarantee so we just re-try
|
|
|
|
}
|
|
|
|
|
|
|
|
return attemptCount !== COPY_ATTEMPT_COUNT;
|
|
|
|
}
|
|
|
|
|
|
|
|
async function backupNow(name) {
|
2018-01-05 10:37:36 +08:00
|
|
|
// we don't want to backup DB in the middle of sync with potentially inconsistent DB state
|
2020-05-30 03:55:08 +08:00
|
|
|
return await syncMutexService.doExclusively(async () => {
|
2018-07-30 22:40:50 +08:00
|
|
|
const backupFile = `${dataDir.BACKUP_DIR}/backup-${name}.db`;
|
2017-10-25 10:17:48 +08:00
|
|
|
|
2020-06-03 18:16:16 +08:00
|
|
|
const success = await copyFile(backupFile);
|
2020-05-30 03:55:08 +08:00
|
|
|
|
2020-06-03 05:13:55 +08:00
|
|
|
if (success) {
|
2020-06-03 18:16:16 +08:00
|
|
|
log.info("Created backup at " + backupFile);
|
2020-05-30 03:55:08 +08:00
|
|
|
}
|
|
|
|
else {
|
2020-06-03 18:16:16 +08:00
|
|
|
log.error(`Creating backup ${backupFile} failed`);
|
2020-05-30 03:55:08 +08:00
|
|
|
}
|
2017-10-15 11:31:44 +08:00
|
|
|
|
2020-05-30 03:55:08 +08:00
|
|
|
return backupFile;
|
2017-10-15 11:31:44 +08:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2020-06-03 05:13:55 +08:00
|
|
|
async function anonymize() {
|
|
|
|
if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
|
|
|
|
fs.mkdirSync(dataDir.ANONYMIZED_DB_DIR, 0o700);
|
|
|
|
}
|
|
|
|
|
|
|
|
const anonymizedFile = dataDir.ANONYMIZED_DB_DIR + "/" + "anonymized-" + dateUtils.getDateTimeForFile() + ".db";
|
|
|
|
|
|
|
|
const success = await copyFile(anonymizedFile);
|
|
|
|
|
|
|
|
if (!success) {
|
|
|
|
return { success: false };
|
|
|
|
}
|
|
|
|
|
|
|
|
const db = await sqlite.open({
|
|
|
|
filename: anonymizedFile,
|
|
|
|
driver: sqlite3.Database
|
|
|
|
});
|
|
|
|
|
2020-06-03 15:55:05 +08:00
|
|
|
await db.run("UPDATE api_tokens SET token = 'API token value'");
|
2020-06-03 05:13:55 +08:00
|
|
|
await db.run("UPDATE notes SET title = 'title'");
|
|
|
|
await db.run("UPDATE note_contents SET content = 'text'");
|
|
|
|
await db.run("UPDATE note_revisions SET title = 'title'");
|
|
|
|
await db.run("UPDATE note_revision_contents SET content = 'title'");
|
|
|
|
await db.run("UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label'");
|
|
|
|
await db.run("UPDATE attributes SET name = 'name' WHERE type = 'relation' AND name != 'template'");
|
|
|
|
await db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL");
|
|
|
|
await db.run(`UPDATE options SET value = 'anonymized' WHERE name IN
|
2020-06-03 15:55:05 +08:00
|
|
|
('documentId', 'documentSecret', 'encryptedDataKey', 'passwordVerificationHash',
|
|
|
|
'passwordVerificationSalt', 'passwordDerivedKeySalt', 'username', 'syncServerHost', 'syncProxy')`);
|
2020-06-03 05:13:55 +08:00
|
|
|
await db.run("VACUUM");
|
|
|
|
|
|
|
|
await db.close();
|
|
|
|
|
|
|
|
return {
|
|
|
|
success: true,
|
|
|
|
anonymizedFilePath: anonymizedFile
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2017-10-29 00:23:11 +08:00
|
|
|
if (!fs.existsSync(dataDir.BACKUP_DIR)) {
|
|
|
|
fs.mkdirSync(dataDir.BACKUP_DIR, 0o700);
|
|
|
|
}
|
|
|
|
|
2018-04-03 09:25:20 +08:00
|
|
|
sqlInit.dbReady.then(() => {
|
2020-05-31 16:24:59 +08:00
|
|
|
setInterval(cls.wrap(regularBackup), 4 * 60 * 60 * 1000);
|
2017-10-29 00:23:11 +08:00
|
|
|
|
2020-05-31 16:24:59 +08:00
|
|
|
// kickoff first backup soon after start up
|
|
|
|
setTimeout(cls.wrap(regularBackup), 5 * 60 * 1000);
|
2017-12-04 08:18:33 +08:00
|
|
|
});
|
2017-10-29 00:23:11 +08:00
|
|
|
|
2017-10-15 11:31:44 +08:00
|
|
|
module.exports = {
|
2020-06-03 05:13:55 +08:00
|
|
|
backupNow,
|
|
|
|
anonymize
|
2020-05-30 03:55:08 +08:00
|
|
|
};
|