zadam/trilium
1
1
Fork 0
mirror of https://github.com/zadam/trilium.git synced 2025-01-01 12:52:17 +08:00
Code Issues Projects Releases Packages Wiki Activity

image sync

Browse source
This commit is contained in:
azivner 2018-01-06 15:56:00 -05:00
parent 91cf090820
commit 784cd62df1
10 changed files with 201 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
package-lock.json generated
View file

@ -4167,6 +4167,34 @@
"es5-ext": "0.10.35"
}
},
"exec-buffer": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/exec-buffer/-/exec-buffer-3.2.0.tgz",
"integrity": "sha512-wsiD+2Tp6BWHoVv3B+5Dcx6E7u5zky+hUwOHjuH2hKSLR3dvRmX8fk8UD8uqQixHs4Wk6eDmiegVrMPjKj7wpA==",
"requires": {
"execa": "0.7.0",
"p-finally": "1.0.0",
"pify": "3.0.0",
"rimraf": "2.6.2",
"tempfile": "2.0.0"
},
"dependencies": {
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
},
"tempfile": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/tempfile/-/tempfile-2.0.0.tgz",
"integrity": "sha1-awRGhWqbERTRhW/8vlCczLCXcmU=",
"requires": {
"temp-dir": "1.0.0",
"uuid": "3.1.0"
}
}
}
},
"exec-series": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/exec-series/-/exec-series-1.0.3.tgz",
@ -4180,7 +4208,6 @@
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
"integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
"dev": true,
"requires": {
"cross-spawn": "5.1.0",
"get-stream": "3.0.0",
@ -5461,6 +5488,16 @@
}
}
},
"imagemin-pngquant": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/imagemin-pngquant/-/imagemin-pngquant-5.0.1.tgz",
"integrity": "sha1-2KMp2lU6+iJrEc5i3r4Lfje0OeY=",
"requires": {
"exec-buffer": "3.2.0",
"is-png": "1.1.0",
"pngquant-bin": "3.1.1"
}
},
"import-lazy": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz",
@ -5826,6 +5863,11 @@
"integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
"dev": true
},
"is-png": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/is-png/-/is-png-1.1.0.tgz",
"integrity": "sha1-1XSxK/J1wDUEVVcLDltXqwYgd84="
},
"is-posix-bracket": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz",
@ -7826,6 +7868,16 @@
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.3.1.tgz",
"integrity": "sha512-ggXCTsqHRIsGMkHlCEhbHhUmNTA2r1lpkE0NL4Q9S8spkXbm4vE9TVmPso2AGYn90Gltdz8W5CyzhcIGg2Gejg=="
},
"pngquant-bin": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/pngquant-bin/-/pngquant-bin-3.1.1.tgz",
"integrity": "sha1-0STZinWpSH9AwWQLTb/Lsr1aH9E=",
"requires": {
"bin-build": "2.2.0",
"bin-wrapper": "3.0.2",
"logalot": "2.1.0"
}
},
"postcss": {
"version": "5.2.18",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
@ -10309,6 +10361,11 @@
}
}
},
"temp-dir": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz",
"integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0="
},
"tempfile": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz",

1
package.json
View file

@ -35,6 +35,7 @@
"html": "^1.0.0",
"imagemin": "^5.3.1",
"imagemin-mozjpeg": "^7.0.0",
"imagemin-pngquant": "^5.0.1",
"ini": "^1.3.4",
"jimp": "^0.2.28",
"multer": "^1.3.0",

26
routes/api/image.js
View file

@ -5,6 +5,7 @@ const router = express.Router();
const sql = require('../../services/sql');
const auth = require('../../services/auth');
const utils = require('../../services/utils');
const sync_table = require('../../services/sync_table');
const multer = require('multer')();
const imagemin = require('imagemin');
const imageminMozJpeg = require('imagemin-mozjpeg');
@ -24,6 +25,7 @@ router.get('/:imageId/:filename', auth.checkApiAuth, async (req, res, next) => {
});
router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, res, next) => {
const sourceId = req.headers.source_id;
const file = req.file;
const imageId = utils.newNoteId();
@ -37,15 +39,19 @@ router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, r
const resizedImage = await resize(file.buffer);
const optimizedImage = await optimize(resizedImage);
await sql.insert("images", {
image_id: imageId,
format: file.mimetype.substr(6),
name: file.originalname,
checksum: utils.hash(optimizedImage),
data: optimizedImage,
is_deleted: 0,
date_modified: now,
date_created: now
await sql.doInTransaction(async () => {
await sql.insert("images", {
image_id: imageId,
format: file.mimetype.substr(6),
name: file.originalname,
checksum: utils.hash(optimizedImage),
data: optimizedImage,
is_deleted: 0,
date_modified: now,
date_created: now
});
await sync_table.addImageSync(imageId, sourceId);
});
res.send({
@ -60,8 +66,6 @@ const MAX_BYTE_SIZE = 200000; // images should have under 100 KBs
async function resize(buffer) {
const image = await jimp.read(buffer);
console.log("Size: ", buffer.byteLength);
if (image.bitmap.width > image.bitmap.height && image.bitmap.width > MAX_SIZE) {
image.resize(MAX_SIZE, jimp.AUTO);
}

17
routes/api/sync.js
View file

@ -122,6 +122,17 @@ router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, async (req, res, next
res.send(await sql.getFirst("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId]));
});
router.get('/images/:imageId', auth.checkApiAuth, async (req, res, next) => {
const imageId = req.params.imageId;
const entity = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [imageId]);
if (entity && entity.data !== null) {
entity.data = entity.data.toString('base64');
}
res.send(entity);
});
router.put('/notes', auth.checkApiAuth, async (req, res, next) => {
await syncUpdate.updateNote(req.body.entity, req.body.sourceId);
@ -158,4 +169,10 @@ router.put('/recent_notes', auth.checkApiAuth, async (req, res, next) => {
res.send({});
});
router.put('/images', auth.checkApiAuth, async (req, res, next) => {
await syncUpdate.updateImage(req.body.entity, req.body.sourceId);
res.send({});
});
module.exports = router;

5
services/consistency_checks.js
View file

@ -4,8 +4,11 @@ const sql = require('./sql');
const log = require('./log');
const messaging = require('./messaging');
const sync_mutex = require('./sync_mutex');
const utils = require('./utils');
async function runCheck(query, errorText, errorList) {
utils.assertArguments(query, errorText, errorList);
const result = await sql.getFirstColumn(query);
if (result.length > 0) {
@ -138,7 +141,7 @@ async function runAllChecks() {
WHERE
(SELECT COUNT(*) FROM notes_tree WHERE notes.note_id = notes_tree.note_id AND notes_tree.is_deleted = 0) = 0
AND notes.is_deleted = 0
`,);
`, 'No undeleted note trees for note IDs', errorList);
await runCheck(`
SELECT

101
services/content_hash.js
View file

@ -19,51 +19,70 @@ async function getHashes() {
const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(',');
const hashes = {
notes: getHash(await sql.getAll(`SELECT
note_id,
note_title,
note_text,
date_modified,
is_protected,
is_deleted
FROM notes
ORDER BY note_id`)),
notes: getHash(await sql.getAll(`
SELECT
note_id,
note_title,
note_text,
date_modified,
is_protected,
is_deleted
FROM notes
ORDER BY note_id`)),
notes_tree: getHash(await sql.getAll(`SELECT
note_tree_id,
note_id,
parent_note_id,
note_position,
date_modified,
is_deleted,
prefix
FROM notes_tree
ORDER BY note_tree_id`)),
notes_tree: getHash(await sql.getAll(`
SELECT
note_tree_id,
note_id,
parent_note_id,
note_position,
date_modified,
is_deleted,
prefix
FROM notes_tree
ORDER BY note_tree_id`)),
notes_history: getHash(await sql.getAll(`SELECT
note_history_id,
note_id,
note_title,
note_text,
date_modified_from,
date_modified_to
FROM notes_history
ORDER BY note_history_id`)),
notes_history: getHash(await sql.getAll(`
SELECT
note_history_id,
note_id,
note_title,
note_text,
date_modified_from,
date_modified_to
FROM notes_history
ORDER BY note_history_id`)),
recent_notes: getHash(await sql.getAll(`SELECT
note_tree_id,
note_path,
date_accessed,
is_deleted
FROM recent_notes
ORDER BY note_path`)),
recent_notes: getHash(await sql.getAll(`
SELECT
note_tree_id,
note_path,
date_accessed,
is_deleted
FROM recent_notes
ORDER BY note_path`)),
options: getHash(await sql.getAll(`SELECT
opt_name,
opt_value
FROM options
WHERE opt_name IN (${optionsQuestionMarks})
ORDER BY opt_name`, options.SYNCED_OPTIONS))
options: getHash(await sql.getAll(`
SELECT
opt_name,
opt_value
FROM options
WHERE opt_name IN (${optionsQuestionMarks})
ORDER BY opt_name`, options.SYNCED_OPTIONS)),
// we don't include image data on purpose because they are quite large, checksum is good enough
// to represent the data anyway
images: getHash(await sql.getAll(`
SELECT
image_id,
format,
checksum,
name,
is_deleted,
date_modified,
date_created
FROM images
ORDER BY image_id`))
};
const elapseTimeMs = new Date().getTime() - startTime.getTime();

10
services/sync.js
View file

@ -143,6 +143,9 @@ async function pullSync(syncContext) {
else if (sync.entity_name === 'recent_notes') {
await syncUpdate.updateRecentNotes(resp, syncContext.sourceId);
}
else if (sync.entity_name === 'images') {
await syncUpdate.updateImage(resp, syncContext.sourceId);
}
else {
throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
}
@ -214,6 +217,13 @@ async function pushEntity(sync, syncContext) {
else if (sync.entity_name === 'recent_notes') {
entity = await sql.getFirst('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]);
}
else if (sync.entity_name === 'images') {
entity = await sql.getFirst('SELECT * FROM images WHERE image_id = ?', [sync.entity_id]);
if (entity.data !== null) {
entity.data = entity.data.toString('base64');
}
}
else {
throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
}

6
services/sync_table.js
View file

@ -28,6 +28,10 @@ async function addRecentNoteSync(noteTreeId, sourceId) {
await addEntitySync("recent_notes", noteTreeId, sourceId);
}
async function addImageSync(imageId, sourceId) {
await addEntitySync("images", imageId, sourceId);
}
async function addEntitySync(entityName, entityId, sourceId) {
await sql.replace("sync", {
entity_name: entityName,
@ -78,6 +82,7 @@ async function fillAllSyncRows() {
await fillSyncRows("notes_tree", "note_tree_id");
await fillSyncRows("notes_history", "note_history_id");
await fillSyncRows("recent_notes", "note_tree_id");
await fillSyncRows("images", "image_id");
}
module.exports = {
@ -87,6 +92,7 @@ module.exports = {
addNoteHistorySync,
addOptionsSync,
addRecentNoteSync,
addImageSync,
cleanupSyncRowsForMissingEntities,
fillAllSyncRows
};

21
services/sync_update.js
View file

@ -92,11 +92,30 @@ async function updateRecentNotes(entity, sourceId) {
}
}
async function updateImage(entity, sourceId) {
if (entity.data !== null) {
entity.data = Buffer.from(entity.data, 'base64');
}
const origImage = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [entity.image_id]);
if (!origImage || origImage.date_modified <= entity.date_modified) {
await sql.doInTransaction(async () => {
await sql.replace("images", entity);
await sync_table.addImageSync(entity.image_id, sourceId);
});
log.info("Update/sync image " + entity.image_id);
}
}
module.exports = {
updateNote,
updateNoteTree,
updateNoteHistory,
updateNoteReordering,
updateOptions,
updateRecentNotes
updateRecentNotes,
updateImage
};

11
services/utils.js
View file

@ -79,6 +79,14 @@ function sanitizeSql(str) {
return str.replace(/'/g, "\\'");
}
function assertArguments() {
for (const i in arguments) {
if (!arguments[i]) {
throw new Error(`Argument idx#${i} should not be falsy: ${arguments[i]}`);
}
}
}
module.exports = {
randomSecureToken,
randomString,
@ -95,5 +103,6 @@ module.exports = {
hash,
isEmptyOrWhitespace,
getDateTimeForFile,
sanitizeSql
sanitizeSql,
assertArguments
};