From 784cd62df17d3acb3fe38820b823fa181f5e93a8 Mon Sep 17 00:00:00 2001 From: azivner Date: Sat, 6 Jan 2018 15:56:00 -0500 Subject: [PATCH] image sync --- package-lock.json | 59 ++++++++++++++++++- package.json | 1 + routes/api/image.js | 26 +++++---- routes/api/sync.js | 17 ++++++ services/consistency_checks.js | 5 +- services/content_hash.js | 101 ++++++++++++++++++++------------- services/sync.js | 10 ++++ services/sync_table.js | 6 ++ services/sync_update.js | 21 ++++++- services/utils.js | 11 +++- 10 files changed, 201 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index 295617684..3dcdc40dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4167,6 +4167,34 @@ "es5-ext": "0.10.35" } }, + "exec-buffer": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/exec-buffer/-/exec-buffer-3.2.0.tgz", + "integrity": "sha512-wsiD+2Tp6BWHoVv3B+5Dcx6E7u5zky+hUwOHjuH2hKSLR3dvRmX8fk8UD8uqQixHs4Wk6eDmiegVrMPjKj7wpA==", + "requires": { + "execa": "0.7.0", + "p-finally": "1.0.0", + "pify": "3.0.0", + "rimraf": "2.6.2", + "tempfile": "2.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" + }, + "tempfile": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-2.0.0.tgz", + "integrity": "sha1-awRGhWqbERTRhW/8vlCczLCXcmU=", + "requires": { + "temp-dir": "1.0.0", + "uuid": "3.1.0" + } + } + } + }, "exec-series": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/exec-series/-/exec-series-1.0.3.tgz", @@ -4180,7 +4208,6 @@ "version": "0.7.0", "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", - "dev": true, "requires": { "cross-spawn": "5.1.0", "get-stream": "3.0.0", @@ -5461,6 +5488,16 @@ } } }, + "imagemin-pngquant": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/imagemin-pngquant/-/imagemin-pngquant-5.0.1.tgz", + "integrity": "sha1-2KMp2lU6+iJrEc5i3r4Lfje0OeY=", + "requires": { + "exec-buffer": "3.2.0", + "is-png": "1.1.0", + "pngquant-bin": "3.1.1" + } + }, "import-lazy": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", @@ -5826,6 +5863,11 @@ "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", "dev": true }, + "is-png": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-png/-/is-png-1.1.0.tgz", + "integrity": "sha1-1XSxK/J1wDUEVVcLDltXqwYgd84=" + }, "is-posix-bracket": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz", @@ -7826,6 +7868,16 @@ "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.3.1.tgz", "integrity": "sha512-ggXCTsqHRIsGMkHlCEhbHhUmNTA2r1lpkE0NL4Q9S8spkXbm4vE9TVmPso2AGYn90Gltdz8W5CyzhcIGg2Gejg==" }, + "pngquant-bin": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/pngquant-bin/-/pngquant-bin-3.1.1.tgz", + "integrity": "sha1-0STZinWpSH9AwWQLTb/Lsr1aH9E=", + "requires": { + "bin-build": "2.2.0", + "bin-wrapper": "3.0.2", + "logalot": "2.1.0" + } + }, "postcss": { "version": "5.2.18", "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz", @@ -10309,6 +10361,11 @@ } } }, + "temp-dir": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz", + "integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=" + }, "tempfile": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz", diff --git a/package.json b/package.json index 414cb6898..1667cc6ce 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "html": "^1.0.0", "imagemin": "^5.3.1", "imagemin-mozjpeg": "^7.0.0", + "imagemin-pngquant": "^5.0.1", "ini": "^1.3.4", "jimp": "^0.2.28", "multer": "^1.3.0", diff --git a/routes/api/image.js b/routes/api/image.js index 766e0c720..21ab45814 100644 --- a/routes/api/image.js +++ b/routes/api/image.js @@ -5,6 +5,7 @@ const router = express.Router(); const sql = require('../../services/sql'); const auth = require('../../services/auth'); const utils = require('../../services/utils'); +const sync_table = require('../../services/sync_table'); const multer = require('multer')(); const imagemin = require('imagemin'); const imageminMozJpeg = require('imagemin-mozjpeg'); @@ -24,6 +25,7 @@ router.get('/:imageId/:filename', auth.checkApiAuth, async (req, res, next) => { }); router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, res, next) => { + const sourceId = req.headers.source_id; const file = req.file; const imageId = utils.newNoteId(); @@ -37,15 +39,19 @@ router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, r const resizedImage = await resize(file.buffer); const optimizedImage = await optimize(resizedImage); - await sql.insert("images", { - image_id: imageId, - format: file.mimetype.substr(6), - name: file.originalname, - checksum: utils.hash(optimizedImage), - data: optimizedImage, - is_deleted: 0, - date_modified: now, - date_created: now + await sql.doInTransaction(async () => { + await sql.insert("images", { + image_id: imageId, + format: file.mimetype.substr(6), + name: file.originalname, + checksum: utils.hash(optimizedImage), + data: optimizedImage, + is_deleted: 0, + date_modified: now, + date_created: now + }); + + await sync_table.addImageSync(imageId, sourceId); }); res.send({ @@ -60,8 +66,6 @@ const MAX_BYTE_SIZE = 200000; // images should have under 100 KBs async function resize(buffer) { const image = await jimp.read(buffer); - console.log("Size: ", buffer.byteLength); - if (image.bitmap.width > image.bitmap.height && image.bitmap.width > MAX_SIZE) { image.resize(MAX_SIZE, jimp.AUTO); } diff --git a/routes/api/sync.js b/routes/api/sync.js index 1b605ccff..8645c4ec1 100644 --- a/routes/api/sync.js +++ b/routes/api/sync.js @@ -122,6 +122,17 @@ router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, async (req, res, next res.send(await sql.getFirst("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId])); }); +router.get('/images/:imageId', auth.checkApiAuth, async (req, res, next) => { + const imageId = req.params.imageId; + const entity = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [imageId]); + + if (entity && entity.data !== null) { + entity.data = entity.data.toString('base64'); + } + + res.send(entity); +}); + router.put('/notes', auth.checkApiAuth, async (req, res, next) => { await syncUpdate.updateNote(req.body.entity, req.body.sourceId); @@ -158,4 +169,10 @@ router.put('/recent_notes', auth.checkApiAuth, async (req, res, next) => { res.send({}); }); +router.put('/images', auth.checkApiAuth, async (req, res, next) => { + await syncUpdate.updateImage(req.body.entity, req.body.sourceId); + + res.send({}); +}); + module.exports = router; \ No newline at end of file diff --git a/services/consistency_checks.js b/services/consistency_checks.js index 8e12d8e24..9f4754336 100644 --- a/services/consistency_checks.js +++ b/services/consistency_checks.js @@ -4,8 +4,11 @@ const sql = require('./sql'); const log = require('./log'); const messaging = require('./messaging'); const sync_mutex = require('./sync_mutex'); +const utils = require('./utils'); async function runCheck(query, errorText, errorList) { + utils.assertArguments(query, errorText, errorList); + const result = await sql.getFirstColumn(query); if (result.length > 0) { @@ -138,7 +141,7 @@ async function runAllChecks() { WHERE (SELECT COUNT(*) FROM notes_tree WHERE notes.note_id = notes_tree.note_id AND notes_tree.is_deleted = 0) = 0 AND notes.is_deleted = 0 - `,); + `, 'No undeleted note trees for note IDs', errorList); await runCheck(` SELECT diff --git a/services/content_hash.js b/services/content_hash.js index c6798026c..ddfbc2fbc 100644 --- a/services/content_hash.js +++ b/services/content_hash.js @@ -19,51 +19,70 @@ async function getHashes() { const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(','); const hashes = { - notes: getHash(await sql.getAll(`SELECT - note_id, - note_title, - note_text, - date_modified, - is_protected, - is_deleted - FROM notes - ORDER BY note_id`)), + notes: getHash(await sql.getAll(` + SELECT + note_id, + note_title, + note_text, + date_modified, + is_protected, + is_deleted + FROM notes + ORDER BY note_id`)), - notes_tree: getHash(await sql.getAll(`SELECT - note_tree_id, - note_id, - parent_note_id, - note_position, - date_modified, - is_deleted, - prefix - FROM notes_tree - ORDER BY note_tree_id`)), + notes_tree: getHash(await sql.getAll(` + SELECT + note_tree_id, + note_id, + parent_note_id, + note_position, + date_modified, + is_deleted, + prefix + FROM notes_tree + ORDER BY note_tree_id`)), - notes_history: getHash(await sql.getAll(`SELECT - note_history_id, - note_id, - note_title, - note_text, - date_modified_from, - date_modified_to - FROM notes_history - ORDER BY note_history_id`)), + notes_history: getHash(await sql.getAll(` + SELECT + note_history_id, + note_id, + note_title, + note_text, + date_modified_from, + date_modified_to + FROM notes_history + ORDER BY note_history_id`)), - recent_notes: getHash(await sql.getAll(`SELECT - note_tree_id, - note_path, - date_accessed, - is_deleted - FROM recent_notes - ORDER BY note_path`)), + recent_notes: getHash(await sql.getAll(` + SELECT + note_tree_id, + note_path, + date_accessed, + is_deleted + FROM recent_notes + ORDER BY note_path`)), - options: getHash(await sql.getAll(`SELECT - opt_name, - opt_value - FROM options - WHERE opt_name IN (${optionsQuestionMarks}) - ORDER BY opt_name`, options.SYNCED_OPTIONS)) + options: getHash(await sql.getAll(` + SELECT + opt_name, + opt_value + FROM options + WHERE opt_name IN (${optionsQuestionMarks}) + ORDER BY opt_name`, options.SYNCED_OPTIONS)), + + // we don't include image data on purpose because they are quite large, checksum is good enough + // to represent the data anyway + images: getHash(await sql.getAll(` + SELECT + image_id, + format, + checksum, + name, + is_deleted, + date_modified, + date_created + FROM images + ORDER BY image_id`)) }; const elapseTimeMs = new Date().getTime() - startTime.getTime(); diff --git a/services/sync.js b/services/sync.js index 17f0298d4..3b45d9faf 100644 --- a/services/sync.js +++ b/services/sync.js @@ -143,6 +143,9 @@ async function pullSync(syncContext) { else if (sync.entity_name === 'recent_notes') { await syncUpdate.updateRecentNotes(resp, syncContext.sourceId); } + else if (sync.entity_name === 'images') { + await syncUpdate.updateImage(resp, syncContext.sourceId); + } else { throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`); } @@ -214,6 +217,13 @@ async function pushEntity(sync, syncContext) { else if (sync.entity_name === 'recent_notes') { entity = await sql.getFirst('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]); } + else if (sync.entity_name === 'images') { + entity = await sql.getFirst('SELECT * FROM images WHERE image_id = ?', [sync.entity_id]); + + if (entity.data !== null) { + entity.data = entity.data.toString('base64'); + } + } else { throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`); } diff --git a/services/sync_table.js b/services/sync_table.js index b195d6c5f..0e53aa942 100644 --- a/services/sync_table.js +++ b/services/sync_table.js @@ -28,6 +28,10 @@ async function addRecentNoteSync(noteTreeId, sourceId) { await addEntitySync("recent_notes", noteTreeId, sourceId); } +async function addImageSync(imageId, sourceId) { + await addEntitySync("images", imageId, sourceId); +} + async function addEntitySync(entityName, entityId, sourceId) { await sql.replace("sync", { entity_name: entityName, @@ -78,6 +82,7 @@ async function fillAllSyncRows() { await fillSyncRows("notes_tree", "note_tree_id"); await fillSyncRows("notes_history", "note_history_id"); await fillSyncRows("recent_notes", "note_tree_id"); + await fillSyncRows("images", "image_id"); } module.exports = { @@ -87,6 +92,7 @@ module.exports = { addNoteHistorySync, addOptionsSync, addRecentNoteSync, + addImageSync, cleanupSyncRowsForMissingEntities, fillAllSyncRows }; \ No newline at end of file diff --git a/services/sync_update.js b/services/sync_update.js index ddd38fb90..08beaf53c 100644 --- a/services/sync_update.js +++ b/services/sync_update.js @@ -92,11 +92,30 @@ async function updateRecentNotes(entity, sourceId) { } } +async function updateImage(entity, sourceId) { + if (entity.data !== null) { + entity.data = Buffer.from(entity.data, 'base64'); + } + + const origImage = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [entity.image_id]); + + if (!origImage || origImage.date_modified <= entity.date_modified) { + await sql.doInTransaction(async () => { + await sql.replace("images", entity); + + await sync_table.addImageSync(entity.image_id, sourceId); + }); + + log.info("Update/sync image " + entity.image_id); + } +} + module.exports = { updateNote, updateNoteTree, updateNoteHistory, updateNoteReordering, updateOptions, - updateRecentNotes + updateRecentNotes, + updateImage }; \ No newline at end of file diff --git a/services/utils.js b/services/utils.js index ae166e566..512ebb477 100644 --- a/services/utils.js +++ b/services/utils.js @@ -79,6 +79,14 @@ function sanitizeSql(str) { return str.replace(/'/g, "\\'"); } +function assertArguments() { + for (const i in arguments) { + if (!arguments[i]) { + throw new Error(`Argument idx#${i} should not be falsy: ${arguments[i]}`); + } + } +} + module.exports = { randomSecureToken, randomString, @@ -95,5 +103,6 @@ module.exports = { hash, isEmptyOrWhitespace, getDateTimeForFile, - sanitizeSql + sanitizeSql, + assertArguments }; \ No newline at end of file