From 53c361945b3037ddb9c8a66154f58db52bd840dd Mon Sep 17 00:00:00 2001 From: zadam Date: Tue, 18 Aug 2020 22:20:47 +0200 Subject: [PATCH] fix setting mime after import + cleanup of old search code --- src/entities/note.js | 2 +- .../app/services/frontend_script_api.js | 2 +- src/public/app/widgets/type_widgets/image.js | 2 +- src/routes/api/files.js | 1 + src/routes/api/notes.js | 2 + src/services/build_search_query.js | 158 ------------------ src/services/image.js | 6 +- src/services/notes.js | 4 +- src/services/search.js | 89 ---------- src/services/search/parse_filters.js | 118 ------------- 10 files changed, 13 insertions(+), 371 deletions(-) delete mode 100644 src/services/build_search_query.js delete mode 100644 src/services/search.js delete mode 100644 src/services/search/parse_filters.js diff --git a/src/entities/note.js b/src/entities/note.js index 5daae38fa..2d49848e0 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -109,7 +109,7 @@ class Note extends Entity { return sql.getRow(` SELECT LENGTH(content) AS contentLength, - dateModified, + dateModified, utcDateModified FROM note_contents WHERE noteId = ?`, [this.noteId]); diff --git a/src/public/app/services/frontend_script_api.js b/src/public/app/services/frontend_script_api.js index f27d37f23..b12b04382 100644 --- a/src/public/app/services/frontend_script_api.js +++ b/src/public/app/services/frontend_script_api.js @@ -176,7 +176,7 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain * @returns {Promise} */ this.searchForNotes = async searchString => { - const noteIds = await this.runOnServer(async searchString => { + const noteIds = await this.runOnBackend(async searchString => { const notes = await api.searchForNotes(searchString); return notes.map(note => note.noteId); diff --git a/src/public/app/widgets/type_widgets/image.js b/src/public/app/widgets/type_widgets/image.js index 5e5ffc567..2009fb088 100644 --- a/src/public/app/widgets/type_widgets/image.js +++ b/src/public/app/widgets/type_widgets/image.js @@ -127,7 +127,7 @@ class ImageTypeWidget extends TypeWidget { this.$widget.show(); - const noteComplement = await this.tabContext.getNoteComplement(); + const noteComplement = await this.tabContext.getNoteComplement();console.log(noteComplement, note); this.$fileName.text(attributeMap.originalFileName || "?"); this.$fileSize.text(noteComplement.contentLength + " bytes"); diff --git a/src/routes/api/files.js b/src/routes/api/files.js index 4acc6cc71..84b3dd30c 100644 --- a/src/routes/api/files.js +++ b/src/routes/api/files.js @@ -18,6 +18,7 @@ function updateFile(req) { noteRevisionService.createNoteRevision(note); note.mime = file.mimetype.toLowerCase(); + note.save(); note.setContent(file.buffer); diff --git a/src/routes/api/notes.js b/src/routes/api/notes.js index 4da8a3188..616bce2bc 100644 --- a/src/routes/api/notes.js +++ b/src/routes/api/notes.js @@ -25,6 +25,8 @@ function getNote(req) { const contentMetadata = note.getContentMetadata(); + note.contentLength = contentMetadata.contentLength; + note.combinedUtcDateModified = note.utcDateModified > contentMetadata.utcDateModified ? note.utcDateModified : contentMetadata.utcDateModified; note.combinedDateModified = note.utcDateModified > contentMetadata.utcDateModified ? note.dateModified : contentMetadata.dateModified; diff --git a/src/services/build_search_query.js b/src/services/build_search_query.js deleted file mode 100644 index a7867aa03..000000000 --- a/src/services/build_search_query.js +++ /dev/null @@ -1,158 +0,0 @@ -const utils = require('./utils'); - -const VIRTUAL_ATTRIBUTES = [ - "dateCreated", - "dateModified", - "utcDateCreated", - "utcDateModified", - "noteId", - "isProtected", - "title", - "content", - "type", - "mime", - "text", - "parentCount" -]; - -module.exports = function(filters, selectedColumns = 'notes.*') { - // alias => join - const joins = { - "notes": null - }; - - let attrFilterId = 1; - - function getAccessor(property) { - let accessor; - - if (!VIRTUAL_ATTRIBUTES.includes(property)) { - // not reusing existing filters to support multi-valued filters e.g. "@tag=christmas @tag=shopping" - // can match notes because @tag can be both "shopping" and "christmas" - const alias = "attr_" + property + "_" + attrFilterId++; - - // forcing to use particular index since SQLite query planner would often choose something pretty bad - joins[alias] = `LEFT JOIN attributes AS ${alias} INDEXED BY IDX_attributes_noteId_index ` - + `ON ${alias}.noteId = notes.noteId ` - + `AND ${alias}.name = '${property}' AND ${alias}.isDeleted = 0`; - - accessor = `${alias}.value`; - } - else if (property === 'content') { - const alias = "note_contents"; - - if (!(alias in joins)) { - joins[alias] = `LEFT JOIN note_contents ON note_contents.noteId = notes.noteId`; - } - - accessor = `${alias}.${property}`; - } - else if (property === 'parentCount') { - // need to cast as string for the equality operator to work - // for >= etc. it is cast again to DECIMAL - // also cannot use COUNT() in WHERE so using subquery ... - accessor = `CAST((SELECT COUNT(1) FROM branches WHERE branches.noteId = notes.noteId AND isDeleted = 0) AS STRING)`; - } - else { - accessor = "notes." + property; - } - - return accessor; - } - - let orderBy = []; - - const orderByFilter = filters.find(filter => filter.name.toLowerCase() === 'orderby'); - - if (orderByFilter) { - orderBy = orderByFilter.value.split(",").map(prop => { - const direction = prop.includes("-") ? "DESC" : "ASC"; - const cleanedProp = prop.trim().replace(/-/g, ""); - - return getAccessor(cleanedProp) + " " + direction; - }); - } - - let where = '1'; - const params = []; - - for (const filter of filters) { - if (['isarchived', 'in', 'orderby', 'limit'].includes(filter.name.toLowerCase())) { - continue; // these are not real filters - } - - where += " " + filter.relation + " "; - - const accessor = getAccessor(filter.name); - - if (filter.operator === 'exists') { - where += `${accessor} IS NOT NULL`; - } - else if (filter.operator === 'not-exists') { - where += `${accessor} IS NULL`; - } - else if (filter.operator === '=' || filter.operator === '!=') { - where += `${accessor} ${filter.operator} ?`; - params.push(filter.value); - } else if (filter.operator === '*=' || filter.operator === '!*=') { - where += `${accessor}` - + (filter.operator.includes('!') ? ' NOT' : '') - + ` LIKE ` + utils.prepareSqlForLike('%', filter.value, ''); - } else if (filter.operator === '=*' || filter.operator === '!=*') { - where += `${accessor}` - + (filter.operator.includes('!') ? ' NOT' : '') - + ` LIKE ` + utils.prepareSqlForLike('', filter.value, '%'); - } else if (filter.operator === '*=*' || filter.operator === '!*=*') { - const columns = filter.name === 'text' ? [getAccessor("title"), getAccessor("content")] : [accessor]; - - let condition = "(" + columns.map(column => - `${column}` + ` LIKE ` + utils.prepareSqlForLike('%', filter.value, '%')) - .join(" OR ") + ")"; - - if (filter.operator.includes('!')) { - condition = "NOT(" + condition + ")"; - } - - if (['text', 'title', 'content'].includes(filter.name)) { - // for title/content search does not make sense to search for protected notes - condition = `(${condition} AND notes.isProtected = 0)`; - } - - where += condition; - } - else if ([">", ">=", "<", "<="].includes(filter.operator)) { - let floatParam; - - // from https://stackoverflow.com/questions/12643009/regular-expression-for-floating-point-numbers - if (/^[+-]?([0-9]*[.])?[0-9]+$/.test(filter.value)) { - floatParam = parseFloat(filter.value); - } - - if (floatParam === undefined || isNaN(floatParam)) { - // if the value can't be parsed as float then we assume that string comparison should be used instead of numeric - where += `${accessor} ${filter.operator} ?`; - params.push(filter.value); - } else { - where += `CAST(${accessor} AS DECIMAL) ${filter.operator} ?`; - params.push(floatParam); - } - } else { - throw new Error("Unknown operator " + filter.operator); - } - } - - if (orderBy.length === 0) { - // if no ordering is given then order at least by note title - orderBy.push("notes.title"); - } - - const query = `SELECT ${selectedColumns} FROM notes - ${Object.values(joins).join('\r\n')} - WHERE - notes.isDeleted = 0 - AND (${where}) - GROUP BY notes.noteId - ORDER BY ${orderBy.join(", ")}`; - - return { query, params }; -}; diff --git a/src/services/image.js b/src/services/image.js index 11e27243e..b1894c6d2 100644 --- a/src/services/image.js +++ b/src/services/image.js @@ -15,7 +15,7 @@ const isSvg = require('is-svg'); async function processImage(uploadBuffer, originalName, shrinkImageSwitch) { const origImageFormat = getImageType(uploadBuffer); - if (origImageFormat && ["webp", "svg"].includes(origImageFormat.ext)) { + if (origImageFormat && ["webp", "svg", "gif"].includes(origImageFormat.ext)) { // JIMP does not support webp at the moment: https://github.com/oliver-moran/jimp/issues/144 shrinkImageSwitch = false; } @@ -61,6 +61,8 @@ function updateImage(noteId, uploadBuffer, originalName) { processImage(uploadBuffer, originalName, true).then(({buffer, imageFormat}) => { sql.transactional(() => { note.mime = getImageMimeFromExtension(imageFormat.ext); + note.save(); + note.setContent(buffer); }) }); @@ -88,6 +90,8 @@ function saveImage(parentNoteId, uploadBuffer, originalName, shrinkImageSwitch) processImage(uploadBuffer, originalName, shrinkImageSwitch).then(({buffer, imageFormat}) => { sql.transactional(() => { note.mime = getImageMimeFromExtension(imageFormat.ext); + note.save(); + note.setContent(buffer); }) }); diff --git a/src/services/notes.js b/src/services/notes.js index e2db234f1..2f8d33b6f 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -105,7 +105,7 @@ function createNewNote(params) { throw new Error(`Note title must not be empty`); } - sql.transactional(() => { + return sql.transactional(() => { const note = new Note({ noteId: params.noteId, // optionally can force specific noteId title: params.title, @@ -744,8 +744,8 @@ function duplicateNote(noteId, parentNoteId) { const newNote = new Note(origNote); newNote.noteId = undefined; // force creation of new note newNote.title += " (dup)"; - newNote.save(); + newNote.setContent(origNote.getContent()); const newBranch = new Branch({ diff --git a/src/services/search.js b/src/services/search.js deleted file mode 100644 index 8c244665c..000000000 --- a/src/services/search.js +++ /dev/null @@ -1,89 +0,0 @@ -"use strict"; - -/** - * Missing things from the OLD search: - * - orderBy - * - limit - * - in - replaced with note.ancestors - * - content in attribute search - * - not - pherhaps not necessary - * - * other potential additions: - * - targetRelations - either named or not - * - any relation without name - */ - -const repository = require('./repository'); -const sql = require('./sql'); -const log = require('./log'); -const parseFilters = require('./search/parse_filters.js'); -const buildSearchQuery = require('./build_search_query'); -const noteCacheService = require('./note_cache/note_cache_service'); - -function searchForNotes(searchString) { - const noteIds = searchForNoteIds(searchString); - - return repository.getNotes(noteIds); -} - -function searchForNoteIds(searchString) { - const filters = parseFilters(searchString); - - const {query, params} = buildSearchQuery(filters, 'notes.noteId'); - - try { - let noteIds = sql.getColumn(query, params); - - noteIds = noteIds.filter(noteCacheService.isAvailable); - - const isArchivedFilter = filters.find(filter => filter.name.toLowerCase() === 'isarchived'); - - if (isArchivedFilter) { - if (isArchivedFilter.operator === 'exists') { - noteIds = noteIds.filter(noteCacheService.isArchived); - } - else if (isArchivedFilter.operator === 'not-exists') { - noteIds = noteIds.filter(noteId => !noteCacheService.isArchived(noteId)); - } - else { - throw new Error(`Unrecognized isArchived operator ${isArchivedFilter.operator}`); - } - } - - const isInFilters = filters.filter(filter => filter.name.toLowerCase() === 'in'); - - for (const isInFilter of isInFilters) { - if (isInFilter.operator === '=') { - noteIds = noteIds.filter(noteId => noteCacheService.isInAncestor(noteId, isInFilter.value)); - } - else if (isInFilter.operator === '!=') { - noteIds = noteIds.filter(noteId => !noteCacheService.isInAncestor(noteId, isInFilter.value)); - } - else { - throw new Error(`Unrecognized isIn operator ${isInFilter.operator}`); - } - } - - const limitFilter = filters.find(filter => filter.name.toLowerCase() === 'limit'); - - if (limitFilter) { - const limit = parseInt(limitFilter.value); - - return noteIds.splice(0, limit); - } - else { - return noteIds; - } - - } - catch (e) { - log.error("Search failed for " + query); - - throw e; - } -} - -module.exports = { - searchForNotes, - searchForNoteIds -}; diff --git a/src/services/search/parse_filters.js b/src/services/search/parse_filters.js deleted file mode 100644 index e21b67c09..000000000 --- a/src/services/search/parse_filters.js +++ /dev/null @@ -1,118 +0,0 @@ -const dayjs = require("dayjs"); - -const filterRegex = /(\b(AND|OR)\s+)?@(!?)([\p{L}\p{Number}_]+|"[^"]+")\s*((=|!=|<|<=|>|>=|!?\*=|!?=\*|!?\*=\*)\s*([^\s=*"]+|"[^"]+"))?/igu; -const smartValueRegex = /^(NOW|TODAY|WEEK|MONTH|YEAR) *([+\-] *\d+)?$/i; - -function calculateSmartValue(v) { - const match = smartValueRegex.exec(v); - if (match === null) { - return; - } - - const keyword = match[1].toUpperCase(); - const num = match[2] ? parseInt(match[2].replace(/ /g, "")) : 0; // can contain spaces between sign and digits - - let format, date; - - if (keyword === 'NOW') { - date = dayjs().add(num, 'second'); - format = "YYYY-MM-DD HH:mm:ss"; - } - else if (keyword === 'TODAY') { - date = dayjs().add(num, 'day'); - format = "YYYY-MM-DD"; - } - else if (keyword === 'WEEK') { - // FIXME: this will always use sunday as start of the week - date = dayjs().startOf('week').add(7 * num, 'day'); - format = "YYYY-MM-DD"; - } - else if (keyword === 'MONTH') { - date = dayjs().add(num, 'month'); - format = "YYYY-MM"; - } - else if (keyword === 'YEAR') { - date = dayjs().add(num, 'year'); - format = "YYYY"; - } - else { - throw new Error("Unrecognized keyword: " + keyword); - } - - return date.format(format); -} - -module.exports = function (searchText) { - searchText = searchText.trim(); - - // if the string doesn't start with attribute then we consider it as just standard full text search - if (!searchText.startsWith("@")) { - // replace with space instead of empty string since these characters are probably separators - const filters = []; - - if (searchText.startsWith('"') && searchText.endsWith('"')) { - // "bla bla" will search for exact match - searchText = searchText.substr(1, searchText.length - 2); - - filters.push({ - relation: 'and', - name: 'text', - operator: '*=*', - value: searchText - }); - } - else { - const tokens = searchText.split(/\s+/); - - for (const token of tokens) { - filters.push({ - relation: 'and', - name: 'text', - operator: '*=*', - value: token - }); - } - } - - filters.push({ - relation: 'and', - name: 'isArchived', - operator: 'not-exists' - }); - - filters.push({ - relation: 'or', - name: 'noteId', - operator: '=', - value: searchText - }); - - return filters; - } - - const filters = []; - - function trimQuotes(str) { return str.startsWith('"') ? str.substr(1, str.length - 2) : str; } - - let match; - - while (match = filterRegex.exec(searchText)) { - const relation = match[2] !== undefined ? match[2].toLowerCase() : 'and'; - const operator = match[3] === '!' ? 'not-exists' : 'exists'; - - const value = match[7] !== undefined ? trimQuotes(match[7]) : null; - - filters.push({ - relation: relation, - name: trimQuotes(match[4]), - operator: match[6] !== undefined ? match[6] : operator, - value: ( - value && value.match(smartValueRegex) - ? calculateSmartValue(value) - : value - ) - }); - } - - return filters; -};