Merge pull request #2798 from thfrei/excalidraw

New note type `canvas-note` using excalidraw (hand drawn notes, sketching, pen)

libraries/lodash.debounce.js

@@ -0,0 +1,70 @@
+/**
+ * Returns a function, that, as long as it continues to be invoked, will not
+ * be triggered. The function will be called after it stops being called for
+ * N milliseconds. If `immediate` is passed, trigger the function on the
+ * leading edge, instead of the trailing. The function also has a property 'clear'
+ * that is a function which will clear the timer to prevent previously scheduled executions.
+ *
+ * @source underscore.js
+ * @see http://unscriptable.com/2009/03/20/debouncing-javascript-methods/
+ * @param {Function} function to wrap
+ * @param {Number} timeout in ms (`100`)
+ * @param {Boolean} whether to execute at the beginning (`false`)
+ * @api public
+ */
+function debounce(func, wait_ms, immediate){
+  var timeout, args, context, timestamp, result;
+  if (null == wait_ms) wait_ms = 100;
+
+  function later() {
+    var last = Date.now() - timestamp;
+
+    if (last < wait_ms && last >= 0) {
+      timeout = setTimeout(later, wait_ms - last);
+    } else {
+      timeout = null;
+      if (!immediate) {
+        result = func.apply(context, args);
+        context = args = null;
+      }
+    }
+  };
+
+  var debounced = function(){
+    context = this;
+    args = arguments;
+    timestamp = Date.now();
+    var callNow = immediate && !timeout;
+    if (!timeout) timeout = setTimeout(later, wait_ms);
+    if (callNow) {
+      result = func.apply(context, args);
+      context = args = null;
+    }
+
+    return result;
+  };
+
+  debounced.clear = function() {
+    if (timeout) {
+      clearTimeout(timeout);
+      timeout = null;
+    }
+  };
+
+  debounced.flush = function() {
+    if (timeout) {
+      result = func.apply(context, args);
+      context = args = null;
+
+      clearTimeout(timeout);
+      timeout = null;
+    }
+  };
+
+  return debounced;
+};
+
+// Adds compatibility for ES modules
+debounce.debounce = debounce;
+
+export default debounce;

package.json

@@ -24,6 +24,7 @@
     "test-all": "npm run test && npm run test-es6"
   },
   "dependencies": {
+    "@excalidraw/excalidraw": "0.11.0",
     "archiver": "5.3.1",
     "async-mutex": "0.3.2",
     "axios": "0.27.2",
@@ -64,6 +65,8 @@
     "open": "8.4.0",
     "portscanner": "2.2.0",
     "rand-token": "1.0.1",
+    "react": "17.0.2",
+    "react-dom": "17.0.2",
     "request": "2.88.2",
     "rimraf": "3.0.2",
     "sanitize-filename": "1.6.3",

src/app.js

@@ -30,6 +30,11 @@ app.use(express.urlencoded({extended: false}));
 app.use(cookieParser());
 app.use(express.static(path.join(__dirname, 'public')));
 app.use('/libraries', express.static(path.join(__dirname, '..', 'libraries')));
+// excalidraw-view mode in shared notes
+app.use('/node_modules/react/umd/react.production.min.js', express.static(path.join(__dirname, '..', 'node_modules/react/umd/react.production.min.js')));
+app.use('/node_modules/react-dom/umd/react-dom.production.min.js', express.static(path.join(__dirname, '..', 'node_modules/react-dom/umd/react-dom.production.min.js')));
+// expose whole dist folder since complete assets are needed in edit and share
+app.use('/node_modules/@excalidraw/excalidraw/dist/', express.static(path.join(__dirname, '..', 'node_modules/@excalidraw/excalidraw/dist/')));
 app.use('/images', express.static(path.join(__dirname, '..', 'images')));
 const sessionParser = session({
     secret: sessionSecret,

src/public/app/dialogs/note_revisions.js

@@ -171,6 +171,28 @@ async function setContentPane() {
 
         $content.html($table);
     }
+    else if (revisionItem.type === 'canvas') {
+        /**
+         * FIXME: We load a font called Virgil.wof2, which originates from excalidraw.com
+         *        REMOVE external dependency!!!! This is defined in the svg in defs.style
+         */
+        const content = fullNoteRevision.content;
+
+        try {
+            const data = JSON.parse(content)
+            const svg = data.svg || "no svg present."
+
+            /**
+             * maxWidth: 100% use full width of container but do not enlarge!
+             * height:auto to ensure that height scales with width
+             */
+            const $svgHtml = $(svg).css({maxWidth: "100%", height: "auto"});
+            $content.html($('<div>').append($svgHtml));
+        } catch(err) {
+            console.error("error parsing fullNoteRevision.content as JSON", fullNoteRevision.content, err);
+            $content.html($("<div>").text("Error parsing content. Please check console.error() for more details."));
+        }
+    }
     else {
         $content.text("Preview isn't available for this note type.");
     }

src/public/app/entities/note_short.js

@@ -16,7 +16,8 @@ const NOTE_TYPE_ICONS = {
     "relation-map": "bx bx-map-alt",
     "book": "bx bx-book",
     "note-map": "bx bx-map-alt",
-    "mermaid": "bx bx-selection"
+    "mermaid": "bx bx-selection",
+    "canvas": "bx bx-pen"
 };
 
 /**

src/public/app/services/library_loader.js

@@ -56,6 +56,17 @@ const MERMAID = {
     js: [ "libraries/mermaid.min.js" ]
 }
 
+const EXCALIDRAW = {
+    js: [
+        "node_modules/react/umd/react.production.min.js",
+        "node_modules/react-dom/umd/react-dom.production.min.js",
+        "node_modules/@excalidraw/excalidraw/dist/excalidraw.production.min.js",
+    ],
+    // css: [
+    //     "stylesheets/somestyle.css"
+    // ]
+};
+
 async function requireLibrary(library) {
     if (library.css) {
         library.css.map(cssUrl => requireCss(cssUrl));
@@ -106,5 +117,6 @@ export default {
     KATEX,
     WHEEL_ZOOM,
     FORCE_GRAPH,
-    MERMAID
+    MERMAID,
+    EXCALIDRAW
 }

src/public/app/services/note_content_renderer.js

@@ -141,6 +141,27 @@ async function getRenderedContent(note, options = {}) {
 
         $renderedContent.append($content);
     }
+    else if (type === 'canvas') {
+        // make sure surrounding container has size of what is visible. Then image is shrinked to its boundaries
+        $renderedContent.css({height: "100%", width:"100%"});
+        
+        const noteComplement = await froca.getNoteComplement(note.noteId);
+        const content = noteComplement.content || "";
+
+        try {
+            const placeHolderSVG = "<svg />";
+            const data = JSON.parse(content)
+            const svg = data.svg || placeHolderSVG;
+            /**
+             * maxWidth: size down to 100% (full) width of container but do not enlarge!
+             * height:auto to ensure that height scales with width
+             */
+            $renderedContent.append($(svg).css({maxWidth: "100%", maxHeight: "100%", height: "auto", width: "auto"}));
+        } catch(err) {
+            console.error("error parsing content as JSON", content, err);
+            $renderedContent.append($("<div>").text("Error parsing content. Please check console.error() for more details."));
+        }
+    }
     else if (!options.tooltip && type === 'protected-session') {
         const $button = $(`<button class="btn btn-sm"><span class="bx bx-log-in"></span> Enter protected session</button>`)
             .on('click', protectedSessionService.enterProtectedSession);

src/public/app/services/note_list_renderer.js

@@ -99,13 +99,15 @@ const TPL = `
         padding: 10px;
     }
     
-    .note-book-content.type-image img {
+    .note-book-content.type-image img, .note-book-content.type-canvas svg {
         max-width: 100%;
         max-height: 100%;
         object-fit: contain;
     }
     
-    .note-book-card.type-image .note-book-content img, .note-book-card.type-text .note-book-content img {
+    .note-book-card.type-image .note-book-content img,
+    .note-book-card.type-text .note-book-content img,
+    .note-book-card.type-canvas .note-book-content img {
         max-width: 100%;
         max-height: 100%;
     }

src/public/app/services/tree_context_menu.js

@@ -33,7 +33,8 @@ class TreeContextMenu {
             { title: "Note Map", command: command, type: "note-map", uiIcon: "map-alt" },
             { title: "Render HTML note", command: command, type: "render", uiIcon: "extension" },
             { title: "Book", command: command, type: "book", uiIcon: "book" },
-            { title: "Mermaid diagram", command: command, type: "mermaid", uiIcon: "selection" }
+            { title: "Mermaid diagram", command: command, type: "mermaid", uiIcon: "selection" },
+            { title: "Canvas", command: command, type: "canvas", uiIcon: "pen" },
         ];
     }
 

src/public/app/services/utils.js

@@ -359,6 +359,12 @@ function isValidAttributeName(name) {
     return ATTR_NAME_MATCHER.test(name);
 }
 
+function sleep(time_ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, time_ms);
+    });
+};
+
 export default {
     reloadFrontendApp,
     parseDate,
@@ -402,5 +408,6 @@ export default {
     initHelpButtons,
     openHelp,
     filterAttributeName,
-    isValidAttributeName
+    isValidAttributeName,
+    sleep,
 };

src/public/app/widgets/note_detail.js

@@ -10,6 +10,7 @@ import FileTypeWidget from "./type_widgets/file.js";
 import ImageTypeWidget from "./type_widgets/image.js";
 import RenderTypeWidget from "./type_widgets/render.js";
 import RelationMapTypeWidget from "./type_widgets/relation_map.js";
+import CanvasTypeWidget from "./type_widgets/canvas.js";
 import ProtectedSessionTypeWidget from "./type_widgets/protected_session.js";
 import BookTypeWidget from "./type_widgets/book.js";
 import appContext from "../services/app_context.js";
@@ -50,6 +51,7 @@ const typeWidgetClasses = {
     'search': NoneTypeWidget,
     'render': RenderTypeWidget,
     'relation-map': RelationMapTypeWidget,
+    'canvas': CanvasTypeWidget,
     'protected-session': ProtectedSessionTypeWidget,
     'book': BookTypeWidget,
     'note-map': NoteMapTypeWidget
@@ -66,7 +68,7 @@ export default class NoteDetailWidget extends NoteContextAwareWidget {
             const {noteId} = note;
 
             const dto = note.dto;
-            dto.content = this.getTypeWidget().getContent();
+            dto.content = await this.getTypeWidget().getContent();
 
             // for read only notes
             if (dto.content === undefined) {
@@ -145,11 +147,14 @@ export default class NoteDetailWidget extends NoteContextAwareWidget {
         this.checkFullHeight();
     }
 
+    /**
+     * sets full height of container that contains note content for a subset of note-types
+     */
     checkFullHeight() {
         // https://github.com/zadam/trilium/issues/2522
         this.$widget.toggleClass("full-height",
             !this.noteContext.hasNoteList()
-            && ['editable-text', 'editable-code'].includes(this.type)
+            && ['editable-text', 'editable-code', 'canvas'].includes(this.type)
             && this.mime !== 'text/x-sqlite;schema=trilium');
     }
 

src/public/app/widgets/note_type.js

@@ -11,6 +11,7 @@ const NOTE_TYPES = [
     { type: "text", mime: "text/html", title: "Text", selectable: true },
     { type: "relation-map", mime: "application/json", title: "Relation Map", selectable: true },
     { type: "render", mime: '', title: "Render Note", selectable: true },
+    { type: "canvas", mime: 'application/json', title: "Canvas", selectable: true },
     { type: "book", mime: '', title: "Book", selectable: true },
     { type: "mermaid", mime: 'text/mermaid', title: "Mermaid Diagram", selectable: true },
     { type: "code", mime: 'text/plain', title: "Code", selectable: true }

src/public/app/widgets/note_wrapper.js

@@ -32,7 +32,7 @@ export default class NoteWrapperWidget extends FlexContainer {
 
     refresh(noteContext) {
         this.$widget.toggleClass("full-content-width",
-            ['image', 'mermaid', 'book', 'render'].includes(noteContext?.note?.type)
+            ['image', 'mermaid', 'book', 'render', 'canvas'].includes(noteContext?.note?.type)
             || !!noteContext?.note?.hasLabel('fullContentWidth')
         );
     }

src/public/app/widgets/type_widgets/canvas.js

@@ -0,0 +1,477 @@
+import libraryLoader from "../../services/library_loader.js";
+import TypeWidget from "./type_widget.js";
+import utils from '../../services/utils.js';
+import froca from "../../services/froca.js";
+import debounce from "../../../../../libraries/lodash.debounce.js";
+
+const {sleep} = utils;
+
+const TPL = `
+    <div class="canvas-widget note-detail-canvas note-detail-printable note-detail">
+        <style type="text/css">
+        .excalidraw .App-menu_top .buttonList {
+            display: flex;
+        }
+
+        .excalidraw-wrapper {
+            height: 100%;
+        }
+
+        :root[dir="ltr"]
+        .excalidraw
+        .layer-ui__wrapper
+        .zen-mode-transition.App-menu_bottom--transition-left {
+            transform: none;
+        }
+
+        </style>
+        <!-- height here necessary. otherwise excalidraw not shown -->
+        <div class="canvas-render" style="height: 100%"></div>
+    </div>
+`;
+
+/**
+ * # Canvas note with excalidraw
+ * @author thfrei 2022-05-11
+ * 
+ * Background:
+ * excalidraw gives great support for hand drawn notes. It also allows to include images and support
+ * for sketching. Excalidraw has a vibrant and active community.
+ * 
+ * Functionality:
+ * We store the excalidraw assets (elements, appState, files) in the note. In addition to that, we 
+ * export the SVG from the canvas on every update. The SVG is also saved in the note. It is used
+ * for displaying any canvas note inside of a text note as an image.
+ * 
+ * Paths not taken.
+ *  - excalidraw-to-svg (node.js) could be used to avoid storing the svg in the backend.
+ *    We could render the SVG on the fly. However, as of now, it does not render any hand drawn
+ *    (freedraw) paths. There is an issue with Path2D object not present in node-canvas library 
+ *    used by jsdom. (See Trilium PR for samples and other issues in respective library.
+ *    Link will be added later). Related links:
+ *     - https://github.com/Automattic/node-canvas/pull/2013
+ *     - https://github.com/google/canvas-5-polyfill 
+ *     - https://github.com/Automattic/node-canvas/issues/1116 
+ *     - https://www.npmjs.com/package/path2d-polyfill 
+ *  - excalidraw-to-svg (node.js) takes quite some time to load an image (1-2s)
+ *  - excalidraw-utils (browser) does render freedraw, however NOT freedraw with background. It is not
+ *    used, since it is a big dependency, and has the same functionality as react + excalidraw.
+ *  - infinite-drawing-canvas with fabric.js. This library lacked a lot of feature, excalidraw already
+ *    has.
+ * 
+ * Known issues:
+ *  - v0.11.0 of excalidraw does not render freedraw backgrounds in the svg
+ *  - the 3 excalidraw fonts should be included in the share and everywhere, so that it is shown
+ *    when requiring svg.
+ * 
+ * Discussion of storing svg in the note:
+ *  - Pro: we will combat bit-rot. Showing the SVG will be very fast and easy, since it is already there.
+ *  - Con: The note will get bigger (~40-50%?), we will generate more bandwith. However, using trilium 
+ *         desktop instance mitigates that issue.
+ * 
+ * Roadmap:
+ *  - Support image-notes as reference in excalidraw
+ *  - Support canvas note as reference (svg) in other canvas notes.
+ *  - Make it easy to include a canvas note inside a text note
+ *  - Support for excalidraw libraries. Maybe special code notes with a tag.
+ */
+export default class ExcalidrawTypeWidget extends TypeWidget {
+    constructor() {
+        super();
+
+        // constants
+        this.SCENE_VERSION_INITIAL = -1;
+        this.SCENE_VERSION_ERROR = -2;
+
+        // config
+        this.DEBOUNCE_TIME_ONCHANGEHANDLER = 750; // ms
+        // ensure that assets are loaded from trilium
+        window.EXCALIDRAW_ASSET_PATH = `${window.location.origin}/node_modules/@excalidraw/excalidraw/dist/`;
+
+        // temporary vars
+        this.currentNoteId = "";
+        this.currentSceneVersion = this.SCENE_VERSION_INITIAL;
+
+        // will be overwritten
+        this.excalidrawRef;
+        this.$render;
+        this.renderElement;
+        this.$widget;
+        this.reactHandlers; // used to control react state
+        
+        this.createExcalidrawReactApp = this.createExcalidrawReactApp.bind(this);
+        this.onChangeHandler = this.onChangeHandler.bind(this);
+        this.isNewSceneVersion = this.isNewSceneVersion.bind(this);
+    }
+
+    /**
+     * (trilium)
+     * @returns {string} "canvas"
+     */
+    static getType() {
+        return "canvas";
+    }
+
+    /**
+     * (trilium)
+     * renders note
+     */
+    doRender() {
+        this.$widget = $(TPL);
+
+        this.$widget.toggleClass("full-height", true); // only add
+        this.$render = this.$widget.find('.canvas-render');
+        this.renderElement = this.$render.get(0);
+
+        libraryLoader
+            .requireLibrary(libraryLoader.EXCALIDRAW)
+            .then(() => {
+                const React = window.React;
+                const ReactDOM = window.ReactDOM;
+                
+                ReactDOM.unmountComponentAtNode(this.renderElement);
+                ReactDOM.render(React.createElement(this.createExcalidrawReactApp), this.renderElement);
+            })
+
+        return this.$widget;
+    }
+
+    /**
+     * (trilium)
+     * called to populate the widget container with the note content
+     * 
+     * @param {note} note 
+     */
+    async doRefresh(note) {
+        // see if note changed, since we do not get a new class for a new note
+        const noteChanged = this.currentNoteId !== note.noteId;
+        if (noteChanged) {
+            // reset scene to omit unnecessary onchange handler
+            this.currentSceneVersion = this.SCENE_VERSION_INITIAL;
+        }
+        this.currentNoteId = note.noteId;
+        
+        // get note from backend and put into canvas
+        const noteComplement = await froca.getNoteComplement(note.noteId);
+
+        // before we load content into excalidraw, make sure excalidraw has loaded
+        while (!this.excalidrawRef || !this.excalidrawRef.current) {
+            this.log("excalidrawRef not yet loeaded, sleep 200ms...");
+            await sleep(200);
+        }
+
+        /**
+         * new and empty note - make sure that canvas is empty.
+         * If we do not set it manually, we occasionally get some "bleeding" from another
+         * note into this fresh note. Probably due to that this note-instance does not get
+         * newly instantiated?
+         */
+        if (this.excalidrawRef.current && noteComplement.content === "") {
+            const sceneData = {
+                elements: [], 
+                appState: {},
+                collaborators: []
+            };
+            
+            this.excalidrawRef.current.updateScene(sceneData);
+        }
+
+        /**
+         * load saved content into excalidraw canvas
+         */
+        else if (this.excalidrawRef.current && noteComplement.content) {
+            let content ={
+                elements: [],
+                appState: [],
+                files: [],
+            };
+
+            try {
+                content = JSON.parse(noteComplement.content || "");
+            } catch(err) {
+                console.error("Error parsing content. Probably note.type changed",
+                              "Starting with empty canvas"
+                              , note, noteComplement, err);
+            }
+
+            const {elements, appState, files} = content;
+
+            /**
+             * use widths and offsets of current view, since stored appState has the state from
+             * previous edit. using the stored state would lead to pointer mismatch.
+             */
+            const boundingClientRect = this.excalidrawWrapperRef.current.getBoundingClientRect();
+            appState.width = boundingClientRect.width;
+            appState.height = boundingClientRect.height;
+            appState.offsetLeft = boundingClientRect.left;
+            appState.offsetTop = boundingClientRect.top;
+
+            const sceneData = {
+                elements, 
+                appState,
+                collaborators: []
+            };
+
+            // files are expected in an array when loading. they are stored as an key-index object
+            // see example for loading here:
+            // https://github.com/excalidraw/excalidraw/blob/c5a7723185f6ca05e0ceb0b0d45c4e3fbcb81b2a/src/packages/excalidraw/example/App.js#L68
+            const fileArray = [];
+            for (const fileId in files) {
+                const file = files[fileId];
+                // TODO: dataURL is replaceable with a trilium image url
+                //       maybe we can save normal images (pasted) with base64 data url, and trilium images
+                //       with their respective url! nice
+                // file.dataURL = "http://localhost:8080/api/images/ltjOiU8nwoZx/start.png";
+                fileArray.push(file);
+            }
+
+            this.sceneVersion = window.Excalidraw.getSceneVersion(elements);
+
+            this.excalidrawRef.current.updateScene(sceneData);
+            this.excalidrawRef.current.addFiles(fileArray);
+        }
+        
+        // set initial scene version
+        if (this.currentSceneVersion === this.SCENE_VERSION_INITIAL) {
+            this.currentSceneVersion = this.getSceneVersion();
+        }
+    }
+
+    /**
+     * (trilium)
+     * gets data from widget container that will be sent via spacedUpdate.scheduleUpdate();
+     * this is automatically called after this.saveData();
+     */
+    async getContent() {
+        const elements = this.excalidrawRef.current.getSceneElements();
+        const appState = this.excalidrawRef.current.getAppState();
+        
+        /**
+         * A file is not deleted, even though removed from canvas. therefore we only keep
+         * files that are referenced by an element. Maybe this will change with new excalidraw version?
+         */
+        const files = this.excalidrawRef.current.getFiles();
+        
+        /**
+         * parallel svg export to combat bitrot and enable rendering image for note inclusion,
+         * preview and share.
+         */
+        const svg = await window.Excalidraw.exportToSvg({
+            elements,
+            appState,
+            exportPadding: 5, // 5 px padding
+            metadata: 'trilium-export',
+            files
+        });
+        const svgString = svg.outerHTML;
+
+        /**
+         * workaround until https://github.com/excalidraw/excalidraw/pull/5065 is merged and published
+         */
+        const svgSafeString = this.replaceExternalAssets(svgString);
+
+        const activeFiles = {};
+        elements.forEach((element) => {
+            if (element.fileId) {
+                activeFiles[element.fileId] = files[element.fileId];
+            }
+        })
+
+        const content = {
+            _meta: "This note has type `canvas`. It uses excalidraw and stores an exported svg alongside.",
+            elements, // excalidraw
+            appState, // excalidraw
+            files: activeFiles, // excalidraw
+            svg: svgSafeString, // not needed for excalidraw, used for note_short, content, and image api
+        };
+
+        const contentString = JSON.stringify(content);
+
+        return contentString;
+    }
+
+    /**
+     * (trilium)
+     * save content to backend
+     * spacedUpdate is kind of a debouncer.
+     */
+    saveData() {
+        this.spacedUpdate.scheduleUpdate();
+    }
+
+    onChangeHandler() {
+        const appState = this.excalidrawRef.current.getAppState() || {};
+
+        // changeHandler is called upon any tiny change in excalidraw. button clicked, hover, etc.
+        // make sure only when a new element is added, we actually save something.
+        const isNewSceneVersion = this.isNewSceneVersion();
+        /**
+         * FIXME: however, we might want to make an exception, if viewport changed, since viewport
+         *        is desired to save? (add) and appState background, and some things
+         */
+
+        // upon updateScene, onchange is called, even though "nothing really changed" that is worth saving
+        const isNotInitialScene = this.currentSceneVersion !== this.SCENE_VERSION_INITIAL;
+
+        const shouldSave = isNewSceneVersion && isNotInitialScene;
+
+        if (shouldSave) {
+            this.updateSceneVersion();
+            this.saveData();
+        } else {
+            // do nothing
+        }
+    }
+
+    createExcalidrawReactApp() {
+        const React = window.React;
+        const Excalidraw = window.Excalidraw;
+
+        const excalidrawRef = React.useRef(null);
+        this.excalidrawRef = excalidrawRef;
+        const excalidrawWrapperRef = React.useRef(null);
+        this.excalidrawWrapperRef = excalidrawWrapperRef;
+        const [dimensions, setDimensions] = React.useState({
+            width: undefined,
+            height: undefined
+        });
+
+        const [viewModeEnabled, setViewModeEnabled] = React.useState(false);
+        const [zenModeEnabled, setZenModeEnabled] = React.useState(false);
+        const [gridModeEnabled, setGridModeEnabled] = React.useState(false);
+        const [synchronized, setSynchronized] = React.useState(true);
+        
+        React.useEffect(() => {
+            const dimensions = {
+                width: excalidrawWrapperRef.current.getBoundingClientRect().width,
+                height: excalidrawWrapperRef.current.getBoundingClientRect().height
+            };
+            setDimensions(dimensions);
+
+            const onResize = () => {
+                const dimensions = {
+                    width: excalidrawWrapperRef.current.getBoundingClientRect().width,
+                    height: excalidrawWrapperRef.current.getBoundingClientRect().height
+                };
+                setDimensions(dimensions);
+            };
+            
+            window.addEventListener("resize", onResize);
+            
+            return () => window.removeEventListener("resize", onResize);
+        }, [excalidrawWrapperRef]);
+
+        const onLinkOpen = React.useCallback((element, event) => {
+            const link = element.link;
+            const { nativeEvent } = event.detail;
+            const isNewTab = nativeEvent.ctrlKey || nativeEvent.metaKey;
+            const isNewWindow = nativeEvent.shiftKey;
+            const isInternalLink = link.startsWith("/") 
+                || link.includes(window.location.origin);
+                
+            if (isInternalLink && !isNewTab && !isNewWindow) {
+                // signal that we're handling the redirect ourselves
+                event.preventDefault();
+                // do a custom redirect, such as passing to react-router
+                // ...
+            } else {
+                // open in same tab
+            }
+          }, []);
+
+        return React.createElement(
+            React.Fragment,
+            null,
+            React.createElement(
+                "div",
+                {
+                    className: "excalidraw-wrapper",
+                    ref: excalidrawWrapperRef
+                },
+                React.createElement(Excalidraw.default, {
+                    ref: excalidrawRef,
+                    width: dimensions.width,
+                    height: dimensions.height,
+                    // initialData: InitialData,
+                    onPaste: (data, event) => {
+                        this.log("Verbose: excalidraw internal paste. No trilium action implemented.", data, event);
+                    },
+                    onChange: debounce(this.onChangeHandler, this.DEBOUNCE_TIME_ONCHANGEHANDLER),
+                    // onPointerUpdate: (payload) => console.log(payload),
+                    onCollabButtonClick: () => {
+                        window.alert("You clicked on collab button. No collaboration is implemented.");
+                    },
+                    viewModeEnabled: viewModeEnabled,
+                    zenModeEnabled: zenModeEnabled,
+                    gridModeEnabled: gridModeEnabled,
+                    isCollaborating: false,
+                    detectScroll: false,
+                    handleKeyboardGlobally: false,
+                    autoFocus: true,
+                    onLinkOpen,
+                })
+            )
+        );
+    }    
+
+    /**
+     * needed to ensure, that multipleOnChangeHandler calls do not trigger a safe.
+     * we compare the scene version as suggested in:
+     * https://github.com/excalidraw/excalidraw/issues/3014#issuecomment-778115329
+     * 
+     * info: sceneVersions are not incrementing. it seems to be a pseudo-random number
+     */
+     isNewSceneVersion() {
+        const sceneVersion = this.getSceneVersion();
+        
+        return this.currentSceneVersion === this.SCENE_VERSION_INITIAL // initial scene version update
+            || this.currentSceneVersion !== sceneVersion // ensure scene changed
+        ;
+    }
+
+    getSceneVersion() {
+        if (this.excalidrawRef) {
+            const elements = this.excalidrawRef.current.getSceneElements();
+            const sceneVersion = window.Excalidraw.getSceneVersion(elements);
+            return sceneVersion;
+        } else {
+            return this.SCENE_VERSION_ERROR;
+        }
+    }
+
+    updateSceneVersion() {
+        this.currentSceneVersion = this.getSceneVersion();
+    }
+    
+    /**
+     * logs to console.log with some predefined title
+     * 
+     * @param  {...any} args 
+     */
+    log(...args) {
+        let title = '';
+        if (this.note) {
+            title = this.note.title;
+        } else {
+            title = this.noteId + "nt/na";
+        }
+
+        console.log(title, "=", this.noteId, "==",  ...args);
+    }
+
+    /**
+     * replaces exlicraw.com with own assets
+     * 
+     * workaround until https://github.com/excalidraw/excalidraw/pull/5065 is merged and published
+     * needed for v0.11.0
+     * 
+     * @param {string} string 
+     * @returns 
+     */
+    replaceExternalAssets = (string) => {
+        let result = string;
+        // exlidraw.com asset in react usage
+        result = result.replaceAll("https://excalidraw.com/", window.EXCALIDRAW_ASSET_PATH+"excalidraw-assets/");
+        return result;
+    }
+}

src/public/app/widgets/type_widgets/type_widget.js

@@ -38,6 +38,9 @@ export default class TypeWidget extends NoteContextAwareWidget {
         return this.$widget.is(":visible") && this.noteContext?.ntxId === appContext.tabManager.activeNtxId;
     }
 
+    /**
+     * @returns {Promise|*} promise resolving content or directly the content
+     */ 
     getContent() {}
 
     focus() {}

src/routes/api/image.js

@@ -11,7 +11,7 @@ function returnImage(req, res) {
     if (!image) {
         return res.sendStatus(404);
     }
-    else if (image.type !== 'image') {
+    else if (!["image", "canvas"].includes(image.type)){
         return res.sendStatus(400);
     }
     else if (image.isDeleted || image.data === null) {
@@ -19,10 +19,27 @@ function returnImage(req, res) {
         return res.send(fs.readFileSync(RESOURCE_DIR + '/db/image-deleted.png'));
     }
 
-    res.set('Content-Type', image.mime);
-    res.set("Cache-Control", "no-cache, no-store, must-revalidate");
-
-    res.send(image.getContent());
+    /**
+     * special "image" type. the canvas is actually type application/json 
+     * to avoid bitrot and enable usage as referenced image the svg is included.
+     */
+    if (image.type === 'canvas') {
+        const content = image.getContent();
+        try {
+            const data = JSON.parse(content);
+            
+            const svg = data.svg || '<svg />'
+            res.set('Content-Type', "image/svg+xml");
+            res.set("Cache-Control", "no-cache, no-store, must-revalidate");
+            res.send(svg);
+        } catch(err) {
+            res.status(500).send("there was an error parsing excalidraw to svg");
+        }
+    } else {
+        res.set('Content-Type', image.mime);
+        res.set("Cache-Control", "no-cache, no-store, must-revalidate");
+        res.send(image.getContent());
+    }
 }
 
 function uploadImage(req) {

src/routes/routes.js

@@ -285,6 +285,7 @@ function register(app) {
     apiRoute(POST, '/api/special-notes/search-note', specialNotesRoute.createSearchNote);
     apiRoute(POST, '/api/special-notes/save-search-note', specialNotesRoute.saveSearchNote);
 
+    // :filename is not used by trilium, but instead used for "save as" to assign a human readable filename
     route(GET, '/api/images/:noteId/:filename', [auth.checkApiAuthOrElectron], imageRoute.returnImage);
     route(POST, '/api/images', [auth.checkApiAuthOrElectron, uploadMiddleware, csrfMiddleware], imageRoute.uploadImage, apiResultHandler);
     route(PUT, '/api/images/:noteId', [auth.checkApiAuthOrElectron, uploadMiddleware, csrfMiddleware], imageRoute.updateImage, apiResultHandler);

src/services/backend_script_api.js

@@ -210,7 +210,7 @@ function BackendScriptApi(currentNote, apiParams) {
      * @property {string} parentNoteId - MANDATORY
      * @property {string} title - MANDATORY
      * @property {string|buffer} content - MANDATORY
-     * @property {string} type - text, code, file, image, search, book, relation-map - MANDATORY
+     * @property {string} type - text, code, file, image, search, book, relation-map, canvas - MANDATORY
      * @property {string} mime - value is derived from default mimes for type
      * @property {boolean} isProtected - default is false
      * @property {boolean} isExpanded - default is false

src/services/export/single.js

@@ -41,7 +41,7 @@ function exportSingleNote(taskContext, branch, format, res) {
         extension = mimeTypes.extension(note.mime) || 'code';
         mime = note.mime;
     }
-    else if (note.type === 'relation-map' || note.type === 'search') {
+    else if (note.type === 'relation-map' || note.type === 'canvas' || note.type === 'search') {
         payload = content;
         extension = 'json';
         mime = 'application/json';

src/services/note_types.js

@@ -7,6 +7,7 @@ module.exports = [
     'search', 
     'relation-map', 
     'book', 
-    'note-map', 
-    'mermaid'
+    'note-map',
+    'mermaid',
+    'canvas'
 ];

src/services/notes.js

@@ -53,7 +53,7 @@ function deriveMime(type, mime) {
         mime = 'text/html';
     } else if (type === 'code' || type === 'mermaid') {
         mime = 'text/plain';
-    } else if (['relation-map', 'search'].includes(type)) {
+    } else if (['relation-map', 'search', 'canvas'].includes(type)) {
         mime = 'application/json';
     } else if (['render', 'book'].includes(type)) {
         mime = '';
@@ -84,7 +84,7 @@ function copyChildAttributes(parentNote, childNote) {
  * - {string} parentNoteId
  * - {string} title
  * - {*} content
- * - {string} type - text, code, file, image, search, book, relation-map, render
+ * - {string} type - text, code, file, image, search, book, relation-map, canvas, render
  *
  * Following are optional (have defaults)
  * - {string} mime - value is derived from default mimes for type

src/services/utils.js

@@ -168,7 +168,7 @@ const STRING_MIME_TYPES = [
 
 function isStringNote(type, mime) {
     // render and book are string note in the sense that they are expected to contain empty string
-    return ["text", "code", "relation-map", "search", "render", "book", "mermaid"].includes(type)
+    return ["text", "code", "relation-map", "search", "render", "book", "mermaid", "canvas"].includes(type)
         || mime.startsWith('text/')
         || STRING_MIME_TYPES.includes(mime);
 }
@@ -192,7 +192,7 @@ function formatDownloadTitle(filename, type, mime) {
 
     if (type === 'text') {
         return filename + '.html';
-    } else if (['relation-map', 'search'].includes(type)) {
+    } else if (['relation-map', 'canvas', 'search'].includes(type)) {
         return filename + '.json';
     } else {
         if (!mime) {

src/share/canvas_share.js

@@ -0,0 +1,102 @@
+/**
+ * this is used as a "standalone js" file and required by a shared note directly via script-tags
+ * 
+ * data input comes via window variable as follow
+ * const {elements, appState, files} = window.triliumExcalidraw;
+ */
+
+document.getElementById("excalidraw-app").style.height = appState.height+"px";
+
+const App = () => {
+    const excalidrawRef = React.useRef(null);
+    const excalidrawWrapperRef = React.useRef(null);
+    const [dimensions, setDimensions] = React.useState({
+        width: undefined,
+        height: appState.height,
+    });
+    const [viewModeEnabled, setViewModeEnabled] = React.useState(false);
+    
+    // ensure that assets are loaded from trilium
+    
+    /**
+     * resizing
+     */
+    React.useEffect(() => {
+        const dimensions = {
+            width: excalidrawWrapperRef.current.getBoundingClientRect().width,
+            height: excalidrawWrapperRef.current.getBoundingClientRect().height
+        };
+        setDimensions(dimensions);
+
+        const onResize = () => {
+            const dimensions = {
+                width: excalidrawWrapperRef.current.getBoundingClientRect().width,
+                height: excalidrawWrapperRef.current.getBoundingClientRect().height
+            };
+            setDimensions(dimensions);
+        };
+        
+        window.addEventListener("resize", onResize);
+        // ensure that resize is also called for split creation and deletion
+        // not really the problem. problem is saved appState!
+        // self.$renderElement.addEventListener("resize", onResize);
+        
+        return () => window.removeEventListener("resize", onResize);
+    }, [excalidrawWrapperRef]);
+
+    return React.createElement(
+        React.Fragment,
+        null,
+        React.createElement(
+            "div",
+            {
+                className: "excalidraw-wrapper",
+                ref: excalidrawWrapperRef
+            },        
+            React.createElement(Excalidraw.default, {
+                ref: excalidrawRef,
+                width: dimensions.width,
+                height: dimensions.height,
+                initialData: {
+                    elements, appState, files
+                },
+                viewModeEnabled: !viewModeEnabled,
+                zenModeEnabled: false,
+                gridModeEnabled: false,
+                isCollaborating: false,
+                detectScroll: false,
+                handleKeyboardGlobally: false,
+                autoFocus: true,
+                renderFooter: () => {
+                    return React.createElement(
+                        React.Fragment,
+                        null,
+                        React.createElement(
+                            "div",
+                            {
+                                className: "excalidraw-top-right-ui excalidraw Island",
+                            },
+                            React.createElement(
+                                "label",
+                                {
+                                    style: {
+                                        padding: "5px",
+                                    },
+                                    className: "excalidraw Stack",
+                                },
+                                React.createElement(
+                                    "button", 
+                                    {
+                                        onClick: () => setViewModeEnabled(!viewModeEnabled)
+                                    }, 
+                                    viewModeEnabled ? " Enter simple view mode " : " Enter extended view mode "
+                                ),
+                                ""
+                            ),
+                        ));
+                },
+            })
+        )
+    );
+};
+ReactDOM.render(React.createElement(App), document.getElementById("excalidraw-app"));

src/share/content_renderer.js

@@ -85,6 +85,38 @@ document.addEventListener("DOMContentLoaded", function() {
     else if (note.type === 'book') {
         isEmpty = true;
     }
+    else if (note.type === 'canvas') {
+        header += `<script>
+                    window.EXCALIDRAW_ASSET_PATH = window.location.origin + "/node_modules/@excalidraw/excalidraw/dist/";
+                   </script>`;
+        header += `<script src="../../node_modules/react/umd/react.production.min.js"></script>`;
+        header += `<script src="../../node_modules/react-dom/umd/react-dom.production.min.js"></script>`;
+        header += `<script src="../../node_modules/@excalidraw/excalidraw/dist/excalidraw.production.min.js"></script>`;
+        header += `<style type="text/css">
+
+            .excalidraw-wrapper {
+                height: 100%;
+            }
+
+            :root[dir="ltr"]
+            .excalidraw
+            .layer-ui__wrapper
+            .zen-mode-transition.App-menu_bottom--transition-left {
+                transform: none;
+            }
+        </style>`;
+
+        content = `<div>
+            <script>
+                const {elements, appState, files} = JSON.parse(${JSON.stringify(content)});
+                window.triliumExcalidraw = {elements, appState, files}
+            </script>
+            <div id="excalidraw-app"></div>
+            <hr>
+            <a href="api/images/${note.noteId}/${note.title}?utc=${note.utcDateModified}">Get Image Link</a>
+            <script src="./canvas_share.js"></script>
+        </div>`;
+    }
     else {
         content = '<p>This note type cannot be displayed.</p>';
     }
@@ -99,7 +131,3 @@ document.addEventListener("DOMContentLoaded", function() {
 module.exports = {
     getContent
 };
-
-
-
-

src/share/routes.js

@@ -1,3 +1,6 @@
+const express = require('express');
+const path = require('path');
+
 const shaca = require("./shaca/shaca");
 const shacaLoader = require("./shaca/shaca_loader");
 const shareRoot = require("./share_root");
@@ -55,6 +58,8 @@ function register(router) {
         });
     }
 
+    router.use('/share/canvas_share.js', express.static(path.join(__dirname, 'canvas_share.js')));
+
     router.get(['/share', '/share/'], (req, res, next) => {
         shacaLoader.ensureLoad();
 
@@ -110,6 +115,7 @@ function register(router) {
         res.send(note.getContent());
     });
 
+    // :filename is not used by trilium, but instead used for "save as" to assign a human readable filename
     router.get('/share/api/images/:noteId/:filename', (req, res, next) => {
         shacaLoader.ensureLoad();
 
@@ -118,15 +124,31 @@ function register(router) {
         if (!image) {
             return res.status(404).send(`Note '${req.params.noteId}' not found`);
         }
-        else if (image.type !== 'image') {
-            return res.status(400).send("Requested note is not an image");
+        else if (!["image", "canvas"].includes(image.type)) {
+            return res.status(400).send("Requested note is not a shareable image");
+        } else if (image.type === "canvas") {
+            /**
+             * special "image" type. the canvas is actually type application/json 
+             * to avoid bitrot and enable usage as referenced image the svg is included.
+             */
+            const content = image.getContent();
+            try {
+                const data = JSON.parse(content);
+
+                const svg = data.svg || '<svg />';
+                addNoIndexHeader(image, res);
+                res.set('Content-Type', "image/svg+xml");
+                res.set("Cache-Control", "no-cache, no-store, must-revalidate");
+                res.send(svg);
+            } catch(err) {
+                res.status(500).send("there was an error parsing excalidraw to svg");
+            }
+        } else {
+            // normal image
+            res.set('Content-Type', image.mime);
+            addNoIndexHeader(image, res);
+            res.send(image.getContent());
         }
-
-        addNoIndexHeader(image, res);
-
-        res.setHeader('Content-Type', image.mime);
-
-        res.send(image.getContent());
     });
 
     // used for PDF viewing