From 15754c9c99cdc3333c107824982370f87687e58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Sat, 19 Nov 2022 15:08:00 +0100 Subject: [PATCH] Add support for image input (#1538) --- assets/js/hooks/image_input.js | 325 ++++++++++++++++++ assets/js/hooks/index.js | 2 + .../live/output/image_input_component.ex | 60 ++++ .../live/output/input_component.ex | 31 ++ 4 files changed, 418 insertions(+) create mode 100644 assets/js/hooks/image_input.js create mode 100644 lib/livebook_web/live/output/image_input_component.ex diff --git a/assets/js/hooks/image_input.js b/assets/js/hooks/image_input.js new file mode 100644 index 000000000..70b3f0074 --- /dev/null +++ b/assets/js/hooks/image_input.js @@ -0,0 +1,325 @@ +import { + getAttributeOrDefault, + getAttributeOrThrow, + parseInteger, +} from "../lib/attribute"; + +const dropClasses = ["bg-yellow-100", "border-yellow-300"]; + +/** + * A hook for client-preprocessed image input. + * + * ## Configuration + * + * * `data-id` - a unique id + * + * * `data-phx-target` - the component to send the `"change"` event to + * + * * `data-height` - the image bounding height + * + * * `data-width` - the image bounding width + * + * * `data-format` - the desired image format + * + * * `data-fit` - the fit strategy + * + * The element should have the following children: + * + * * `[data-input]` - a file input used for file selection + * + * * `[data-preview]` - a container to put image preview in + */ +const ImageInput = { + mounted() { + this.props = this.getProps(); + + this.inputEl = this.el.querySelector(`[data-input]`); + this.previewEl = this.el.querySelector(`[data-preview]`); + + // Render initial value + this.handleEvent(`image_input_init:${this.props.id}`, (imageInfo) => { + const canvas = imageInfoToElement(imageInfo); + this.setPreview(canvas); + }); + + // File selection + + this.el.addEventListener("click", (event) => { + this.inputEl.click(); + }); + + this.inputEl.addEventListener("change", (event) => { + const [file] = event.target.files; + file && this.loadFile(file); + }); + + // Drag and drop + + this.el.addEventListener("dragover", (event) => { + event.stopPropagation(); + event.preventDefault(); + event.dataTransfer.dropEffect = "copy"; + }); + + this.el.addEventListener("drop", (event) => { + event.stopPropagation(); + event.preventDefault(); + const [file] = event.dataTransfer.files; + file && this.loadFile(file); + }); + + this.el.addEventListener("dragenter", (event) => { + this.el.classList.add(...dropClasses); + }); + + this.el.addEventListener("dragleave", (event) => { + if (!this.el.contains(event.relatedTarget)) { + this.el.classList.remove(...dropClasses); + } + }); + + this.el.addEventListener("drop", (event) => { + this.el.classList.remove(...dropClasses); + }); + }, + + updated() { + this.props = this.getProps(); + }, + + getProps() { + return { + id: getAttributeOrThrow(this.el, "data-id"), + phxTarget: getAttributeOrThrow(this.el, "data-phx-target", parseInteger), + height: getAttributeOrDefault(this.el, "data-height", null, parseInteger), + width: getAttributeOrDefault(this.el, "data-width", null, parseInteger), + format: getAttributeOrThrow(this.el, "data-format"), + fit: getAttributeOrThrow(this.el, "data-fit"), + }; + }, + + loadFile(file) { + const reader = new FileReader(); + + reader.onload = (readerEvent) => { + const imgEl = document.createElement("img"); + + imgEl.addEventListener("load", (loadEvent) => { + const canvas = this.toCanvas(imgEl); + + this.setPreview(canvas); + + this.pushEventTo(this.props.phxTarget, "change", { + value: { + data: canvasToBase64(canvas, this.props.format), + height: canvas.height, + width: canvas.width, + }, + }); + }); + + imgEl.src = readerEvent.target.result; + }; + + reader.readAsDataURL(file); + }, + + toCanvas(imgEl) { + const { width, height } = imgEl; + const { width: boundWidth, height: boundHeight } = this.props; + + const canvas = document.createElement("canvas"); + const ctx = canvas.getContext("2d"); + + if ( + (boundWidth === null && boundHeight === null) || + (boundWidth === width && boundHeight === height) + ) { + canvas.width = width; + canvas.height = height; + + canvas + .getContext("2d") + .drawImage(imgEl, 0, 0, width, height, 0, 0, width, height); + } else if (this.props.fit === "contain") { + const widthScale = boundWidth / width; + const heightScale = boundHeight / height; + const scale = Math.min(widthScale, heightScale); + + const scaledWidth = Math.round(width * scale); + const scaledHeight = Math.round(height * scale); + + canvas.width = scaledWidth; + canvas.height = scaledHeight; + + ctx.drawImage( + imgEl, + 0, + 0, + width, + height, + 0, + 0, + scaledWidth, + scaledHeight + ); + } else if (this.props.fit === "crop") { + const widthScale = boundWidth / width; + const heightScale = boundHeight / height; + const scale = Math.max(widthScale, heightScale); + + const scaledWidth = Math.round(width * scale); + const scaledHeight = Math.round(height * scale); + + canvas.width = boundWidth; + canvas.height = boundHeight; + + ctx.drawImage( + imgEl, + Math.round((scaledWidth - boundWidth) / scale / 2), + Math.round((scaledHeight - boundHeight) / scale / 2), + width - Math.round((scaledWidth - boundWidth) / scale), + height - Math.round((scaledHeight - boundHeight) / scale), + 0, + 0, + boundWidth, + boundHeight + ); + } else if (this.props.fit === "pad") { + const widthScale = boundWidth / width; + const heightScale = boundHeight / height; + const scale = Math.min(widthScale, heightScale); + + const scaledWidth = Math.round(width * scale); + const scaledHeight = Math.round(height * scale); + + canvas.width = boundWidth; + canvas.height = boundHeight; + + ctx.fillStyle = "black"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + + ctx.drawImage( + imgEl, + 0, + 0, + width, + height, + Math.round((boundWidth - scaledWidth) / 2), + Math.round((boundHeight - scaledHeight) / 2), + scaledWidth, + scaledHeight + ); + } else { + canvas.width = boundWidth; + canvas.height = boundHeight; + + ctx.drawImage(imgEl, 0, 0, width, height, 0, 0, boundWidth, boundHeight); + } + + return canvas; + }, + + setPreview(element) { + element.style.maxHeight = "300px"; + this.previewEl.replaceChildren(element); + }, +}; + +function canvasToBase64(canvas, format) { + if (format === "png" || format === "jpeg") { + const prefix = `data:image/${format};base64,`; + const dataUrl = canvas.toDataURL(`image/${format}`); + return dataUrl.slice(prefix.length); + } + + if (format === "rgb") { + const imageData = canvas + .getContext("2d") + .getImageData(0, 0, canvas.width, canvas.height); + + const buffer = imageDataToRGBBuffer(imageData); + return bufferToBase64(buffer); + } + + throw new Error(`Unexpected format: ${format}`); +} + +function imageDataToRGBBuffer(imageData) { + const pixelCount = imageData.width * imageData.height; + const bytes = new Uint8ClampedArray(pixelCount * 3); + + for (let i = 0; i < pixelCount; i++) { + bytes[i * 3] = imageData.data[i * 4]; + bytes[i * 3 + 1] = imageData.data[i * 4 + 1]; + bytes[i * 3 + 2] = imageData.data[i * 4 + 2]; + } + + return bytes.buffer; +} + +function bufferToBase64(buffer) { + let binaryString = ""; + const bytes = new Uint8Array(buffer); + const length = bytes.byteLength; + + for (let i = 0; i < length; i++) { + binaryString += String.fromCharCode(bytes[i]); + } + + return btoa(binaryString); +} + +function imageInfoToElement(imageInfo) { + if (imageInfo.format === "png" || imageInfo.format === "jpeg") { + const src = `data:image/${imageInfo.format};base64,${imageInfo.data}`; + const img = document.createElement("img"); + img.src = src; + return img; + } + + if (imageInfo.format === "rgb") { + const canvas = document.createElement("canvas"); + canvas.height = imageInfo.height; + canvas.width = imageInfo.width; + const buffer = bufferFromBase64(imageInfo.data); + const imageData = imageDataFromRGBBuffer( + buffer, + imageInfo.width, + imageInfo.height + ); + canvas.getContext("2d").putImageData(imageData, 0, 0); + return canvas; + } + + throw new Error(`Unexpected format: ${imageInfo.format}`); +} + +function imageDataFromRGBBuffer(buffer, width, height) { + const pixelCount = width * height; + const bytes = new Uint8Array(buffer); + const data = new Uint8ClampedArray(pixelCount * 4); + + for (let i = 0; i < pixelCount; i++) { + data[i * 4] = bytes[i * 3]; + data[i * 4 + 1] = bytes[i * 3 + 1]; + data[i * 4 + 2] = bytes[i * 3 + 2]; + data[i * 4 + 3] = 255; + } + + return new ImageData(data, width, height); +} + +function bufferFromBase64(base64) { + const binaryString = atob(base64); + const bytes = new Uint8Array(binaryString.length); + const length = bytes.byteLength; + + for (let i = 0; i < length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + return bytes.buffer; +} + +export default ImageInput; diff --git a/assets/js/hooks/index.js b/assets/js/hooks/index.js index 2f11c5607..a75f5d117 100644 --- a/assets/js/hooks/index.js +++ b/assets/js/hooks/index.js @@ -6,6 +6,7 @@ import EditorSettings from "./editor_settings"; import FocusOnUpdate from "./focus_on_update"; import Headline from "./headline"; import Highlight from "./highlight"; +import ImageInput from "./image_input"; import JSView from "./js_view"; import KeyboardControl from "./keyboard_control"; import MarkdownRenderer from "./markdown_renderer"; @@ -24,6 +25,7 @@ export default { FocusOnUpdate, Headline, Highlight, + ImageInput, JSView, KeyboardControl, MarkdownRenderer, diff --git a/lib/livebook_web/live/output/image_input_component.ex b/lib/livebook_web/live/output/image_input_component.ex new file mode 100644 index 000000000..1461991d3 --- /dev/null +++ b/lib/livebook_web/live/output/image_input_component.ex @@ -0,0 +1,60 @@ +defmodule LivebookWeb.Output.ImageInputComponent do + use LivebookWeb, :live_component + + @impl true + def mount(socket) do + {:ok, assign(socket, initialized: false)} + end + + @impl true + def update(assigns, socket) do + {value, assigns} = Map.pop!(assigns, :value) + + socket = assign(socket, assigns) + + socket = + if socket.assigns.initialized do + socket + else + socket = + if value do + push_event(socket, "image_input_init:#{socket.assigns.id}", %{ + data: Base.encode64(value.data), + height: value.height, + width: value.width, + format: value.format + }) + else + socket + end + + assign(socket, initialize: true) + end + + {:ok, socket} + end + + @impl true + def render(assigns) do + ~H""" +
+ +
+
+ Drag an image file here or click to open file browser +
+
+
+ """ + end +end diff --git a/lib/livebook_web/live/output/input_component.ex b/lib/livebook_web/live/output/input_component.ex index 32dd3ec7b..4b8709b92 100644 --- a/lib/livebook_web/live/output/input_component.ex +++ b/lib/livebook_web/live/output/input_component.ex @@ -19,6 +19,27 @@ defmodule LivebookWeb.Output.InputComponent do end @impl true + def render(%{attrs: %{type: :image}} = assigns) do + ~H""" +
+
+ <%= @attrs.label %> +
+ + <.live_component + module={LivebookWeb.Output.ImageInputComponent} + id={"#{@id}-input"} + value={@value} + height={@attrs.size && elem(@attrs.size, 0)} + width={@attrs.size && elem(@attrs.size, 1)} + format={@attrs.format} + fit={@attrs.fit} + target={@myself} + /> +
+ """ + end + def render(assigns) do ~H"""
@@ -243,6 +264,16 @@ defmodule LivebookWeb.Output.InputComponent do {:ok, html_value} end + defp parse(html_value, %{type: :image} = attrs) do + {:ok, + %{ + data: Base.decode64!(html_value["data"]), + height: html_value["height"], + width: html_value["width"], + format: attrs.format + }} + end + defp report_event(socket, value) do topic = socket.assigns.attrs.ref event = %{value: value, origin: socket.assigns.client_id, type: :change}