Support custom image format with implicit pixel data (#1558)

.github/workflows/test.yml

@@ -6,8 +6,7 @@ on:
       - main
 env:
   otp: "25.0"
-  # TODO: update to v1.14.2 once it is out
+  elixir: "1.14.2"
-  elixir: "main"
 jobs:
   main:
     runs-on: ubuntu-latest

.github/workflows/uffizzi-build.yml

@@ -3,6 +3,10 @@ on:
   pull_request:
     types: [opened, synchronize, reopened, closed]
 
+env:
+  otp: "25.0"
+  elixir: "1.14.0"
+
 jobs:
   build-application:
     name: Build PR image
@@ -13,6 +17,42 @@ jobs:
     steps:
       - name: Checkout git repo
         uses: actions/checkout@v3
+      # --- START build assets
+      - name: Install Erlang & Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          otp-version: ${{ env.otp }}
+          elixir-version: ${{ env.elixir }}
+      - name: Cache Mix
+        uses: actions/cache@v3
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-
+      # Note: we need to get Phoenix and LV because package.json points to them directly
+      - name: Install mix dependencies
+        run: mix deps.get
+      - name: Install Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18.x"
+      - name: Cache npm dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Clean generated assets
+        run: rm -rf static/{js,css}
+      - name: Install npm dependencies
+        run: npm ci --prefix assets
+      - name: Build assets
+        run: npm run deploy --prefix assets
+        # --- END build assets
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Generate UUID image name

assets/js/hooks/image_input.js

@@ -3,6 +3,7 @@ import {
   getAttributeOrThrow,
   parseInteger,
 } from "../lib/attribute";
+import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
 
 const dropClasses = ["bg-yellow-100", "border-yellow-300"];
 
@@ -258,18 +259,6 @@ function imageDataToRGBBuffer(imageData) {
   return bytes.buffer;
 }
 
-function bufferToBase64(buffer) {
-  let binaryString = "";
-  const bytes = new Uint8Array(buffer);
-  const length = bytes.byteLength;
-
-  for (let i = 0; i < length; i++) {
-    binaryString += String.fromCharCode(bytes[i]);
-  }
-
-  return btoa(binaryString);
-}
-
 function imageInfoToElement(imageInfo) {
   if (imageInfo.format === "png" || imageInfo.format === "jpeg") {
     const src = `data:image/${imageInfo.format};base64,${imageInfo.data}`;
@@ -282,7 +271,7 @@ function imageInfoToElement(imageInfo) {
     const canvas = document.createElement("canvas");
     canvas.height = imageInfo.height;
     canvas.width = imageInfo.width;
-    const buffer = bufferFromBase64(imageInfo.data);
+    const buffer = base64ToBuffer(imageInfo.data);
     const imageData = imageDataFromRGBBuffer(
       buffer,
       imageInfo.width,
@@ -310,16 +299,4 @@ function imageDataFromRGBBuffer(buffer, width, height) {
   return new ImageData(data, width, height);
 }
 
-function bufferFromBase64(base64) {
-  const binaryString = atob(base64);
-  const bytes = new Uint8Array(binaryString.length);
-  const length = bytes.byteLength;
-
-  for (let i = 0; i < length; i++) {
-    bytes[i] = binaryString.charCodeAt(i);
-  }
-
-  return bytes.buffer;
-}
-
 export default ImageInput;

assets/js/hooks/image_output.js

@@ -0,0 +1,82 @@
+import { base64ToBuffer } from "../lib/utils";
+
+/**
+ * A hook for the image output.
+ *
+ * Automatically converts image URLs from image/x-pixel to image/png.
+ */
+const ImageOutput = {
+  mounted() {
+    this.updateSrc();
+  },
+
+  updated() {
+    this.updateSrc();
+  },
+
+  updateSrc() {
+    const dataUrl = this.el.src;
+    const prefix = "data:image/x-pixel;base64,";
+
+    const base64Data = dataUrl.slice(prefix.length);
+
+    if (dataUrl.startsWith(prefix)) {
+      const buffer = base64ToBuffer(base64Data);
+
+      const view = new DataView(buffer);
+      const height = view.getUint32(0, false);
+      const width = view.getUint32(4, false);
+      const channels = view.getUint8(8);
+      const pixelBuffer = buffer.slice(9);
+
+      const imageData = imageDataFromPixelBuffer(
+        pixelBuffer,
+        width,
+        height,
+        channels
+      );
+
+      const canvas = document.createElement("canvas");
+      canvas.height = height;
+      canvas.width = width;
+      canvas.getContext("2d").putImageData(imageData, 0, 0);
+      const pngDataUrl = canvas.toDataURL("image/png");
+
+      this.el.src = pngDataUrl;
+    }
+  },
+};
+
+function imageDataFromPixelBuffer(buffer, width, height, channels) {
+  const pixelCount = width * height;
+  const bytes = new Uint8Array(buffer);
+  const data = new Uint8ClampedArray(pixelCount * 4);
+
+  for (let i = 0; i < pixelCount; i++) {
+    if (channels === 1) {
+      data[i * 4] = bytes[i];
+      data[i * 4 + 1] = bytes[i];
+      data[i * 4 + 2] = bytes[i];
+      data[i * 4 + 3] = 255;
+    } else if (channels === 2) {
+      data[i * 4] = bytes[i * 2];
+      data[i * 4 + 1] = bytes[i * 2];
+      data[i * 4 + 2] = bytes[i * 2];
+      data[i * 4 + 3] = bytes[i * 2 + 1];
+    } else if (channels === 3) {
+      data[i * 4] = bytes[i * 3];
+      data[i * 4 + 1] = bytes[i * 3 + 1];
+      data[i * 4 + 2] = bytes[i * 3 + 2];
+      data[i * 4 + 3] = 255;
+    } else if (channels === 4) {
+      data[i * 4] = bytes[i * 4];
+      data[i * 4 + 1] = bytes[i * 4 + 1];
+      data[i * 4 + 2] = bytes[i * 4 + 2];
+      data[i * 4 + 3] = bytes[i * 4 + 3];
+    }
+  }
+
+  return new ImageData(data, width, height);
+}
+
+export default ImageOutput;

assets/js/hooks/index.js

@@ -7,6 +7,7 @@ import FocusOnUpdate from "./focus_on_update";
 import Headline from "./headline";
 import Highlight from "./highlight";
 import ImageInput from "./image_input";
+import ImageOutput from "./image_output";
 import JSView from "./js_view";
 import KeyboardControl from "./keyboard_control";
 import MarkdownRenderer from "./markdown_renderer";
@@ -27,6 +28,7 @@ export default {
   Headline,
   Highlight,
   ImageInput,
+  ImageOutput,
   JSView,
   KeyboardControl,
   MarkdownRenderer,

assets/js/lib/utils.js

@@ -189,3 +189,33 @@ const htmlEscapes = {
 export function escapeHtml(string) {
   return (string || "").replace(/[&<>"']/g, (char) => htmlEscapes[char]);
 }
+
+/**
+ * Encodes the given binary buffer into base64 string.
+ */
+export function bufferToBase64(buffer) {
+  let binaryString = "";
+  const bytes = new Uint8Array(buffer);
+  const length = bytes.byteLength;
+
+  for (let i = 0; i < length; i++) {
+    binaryString += String.fromCharCode(bytes[i]);
+  }
+
+  return btoa(binaryString);
+}
+
+/**
+ * Decodes a base64 string into a binary buffer.
+ */
+export function base64ToBuffer(base64) {
+  const binaryString = atob(base64);
+  const bytes = new Uint8Array(binaryString.length);
+  const length = bytes.byteLength;
+
+  for (let i = 0; i < length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+
+  return bytes.buffer;
+}

lib/livebook_web/live/output.ex

@@ -63,7 +63,7 @@ defmodule LivebookWeb.Output do
     assigns = %{id: id, content: content, mime_type: mime_type}
 
     ~H"""
-    <Output.ImageComponent.render content={@content} mime_type={@mime_type} />
+    <Output.ImageComponent.render id={@id} content={@content} mime_type={@mime_type} />
     """
   end
 

lib/livebook_web/live/output/image_component.ex

@@ -4,7 +4,13 @@ defmodule LivebookWeb.Output.ImageComponent do
   @impl true
   def render(assigns) do
     ~H"""
-    <img src={data_url(@content, @mime_type)} alt="output image" />
+    <img
+      class="max-h-[500px]"
+      src={data_url(@content, @mime_type)}
+      alt="output image"
+      id={@id}
+      phx-hook="ImageOutput"
+    />
     """
   end