Support custom image format with implicit pixel data (#1558)

This commit is contained in:
Jonatan Kłosko 2022-12-02 12:42:31 +01:00 committed by GitHub
parent 55947f08ec
commit 460eb14420
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 165 additions and 29 deletions

View file

@ -6,8 +6,7 @@ on:
- main
env:
otp: "25.0"
# TODO: update to v1.14.2 once it is out
elixir: "main"
elixir: "1.14.2"
jobs:
main:
runs-on: ubuntu-latest

View file

@ -3,6 +3,10 @@ on:
pull_request:
types: [opened, synchronize, reopened, closed]
env:
otp: "25.0"
elixir: "1.14.0"
jobs:
build-application:
name: Build PR image
@ -13,6 +17,42 @@ jobs:
steps:
- name: Checkout git repo
uses: actions/checkout@v3
# --- START build assets
- name: Install Erlang & Elixir
uses: erlef/setup-beam@v1
with:
otp-version: ${{ env.otp }}
elixir-version: ${{ env.elixir }}
- name: Cache Mix
uses: actions/cache@v3
with:
path: |
deps
_build
key: ${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }}
restore-keys: |
${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-
# Note: we need to get Phoenix and LV because package.json points to them directly
- name: Install mix dependencies
run: mix deps.get
- name: Install Node
uses: actions/setup-node@v3
with:
node-version: "18.x"
- name: Cache npm dependencies
uses: actions/cache@v3
with:
path: ~/.npm
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-
- name: Clean generated assets
run: rm -rf static/{js,css}
- name: Install npm dependencies
run: npm ci --prefix assets
- name: Build assets
run: npm run deploy --prefix assets
# --- END build assets
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Generate UUID image name

View file

@ -3,6 +3,7 @@ import {
getAttributeOrThrow,
parseInteger,
} from "../lib/attribute";
import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
const dropClasses = ["bg-yellow-100", "border-yellow-300"];
@ -258,18 +259,6 @@ function imageDataToRGBBuffer(imageData) {
return bytes.buffer;
}
function bufferToBase64(buffer) {
let binaryString = "";
const bytes = new Uint8Array(buffer);
const length = bytes.byteLength;
for (let i = 0; i < length; i++) {
binaryString += String.fromCharCode(bytes[i]);
}
return btoa(binaryString);
}
function imageInfoToElement(imageInfo) {
if (imageInfo.format === "png" || imageInfo.format === "jpeg") {
const src = `data:image/${imageInfo.format};base64,${imageInfo.data}`;
@ -282,7 +271,7 @@ function imageInfoToElement(imageInfo) {
const canvas = document.createElement("canvas");
canvas.height = imageInfo.height;
canvas.width = imageInfo.width;
const buffer = bufferFromBase64(imageInfo.data);
const buffer = base64ToBuffer(imageInfo.data);
const imageData = imageDataFromRGBBuffer(
buffer,
imageInfo.width,
@ -310,16 +299,4 @@ function imageDataFromRGBBuffer(buffer, width, height) {
return new ImageData(data, width, height);
}
function bufferFromBase64(base64) {
const binaryString = atob(base64);
const bytes = new Uint8Array(binaryString.length);
const length = bytes.byteLength;
for (let i = 0; i < length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes.buffer;
}
export default ImageInput;

View file

@ -0,0 +1,82 @@
import { base64ToBuffer } from "../lib/utils";
/**
* A hook for the image output.
*
* Automatically converts image URLs from image/x-pixel to image/png.
*/
const ImageOutput = {
mounted() {
this.updateSrc();
},
updated() {
this.updateSrc();
},
updateSrc() {
const dataUrl = this.el.src;
const prefix = "data:image/x-pixel;base64,";
const base64Data = dataUrl.slice(prefix.length);
if (dataUrl.startsWith(prefix)) {
const buffer = base64ToBuffer(base64Data);
const view = new DataView(buffer);
const height = view.getUint32(0, false);
const width = view.getUint32(4, false);
const channels = view.getUint8(8);
const pixelBuffer = buffer.slice(9);
const imageData = imageDataFromPixelBuffer(
pixelBuffer,
width,
height,
channels
);
const canvas = document.createElement("canvas");
canvas.height = height;
canvas.width = width;
canvas.getContext("2d").putImageData(imageData, 0, 0);
const pngDataUrl = canvas.toDataURL("image/png");
this.el.src = pngDataUrl;
}
},
};
function imageDataFromPixelBuffer(buffer, width, height, channels) {
const pixelCount = width * height;
const bytes = new Uint8Array(buffer);
const data = new Uint8ClampedArray(pixelCount * 4);
for (let i = 0; i < pixelCount; i++) {
if (channels === 1) {
data[i * 4] = bytes[i];
data[i * 4 + 1] = bytes[i];
data[i * 4 + 2] = bytes[i];
data[i * 4 + 3] = 255;
} else if (channels === 2) {
data[i * 4] = bytes[i * 2];
data[i * 4 + 1] = bytes[i * 2];
data[i * 4 + 2] = bytes[i * 2];
data[i * 4 + 3] = bytes[i * 2 + 1];
} else if (channels === 3) {
data[i * 4] = bytes[i * 3];
data[i * 4 + 1] = bytes[i * 3 + 1];
data[i * 4 + 2] = bytes[i * 3 + 2];
data[i * 4 + 3] = 255;
} else if (channels === 4) {
data[i * 4] = bytes[i * 4];
data[i * 4 + 1] = bytes[i * 4 + 1];
data[i * 4 + 2] = bytes[i * 4 + 2];
data[i * 4 + 3] = bytes[i * 4 + 3];
}
}
return new ImageData(data, width, height);
}
export default ImageOutput;

View file

@ -7,6 +7,7 @@ import FocusOnUpdate from "./focus_on_update";
import Headline from "./headline";
import Highlight from "./highlight";
import ImageInput from "./image_input";
import ImageOutput from "./image_output";
import JSView from "./js_view";
import KeyboardControl from "./keyboard_control";
import MarkdownRenderer from "./markdown_renderer";
@ -27,6 +28,7 @@ export default {
Headline,
Highlight,
ImageInput,
ImageOutput,
JSView,
KeyboardControl,
MarkdownRenderer,

View file

@ -189,3 +189,33 @@ const htmlEscapes = {
export function escapeHtml(string) {
return (string || "").replace(/[&<>"']/g, (char) => htmlEscapes[char]);
}
/**
* Encodes the given binary buffer into base64 string.
*/
export function bufferToBase64(buffer) {
let binaryString = "";
const bytes = new Uint8Array(buffer);
const length = bytes.byteLength;
for (let i = 0; i < length; i++) {
binaryString += String.fromCharCode(bytes[i]);
}
return btoa(binaryString);
}
/**
* Decodes a base64 string into a binary buffer.
*/
export function base64ToBuffer(base64) {
const binaryString = atob(base64);
const bytes = new Uint8Array(binaryString.length);
const length = bytes.byteLength;
for (let i = 0; i < length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes.buffer;
}

View file

@ -63,7 +63,7 @@ defmodule LivebookWeb.Output do
assigns = %{id: id, content: content, mime_type: mime_type}
~H"""
<Output.ImageComponent.render content={@content} mime_type={@mime_type} />
<Output.ImageComponent.render id={@id} content={@content} mime_type={@mime_type} />
"""
end

View file

@ -4,7 +4,13 @@ defmodule LivebookWeb.Output.ImageComponent do
@impl true
def render(assigns) do
~H"""
<img src={data_url(@content, @mime_type)} alt="output image" />
<img
class="max-h-[500px]"
src={data_url(@content, @mime_type)}
alt="output image"
id={@id}
phx-hook="ImageOutput"
/>
"""
end