Stream audio/image input values into and out of the server (#2249)

Co-authored-by: José Valim <jose.valim@dashbit.co>

assets/js/hooks/audio_input.js

@@ -1,5 +1,9 @@
-import { getAttributeOrThrow, parseInteger } from "../lib/attribute";
+import {
-import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
+  getAttributeOrDefault,
+  getAttributeOrThrow,
+  parseInteger,
+} from "../lib/attribute";
+import { encodeAnnotatedBuffer, encodePcmAsWav } from "../lib/codec";
 
 const dropClasses = ["bg-yellow-100", "border-yellow-300"];
 
@@ -18,6 +22,8 @@ const dropClasses = ["bg-yellow-100", "border-yellow-300"];
  *
  *   * `data-endianness` - the server endianness, either `"little"` or `"big"`
  *
+ *   * `data-audio-url` - the URL to audio file to use for the current preview
+ *
  */
 const AudioInput = {
   mounted() {
@@ -32,21 +38,8 @@ const AudioInput = {
 
     this.mediaRecorder = null;
 
-    // Render updated value
+    // Set the current value URL
-    this.handleEvent(
+    this.audioEl.src = this.props.audioUrl;
-      `audio_input_change:${this.props.id}`,
-      ({ audio_info: audioInfo }) => {
-        if (audioInfo) {
-          this.updatePreview({
-            data: this.decodeAudio(base64ToBuffer(audioInfo.data)),
-            numChannels: audioInfo.num_channels,
-            samplingRate: audioInfo.sampling_rate,
-          });
-        } else {
-          this.clearPreview();
-        }
-      }
-    );
 
     // File selection
 
@@ -105,6 +98,8 @@ const AudioInput = {
 
   updated() {
     this.props = this.getProps();
+
+    this.audioEl.src = this.props.audioUrl;
   },
 
   getProps() {
@@ -118,6 +113,7 @@ const AudioInput = {
       ),
       endianness: getAttributeOrThrow(this.el, "data-endianness"),
       format: getAttributeOrThrow(this.el, "data-format"),
+      audioUrl: getAttributeOrDefault(this.el, "data-audio-url", null),
     };
   },
 
@@ -176,6 +172,8 @@ const AudioInput = {
   },
 
   loadEncodedAudio(buffer) {
+    this.pushEventTo(this.props.phxTarget, "decoding", {});
+
     const context = new AudioContext({ sampleRate: this.props.samplingRate });
 
     context.decodeAudioData(buffer, (audioBuffer) => {
@@ -184,66 +182,31 @@ const AudioInput = {
     });
   },
 
-  updatePreview(audioInfo) {
-    const oldUrl = this.audioEl.src;
-    const blob = audioInfoToWavBlob(audioInfo);
-    this.audioEl.src = URL.createObjectURL(blob);
-    oldUrl && URL.revokeObjectURL(oldUrl);
-  },
-
-  clearPreview() {
-    const oldUrl = this.audioEl.src;
-    this.audioEl.src = "";
-    oldUrl && URL.revokeObjectURL(oldUrl);
-  },
-
   pushAudio(audioInfo) {
-    this.pushEventTo(this.props.phxTarget, "change", {
+    const meta = {
-      data: bufferToBase64(this.encodeAudio(audioInfo)),
       num_channels: audioInfo.numChannels,
       sampling_rate: audioInfo.samplingRate,
-    });
+    };
+
+    const buffer = this.encodeAudio(audioInfo);
+
+    const blob = new Blob([buffer]);
+    blob.meta = () => meta;
+
+    this.uploadTo(this.props.phxTarget, "file", [blob]);
   },
 
   encodeAudio(audioInfo) {
     if (this.props.format === "pcm_f32") {
-      return this.fixEndianness32(audioInfo.data);
+      return convertEndianness32(audioInfo.data, this.props.endianness);
     } else if (this.props.format === "wav") {
-      return encodeWavData(
+      return encodePcmAsWav(
         audioInfo.data,
         audioInfo.numChannels,
         audioInfo.samplingRate
       );
     }
   },
-
-  decodeAudio(buffer) {
-    if (this.props.format === "pcm_f32") {
-      return this.fixEndianness32(buffer);
-    } else if (this.props.format === "wav") {
-      return decodeWavData(buffer);
-    }
-  },
-
-  fixEndianness32(buffer) {
-    if (getEndianness() === this.props.endianness) {
-      return buffer;
-    }
-
-    // If the server uses different endianness, we swap bytes accordingly
-    for (let i = 0; i < buffer.byteLength / 4; i++) {
-      const b1 = buffer[i];
-      const b2 = buffer[i + 1];
-      const b3 = buffer[i + 2];
-      const b4 = buffer[i + 3];
-      buffer[i] = b4;
-      buffer[i + 1] = b3;
-      buffer[i + 2] = b2;
-      buffer[i + 3] = b1;
-    }
-
-    return buffer;
-  },
 };
 
 function audioBufferToAudioInfo(audioBuffer) {
@@ -267,88 +230,24 @@ function audioBufferToAudioInfo(audioBuffer) {
   return { data: pcmArray.buffer, numChannels, samplingRate };
 }
 
-function audioInfoToWavBlob({ data, numChannels, samplingRate }) {
+function convertEndianness32(buffer, targetEndianness) {
-  const wavBytes = encodeWavData(data, numChannels, samplingRate);
+  if (getEndianness() === targetEndianness) {
-  return new Blob([wavBytes], { type: "audio/wav" });
+    return buffer;
   }
 
-// See http://soundfile.sapp.org/doc/WaveFormat
+  // If the server uses different endianness, we swap bytes accordingly
-function encodeWavData(buffer, numChannels, samplingRate) {
+  for (let i = 0; i < buffer.byteLength / 4; i++) {
-  const HEADER_SIZE = 44;
+    const b1 = buffer[i];
-
+    const b2 = buffer[i + 1];
-  const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength);
+    const b3 = buffer[i + 2];
-  const view = new DataView(wavBuffer);
+    const b4 = buffer[i + 3];
-
+    buffer[i] = b4;
-  const numFrames = buffer.byteLength / 4;
+    buffer[i + 1] = b3;
-  const bytesPerSample = 4;
+    buffer[i + 2] = b2;
-
+    buffer[i + 3] = b1;
-  const blockAlign = numChannels * bytesPerSample;
-  const byteRate = samplingRate * blockAlign;
-  const dataSize = numFrames * blockAlign;
-
-  let offset = 0;
-
-  function writeUint32Big(int) {
-    view.setUint32(offset, int, false);
-    offset += 4;
   }
 
-  function writeUint32(int) {
+  return buffer;
-    view.setUint32(offset, int, true);
-    offset += 4;
-  }
-
-  function writeUint16(int) {
-    view.setUint16(offset, int, true);
-    offset += 2;
-  }
-
-  function writeFloat32(int) {
-    view.setFloat32(offset, int, true);
-    offset += 4;
-  }
-
-  writeUint32Big(0x52494646);
-  writeUint32(36 + dataSize);
-  writeUint32Big(0x57415645);
-
-  writeUint32Big(0x666d7420);
-  writeUint32(16);
-  writeUint16(3); // 3 represents 32-bit float PCM
-  writeUint16(numChannels);
-  writeUint32(samplingRate);
-  writeUint32(byteRate);
-  writeUint16(blockAlign);
-  writeUint16(bytesPerSample * 8);
-
-  writeUint32Big(0x64617461);
-  writeUint32(dataSize);
-
-  const array = new Float32Array(buffer);
-
-  for (let i = 0; i < array.length; i++) {
-    writeFloat32(array[i]);
-  }
-
-  return wavBuffer;
-}
-
-// We assume the exact same format as above, since we only need to
-// decode data we encoded previously
-function decodeWavData(buffer) {
-  const HEADER_SIZE = 44;
-
-  const pcmBuffer = new ArrayBuffer(buffer.byteLength - HEADER_SIZE);
-  const pcmArray = new Float32Array(pcmBuffer);
-
-  const view = new DataView(buffer);
-
-  for (let i = 0; i < pcmArray.length; i++) {
-    const offset = HEADER_SIZE + i * 4;
-    pcmArray[i] = view.getFloat32(offset, true);
-  }
-
-  return pcmBuffer;
 }
 
 function getEndianness() {

assets/js/hooks/image_input.js

@@ -3,7 +3,7 @@ import {
   getAttributeOrThrow,
   parseInteger,
 } from "../lib/attribute";
-import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
+import { encodeAnnotatedBuffer } from "../lib/codec";
 
 const dropClasses = ["bg-yellow-100", "border-yellow-300"];
 
@@ -24,6 +24,12 @@ const dropClasses = ["bg-yellow-100", "border-yellow-300"];
  *
  *   * `data-fit` - the fit strategy
  *
+ *   * `data-image-url` - the URL to the image binary value
+ *
+ *   * `data-value-height` - the height of the current image value
+ *
+ *   * `data-value-width` - the width fo the current image value
+ *
  */
 const ImageInput = {
   mounted() {
@@ -50,18 +56,7 @@ const ImageInput = {
     this.cameraVideoEl = null;
     this.cameraStream = null;
 
-    // Render updated value
+    this.updateImagePreview();
-    this.handleEvent(
-      `image_input_change:${this.props.id}`,
-      ({ image_info: imageInfo }) => {
-        if (imageInfo) {
-          const canvas = imageInfoToElement(imageInfo, this.props.format);
-          this.setPreview(canvas);
-        } else {
-          this.setPreview(this.initialPreviewContentEl);
-        }
-      }
-    );
 
     // File selection
 
@@ -139,6 +134,8 @@ const ImageInput = {
 
   updated() {
     this.props = this.getProps();
+
+    this.updateImagePreview();
   },
 
   getProps() {
@@ -149,9 +146,37 @@ const ImageInput = {
       width: getAttributeOrDefault(this.el, "data-width", null, parseInteger),
       format: getAttributeOrThrow(this.el, "data-format"),
       fit: getAttributeOrThrow(this.el, "data-fit"),
+      imageUrl: getAttributeOrDefault(this.el, "data-image-url", null),
+      valueHeight: getAttributeOrDefault(
+        this.el,
+        "data-value-height",
+        null,
+        parseInteger
+      ),
+      valueWidth: getAttributeOrDefault(
+        this.el,
+        "data-value-width",
+        null,
+        parseInteger
+      ),
     };
   },
 
+  updateImagePreview() {
+    if (this.props.imageUrl) {
+      buildPreviewElement(
+        this.props.imageUrl,
+        this.props.valueHeight,
+        this.props.valueWidth,
+        this.props.format
+      ).then((element) => {
+        this.setPreview(element);
+      });
+    } else {
+      this.setPreview(this.initialPreviewContentEl);
+    }
+  },
+
   loadFile(file) {
     const reader = new FileReader();
 
@@ -272,10 +297,16 @@ const ImageInput = {
   },
 
   pushImage(canvas) {
-    this.pushEventTo(this.props.phxTarget, "change", {
+    canvasToBuffer(canvas, this.props.format).then((buffer) => {
-      data: canvasToBase64(canvas, this.props.format),
+      const meta = {
         height: canvas.height,
         width: canvas.width,
+      };
+
+      const blob = new Blob([buffer]);
+      blob.meta = () => meta;
+
+      this.uploadTo(this.props.phxTarget, "file", [blob]);
     });
   },
 
@@ -397,11 +428,15 @@ const ImageInput = {
   },
 };
 
-function canvasToBase64(canvas, format) {
+function canvasToBuffer(canvas, format) {
   if (format === "png" || format === "jpeg") {
-    const prefix = `data:image/${format};base64,`;
+    return new Promise((resolve, reject) => {
-    const dataUrl = canvas.toDataURL(`image/${format}`);
+      canvas.toBlob((blob) => {
-    return dataUrl.slice(prefix.length);
+        blob.arrayBuffer().then((buffer) => {
+          resolve(buffer);
+        });
+      }, `image/${format}`);
+    });
   }
 
   if (format === "rgb") {
@@ -410,7 +445,7 @@ function canvasToBase64(canvas, format) {
       .getImageData(0, 0, canvas.width, canvas.height);
 
     const buffer = imageDataToRGBBuffer(imageData);
-    return bufferToBase64(buffer);
+    return Promise.resolve(buffer);
   }
 
   throw new Error(`Unexpected format: ${format}`);
@@ -429,26 +464,24 @@ function imageDataToRGBBuffer(imageData) {
   return bytes.buffer;
 }
 
-function imageInfoToElement(imageInfo, format) {
+function buildPreviewElement(imageUrl, height, width, format) {
   if (format === "png" || format === "jpeg") {
-    const src = `data:image/${format};base64,${imageInfo.data}`;
     const img = document.createElement("img");
-    img.src = src;
+    img.src = imageUrl;
-    return img;
+    return Promise.resolve(img);
   }
 
   if (format === "rgb") {
+    return fetch(imageUrl)
+      .then((response) => response.arrayBuffer())
+      .then((buffer) => {
         const canvas = document.createElement("canvas");
-    canvas.height = imageInfo.height;
+        canvas.height = height;
-    canvas.width = imageInfo.width;
+        canvas.width = width;
-    const buffer = base64ToBuffer(imageInfo.data);
+        const imageData = imageDataFromRGBBuffer(buffer, width, height);
-    const imageData = imageDataFromRGBBuffer(
-      buffer,
-      imageInfo.width,
-      imageInfo.height
-    );
         canvas.getContext("2d").putImageData(imageData, 0, 0);
         return canvas;
+      });
   }
 
   throw new Error(`Unexpected format: ${format}`);

assets/js/hooks/js_view/channel.js

@@ -1,4 +1,5 @@
 import { Socket } from "phoenix";
+import { decodeAnnotatedBuffer, encodeAnnotatedBuffer } from "../../lib/codec";
 
 const csrfToken = document
   .querySelector("meta[name='csrf-token']")
@@ -43,7 +44,7 @@ export function transportEncode(meta, payload) {
     payload[1].constructor === ArrayBuffer
   ) {
     const [info, buffer] = payload;
-    return encode([meta, info], buffer);
+    return encodeAnnotatedBuffer([meta, info], buffer);
   } else {
     return { root: [meta, payload] };
   }
@@ -51,7 +52,7 @@ export function transportEncode(meta, payload) {
 
 export function transportDecode(raw) {
   if (raw.constructor === ArrayBuffer) {
-    const [[meta, info], buffer] = decode(raw);
+    const [[meta, info], buffer] = decodeAnnotatedBuffer(raw);
     return [meta, [info, buffer]];
   } else {
     const {
@@ -60,36 +61,3 @@ export function transportDecode(raw) {
     return [meta, payload];
   }
 }
-
-const HEADER_LENGTH = 4;
-
-function encode(meta, buffer) {
-  const encoder = new TextEncoder();
-  const metaArray = encoder.encode(JSON.stringify(meta));
-
-  const raw = new ArrayBuffer(
-    HEADER_LENGTH + metaArray.byteLength + buffer.byteLength
-  );
-  const view = new DataView(raw);
-
-  view.setUint32(0, metaArray.byteLength);
-  new Uint8Array(raw, HEADER_LENGTH, metaArray.byteLength).set(metaArray);
-  new Uint8Array(raw, HEADER_LENGTH + metaArray.byteLength).set(
-    new Uint8Array(buffer)
-  );
-
-  return raw;
-}
-
-function decode(raw) {
-  const view = new DataView(raw);
-  const metaArrayLength = view.getUint32(0);
-
-  const metaArray = new Uint8Array(raw, HEADER_LENGTH, metaArrayLength);
-  const buffer = raw.slice(HEADER_LENGTH + metaArrayLength);
-
-  const decoder = new TextDecoder();
-  const meta = JSON.parse(decoder.decode(metaArray));
-
-  return [meta, buffer];
-}

assets/js/lib/codec.js

@@ -0,0 +1,103 @@
+/**
+ * Encodes PCM float-32 in native endianness into a WAV binary.
+ */
+export function encodePcmAsWav(buffer, numChannels, samplingRate) {
+  // See http://soundfile.sapp.org/doc/WaveFormat
+
+  const HEADER_SIZE = 44;
+
+  const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength);
+  const view = new DataView(wavBuffer);
+
+  const numFrames = buffer.byteLength / 4;
+  const bytesPerSample = 4;
+
+  const blockAlign = numChannels * bytesPerSample;
+  const byteRate = samplingRate * blockAlign;
+  const dataSize = numFrames * blockAlign;
+
+  let offset = 0;
+
+  function writeUint32Big(int) {
+    view.setUint32(offset, int, false);
+    offset += 4;
+  }
+
+  function writeUint32(int) {
+    view.setUint32(offset, int, true);
+    offset += 4;
+  }
+
+  function writeUint16(int) {
+    view.setUint16(offset, int, true);
+    offset += 2;
+  }
+
+  function writeFloat32(int) {
+    view.setFloat32(offset, int, true);
+    offset += 4;
+  }
+
+  writeUint32Big(0x52494646);
+  writeUint32(36 + dataSize);
+  writeUint32Big(0x57415645);
+
+  writeUint32Big(0x666d7420);
+  writeUint32(16);
+  writeUint16(3); // 3 represents 32-bit float PCM
+  writeUint16(numChannels);
+  writeUint32(samplingRate);
+  writeUint32(byteRate);
+  writeUint16(blockAlign);
+  writeUint16(bytesPerSample * 8);
+
+  writeUint32Big(0x64617461);
+  writeUint32(dataSize);
+
+  const array = new Float32Array(buffer);
+
+  for (let i = 0; i < array.length; i++) {
+    writeFloat32(array[i]);
+  }
+
+  return wavBuffer;
+}
+
+const HEADER_LENGTH = 4;
+
+/**
+ * Builds a single buffer with JSON-serialized `meta` and `buffer`.
+ */
+export function encodeAnnotatedBuffer(meta, buffer) {
+  const encoder = new TextEncoder();
+  const metaArray = encoder.encode(JSON.stringify(meta));
+
+  const raw = new ArrayBuffer(
+    HEADER_LENGTH + metaArray.byteLength + buffer.byteLength
+  );
+  const view = new DataView(raw);
+
+  view.setUint32(0, metaArray.byteLength);
+  new Uint8Array(raw, HEADER_LENGTH, metaArray.byteLength).set(metaArray);
+  new Uint8Array(raw, HEADER_LENGTH + metaArray.byteLength).set(
+    new Uint8Array(buffer)
+  );
+
+  return raw;
+}
+
+/**
+ * Decodes binary annotated with JSON-serialized metadata.
+ */
+export function decodeAnnotatedBuffer(raw) {
+  const view = new DataView(raw);
+  const metaArrayLength = view.getUint32(0);
+
+  const metaArray = new Uint8Array(raw, HEADER_LENGTH, metaArrayLength);
+  const buffer = raw.slice(HEADER_LENGTH + metaArrayLength);
+
+  const decoder = new TextDecoder();
+  const meta = JSON.parse(decoder.decode(metaArray));
+
+  return [meta, buffer];
+}

assets/package-lock.json

@@ -51,14 +51,14 @@
       }
     },
     "../deps/phoenix": {
-      "version": "1.7.5",
+      "version": "1.7.7",
       "license": "MIT"
     },
     "../deps/phoenix_html": {
-      "version": "3.3.1"
+      "version": "3.3.2"
     },
     "../deps/phoenix_live_view": {
-      "version": "0.19.5",
+      "version": "0.20.0",
       "license": "MIT"
     },
     "node_modules/@alloc/quick-lru": {

lib/livebook/notebook/cell.ex

@@ -104,10 +104,4 @@ defmodule Livebook.Notebook.Cell do
   """
   @spec setup_cell_id() :: id()
   def setup_cell_id(), do: @setup_cell_id
-
-  @doc """
-  Checks if the given term is a file input value (info map).
-  """
-  defguard is_file_input_value(value)
-           when is_map_key(value, :file_ref) and is_map_key(value, :client_name)
 end

lib/livebook/session.ex

@@ -83,8 +83,6 @@ defmodule Livebook.Session do
 
   use GenServer, restart: :temporary
 
-  import Livebook.Notebook.Cell, only: [is_file_input_value: 1]
-
   alias Livebook.NotebookManager
   alias Livebook.Session.{Data, FileGuard}
   alias Livebook.{Utils, Notebook, Delta, Runtime, LiveMarkdown, FileSystem}
@@ -1854,6 +1852,17 @@ defmodule Livebook.Session do
     end
   end
 
+  @doc """
+  Returns a local path to a session-registered file with the given
+  reference.
+  """
+  @spec registered_file_path(id(), Livebook.Runtime.file_ref()) :: String.t()
+  def registered_file_path(session_id, file_ref) do
+    {:file, file_id} = file_ref
+    %{path: session_dir} = session_tmp_dir(session_id)
+    Path.join([session_dir, "registered_files", file_id])
+  end
+
   defp encode_path_component(component) do
     String.replace(component, [".", "/", "\\", ":"], "_")
   end
@@ -2283,14 +2292,8 @@ defmodule Livebook.Session do
   end
 
   defp handle_action(state, {:clean_up_input_values, input_infos}) do
-    for {_input_id, %{value: value}} <- input_infos do
+    for {_input_id, %{value: %{file_ref: file_ref}}} <- input_infos do
-      case value do
+      schedule_file_deletion(state, file_ref)
-        value when is_file_input_value(value) ->
-          schedule_file_deletion(state, value.file_ref)
-
-        _ ->
-          :ok
-      end
     end
 
     state
@@ -2527,11 +2530,6 @@ defmodule Livebook.Session do
     end
   end
 
-  defp registered_file_path(session_id, {:file, file_id}) do
-    %{path: session_dir} = session_tmp_dir(session_id)
-    Path.join([session_dir, "registered_files", file_id])
-  end
-
   defp schedule_file_deletion(state, file_ref) do
     Process.send_after(
       self(),

lib/livebook_web/channels/js_view_channel.ex

@@ -1,6 +1,8 @@
 defmodule LivebookWeb.JSViewChannel do
   use Phoenix.Channel
 
+  alias LivebookWeb.Helpers.Codec
+
   @impl true
   def join("js_view", %{"session_token" => session_token}, socket) do
     case Phoenix.Token.verify(LivebookWeb.Endpoint, "session", session_token) do
@@ -154,7 +156,7 @@ defmodule LivebookWeb.JSViewChannel do
   # payload accordingly
 
   defp transport_encode!(meta, {:binary, info, binary}) do
-    {:binary, encode!([meta, info], binary)}
+    {:binary, Codec.encode_annotated_binary!([meta, info], binary)}
   end
 
   defp transport_encode!(meta, payload) do
@@ -162,7 +164,7 @@ defmodule LivebookWeb.JSViewChannel do
   end
 
   defp transport_decode!({:binary, raw}) do
-    {[meta, info], binary} = decode!(raw)
+    {[meta, info], binary} = Codec.decode_annotated_binary!(raw)
     {meta, {:binary, info, binary}}
   end
 
@@ -170,16 +172,4 @@ defmodule LivebookWeb.JSViewChannel do
     %{"root" => [meta, payload]} = raw
     {meta, payload}
   end
-
-  defp encode!(meta, binary) do
-    meta = Jason.encode!(meta)
-    meta_size = byte_size(meta)
-    <<meta_size::size(32), meta::binary, binary::binary>>
-  end
-
-  defp decode!(raw) do
-    <<meta_size::size(32), meta::binary-size(meta_size), binary::binary>> = raw
-    meta = Jason.decode!(meta)
-    {meta, binary}
-  end
 end

lib/livebook_web/components/form_components.ex

@@ -603,33 +603,54 @@ defmodule LivebookWeb.FormComponents do
       <.live_file_input upload={@upload} class="hidden" />
       <.label><%= @label %></.label>
       <div :for={entry <- @upload.entries} class="flex flex-col gap-1">
+        <.file_entry entry={entry} on_clear={@on_clear} />
+      </div>
+    </div>
+    """
+  end
+
+  @doc """
+  Renders a file entry with progress.
+
+  ## Examples
+
+      <.file_entry
+        entry={entry}
+        on_clear={JS.push("clear_file", target: @myself)}
+      />
+
+  """
+  attr :entry, Phoenix.LiveView.UploadEntry, required: true
+  attr :on_clear, Phoenix.LiveView.JS, required: true
+  attr :name, :string, default: nil
+
+  def file_entry(assigns) do
+    ~H"""
     <div class="flex flex-col gap-0.5">
-          <div class="flex items-center justify-between text-gray-700">
+      <div class="flex items-center justify-between gap-1 text-gray-700">
-            <span><%= entry.client_name %></span>
+        <span><%= @name || @entry.client_name %></span>
         <button
           type="button"
-              class="ml-1 text-gray-500 hover:text-gray-900"
+          class="text-gray-500 hover:text-gray-900"
           phx-click={@on_clear}
-              phx-value-ref={entry.ref}
+          phx-value-ref={@entry.ref}
           tabindex="-1"
         >
           <.remix_icon icon="close-line" />
         </button>
         <span class="flex-grow"></span>
-            <span :if={entry.preflighted?} class="text-sm font-medium">
+        <span :if={@entry.preflighted?} class="text-sm font-medium">
-              <%= entry.progress %>%
+          <%= @entry.progress %>%
         </span>
       </div>
-          <div :if={entry.preflighted?} class="w-full h-2 rounded-lg bg-blue-200">
+      <div :if={@entry.preflighted?} class="w-full h-2 rounded-lg bg-blue-200">
         <div
           class="h-full rounded-lg bg-blue-600 transition-all ease-out duration-1000"
-              style={"width: #{entry.progress}%"}
+          style={"width: #{@entry.progress}%"}
         >
         </div>
       </div>
     </div>
-      </div>
-    </div>
     """
   end
 

lib/livebook_web/controllers/session_controller.ex

@@ -2,6 +2,7 @@ defmodule LivebookWeb.SessionController do
   use LivebookWeb, :controller
 
   alias Livebook.{Sessions, Session, FileSystem}
+  alias LivebookWeb.Helpers.Codec
 
   def show_file(conn, %{"id" => id, "name" => name}) do
     with {:ok, session} <- Sessions.fetch_session(id),
@@ -173,6 +174,91 @@ defmodule LivebookWeb.SessionController do
     end
   end
 
+  def show_input_audio(conn, %{"token" => token}) do
+    {live_view_pid, input_id} = LivebookWeb.SessionHelpers.verify_input_token!(token)
+
+    case GenServer.call(live_view_pid, {:get_input_value, input_id}) do
+      {:ok, session_id, value} ->
+        path = Livebook.Session.registered_file_path(session_id, value.file_ref)
+
+        conn =
+          conn
+          |> cache_permanently()
+          |> put_resp_header("accept-ranges", "bytes")
+
+        case value.format do
+          :pcm_f32 ->
+            %{size: file_size} = File.stat!(path)
+
+            total_size = Codec.pcm_as_wav_size(file_size)
+
+            case parse_byte_range(conn, total_size) do
+              {range_start, range_end} when range_start > 0 or range_end < total_size - 1 ->
+                stream =
+                  Codec.encode_pcm_as_wav_stream!(
+                    path,
+                    file_size,
+                    value.num_channels,
+                    value.sampling_rate,
+                    range_start,
+                    range_end - range_start + 1
+                  )
+
+                conn
+                |> put_content_range(range_start, range_end, total_size)
+                |> send_stream(206, stream)
+
+              _ ->
+                stream =
+                  Codec.encode_pcm_as_wav_stream!(
+                    path,
+                    file_size,
+                    value.num_channels,
+                    value.sampling_rate,
+                    0,
+                    total_size
+                  )
+
+                conn
+                |> put_resp_header("content-length", Integer.to_string(total_size))
+                |> send_stream(200, stream)
+            end
+
+          :wav ->
+            %{size: total_size} = File.stat!(path)
+
+            case parse_byte_range(conn, total_size) do
+              {range_start, range_end} when range_start > 0 or range_end < total_size - 1 ->
+                conn
+                |> put_content_range(range_start, range_end, total_size)
+                |> send_file(206, path, range_start, range_end - range_start + 1)
+
+              _ ->
+                send_file(conn, 200, path)
+            end
+        end
+
+      :error ->
+        send_resp(conn, 404, "Not found")
+    end
+  end
+
+  def show_input_image(conn, %{"token" => token}) do
+    {live_view_pid, input_id} = LivebookWeb.SessionHelpers.verify_input_token!(token)
+
+    case GenServer.call(live_view_pid, {:get_input_value, input_id}) do
+      {:ok, session_id, value} ->
+        path = Livebook.Session.registered_file_path(session_id, value.file_ref)
+
+        conn
+        |> cache_permanently()
+        |> send_file(200, path)
+
+      :error ->
+        send_resp(conn, 404, "Not found")
+    end
+  end
+
   defp accept_encoding?(conn, encoding) do
     encoding? = &String.contains?(&1, [encoding, "*"])
 
@@ -217,4 +303,55 @@ defmodule LivebookWeb.SessionController do
     content_type = MIME.from_path(path)
     put_resp_header(conn, "content-type", content_type)
   end
+
+  defp parse_byte_range(conn, total_size) do
+    with [range] <- get_req_header(conn, "range"),
+         %{"bytes" => bytes} <- Plug.Conn.Utils.params(range),
+         {range_start, range_end} <- start_and_end(bytes, total_size) do
+      {range_start, range_end}
+    else
+      _ -> :error
+    end
+  end
+
+  defp start_and_end("-" <> rest, total_size) do
+    case Integer.parse(rest) do
+      {last, ""} when last > 0 and last <= total_size -> {total_size - last, total_size - 1}
+      _ -> :error
+    end
+  end
+
+  defp start_and_end(range, total_size) do
+    case Integer.parse(range) do
+      {first, "-"} when first >= 0 ->
+        {first, total_size - 1}
+
+      {first, "-" <> rest} when first >= 0 ->
+        case Integer.parse(rest) do
+          {last, ""} when last >= first -> {first, min(last, total_size - 1)}
+          _ -> :error
+        end
+
+      _ ->
+        :error
+    end
+  end
+
+  defp put_content_range(conn, range_start, range_end, total_size) do
+    put_resp_header(conn, "content-range", "bytes #{range_start}-#{range_end}/#{total_size}")
+  end
+
+  defp send_stream(conn, status, stream) do
+    conn = send_chunked(conn, status)
+
+    Enum.reduce_while(stream, conn, fn chunk, conn ->
+      case Plug.Conn.chunk(conn, chunk) do
+        {:ok, conn} ->
+          {:cont, conn}
+
+        {:error, :closed} ->
+          {:halt, conn}
+      end
+    end)
+  end
 end

lib/livebook_web/helpers/codec.ex

@@ -0,0 +1,130 @@
+defmodule LivebookWeb.Helpers.Codec do
+  @wav_header_size 44
+
+  @doc """
+  Returns the size of a WAV binary that would wrap PCM data of the
+  given size.
+
+  This size matches the result of `encode_pcm_as_wav_stream!/6`.
+  """
+  @spec pcm_as_wav_size(pos_integer()) :: pos_integer()
+  def pcm_as_wav_size(pcm_size) do
+    @wav_header_size + pcm_size
+  end
+
+  @doc """
+  Encodes PCM float-32 in native endianness into a WAV binary.
+
+  Accepts a range of the WAV binary that should be returned. Returns
+  a stream, where the PCM binary is streamed from the given file.
+  """
+  @spec encode_pcm_as_wav_stream!(
+          Path.t(),
+          non_neg_integer(),
+          pos_integer(),
+          pos_integer(),
+          non_neg_integer(),
+          pos_integer()
+        ) :: Enumerable.t()
+  def encode_pcm_as_wav_stream!(path, file_size, num_channels, sampling_rate, offset, length) do
+    {header_enum, file_length} =
+      if offset < @wav_header_size do
+        header = encode_pcm_as_wav_header(file_size, num_channels, sampling_rate)
+        header_length = min(@wav_header_size - offset, length)
+        header_slice = binary_slice(header, offset, header_length)
+        {[header_slice], length - header_length}
+      else
+        {[], length}
+      end
+
+    file_offset = max(offset - @wav_header_size, 0)
+
+    file_stream = raw_file_range_stream!(path, file_offset, file_length)
+
+    file_stream =
+      case System.endianness() do
+        :little ->
+          file_stream
+
+        :big ->
+          Stream.map(file_stream, fn binary ->
+            for <<x::32-float-big <- binary>>, reduce: <<>> do
+              acc -> <<acc::binary, x::32-float-little>>
+            end
+          end)
+      end
+
+    Stream.concat(header_enum, file_stream)
+  end
+
+  defp encode_pcm_as_wav_header(pcm_size, num_channels, sampling_rate) do
+    # See http://soundfile.sapp.org/doc/WaveFormat
+
+    num_frames = div(pcm_size, 4)
+    bytes_per_sample = 4
+
+    block_align = num_channels * bytes_per_sample
+    byte_rate = sampling_rate * block_align
+    data_size = num_frames * block_align
+
+    <<
+      "RIFF",
+      36 + data_size::32-unsigned-integer-little,
+      "WAVE",
+      "fmt ",
+      16::32-unsigned-integer-little,
+      # 3 indicates 32-bit float PCM
+      3::16-unsigned-integer-little,
+      num_channels::16-unsigned-integer-little,
+      sampling_rate::32-unsigned-integer-little,
+      byte_rate::32-unsigned-integer-little,
+      block_align::16-unsigned-integer-little,
+      bytes_per_sample * 8::16-unsigned-integer-little,
+      "data",
+      data_size::32-unsigned-integer-little
+    >>
+  end
+
+  # We assume a local path and open a raw file for efficiency
+  defp raw_file_range_stream!(path, offset, length) do
+    chunk_size = 64_000
+
+    Stream.resource(
+      fn ->
+        {:ok, fd} = :file.open(path, [:raw, :binary, :read, :read_ahead])
+        {:ok, _} = :file.position(fd, offset)
+        {fd, length}
+      end,
+      fn
+        {fd, 0} ->
+          {:halt, {fd, 0}}
+
+        {fd, length} ->
+          size = min(chunk_size, length)
+          {:ok, chunk} = :file.read(fd, size)
+          {[chunk], {fd, length - size}}
+      end,
+      fn {fd, _} -> :file.close(fd) end
+    )
+  end
+
+  @doc """
+  Builds a single binary with JSON-serialized `meta` and `binary`.
+  """
+  @spec encode_annotated_binary!(term(), binary()) :: binary()
+  def encode_annotated_binary!(meta, binary) do
+    meta = Jason.encode!(meta)
+    meta_size = byte_size(meta)
+    <<meta_size::size(32), meta::binary, binary::binary>>
+  end
+
+  @doc """
+  Decodes binary annotated with JSON-serialized metadata.
+  """
+  @spec decode_annotated_binary!(binary()) :: {term(), binary()}
+  def decode_annotated_binary!(raw) do
+    <<meta_size::size(32), meta::binary-size(meta_size), binary::binary>> = raw
+    meta = Jason.decode!(meta)
+    {meta, binary}
+  end
+end

lib/livebook_web/live/output.ex

@@ -45,17 +45,32 @@ defmodule LivebookWeb.Output do
 
   defp render_output(%{type: :terminal_text, text: text}, %{id: id}) do
     text = if(text == :__pruned__, do: nil, else: text)
-    live_component(Output.TerminalTextComponent, id: id, text: text)
+
+    assigns = %{id: id, text: text}
+
+    ~H"""
+    <.live_component module={Output.TerminalTextComponent} id={@id} text={@text} />
+    """
   end
 
   defp render_output(%{type: :plain_text, text: text}, %{id: id}) do
     text = if(text == :__pruned__, do: nil, else: text)
-    live_component(Output.PlainTextComponent, id: id, text: text)
+
+    assigns = %{id: id, text: text}
+
+    ~H"""
+    <.live_component module={Output.PlainTextComponent} id={@id} text={@text} />
+    """
   end
 
   defp render_output(%{type: :markdown, text: text}, %{id: id, session_id: session_id}) do
     text = if(text == :__pruned__, do: nil, else: text)
-    live_component(Output.MarkdownComponent, id: id, session_id: session_id, text: text)
+
+    assigns = %{id: id, session_id: session_id, text: text}
+
+    ~H"""
+    <.live_component module={Output.MarkdownComponent} id={@id} session_id={@session_id} text={@text} />
+    """
   end
 
   defp render_output(%{type: :image} = output, %{id: id}) do
@@ -71,13 +86,23 @@ defmodule LivebookWeb.Output do
          session_id: session_id,
          client_id: client_id
        }) do
-    live_component(LivebookWeb.JSViewComponent,
+    assigns = %{
       id: id,
       js_view: output.js_view,
       session_id: session_id,
-      client_id: client_id,
+      client_id: client_id
-      timeout_message: "Output data no longer available, please reevaluate this cell"
+    }
-    )
+
+    ~H"""
+    <.live_component
+      module={LivebookWeb.JSViewComponent}
+      id={@id}
+      js_view={@js_view}
+      session_id={@session_id}
+      client_id={@client_id}
+      timeout_message="Output data no longer available, please reevaluate this cell"
+    />
+    """
   end
 
   defp render_output(%{type: :frame} = output, %{
@@ -88,7 +113,7 @@ defmodule LivebookWeb.Output do
          client_id: client_id,
          cell_id: cell_id
        }) do
-    live_component(Output.FrameComponent,
+    assigns = %{
       id: id,
       outputs: output.outputs,
       placeholder: output.placeholder,
@@ -97,7 +122,21 @@ defmodule LivebookWeb.Output do
       input_views: input_views,
       client_id: client_id,
       cell_id: cell_id
-    )
+    }
+
+    ~H"""
+    <.live_component
+      module={Output.FrameComponent}
+      id={@id}
+      outputs={@outputs}
+      placeholder={@placeholder}
+      session_id={@session_id}
+      session_pid={@session_pid}
+      input_views={@input_views}
+      client_id={@client_id}
+      cell_id={@cell_id}
+    />
+    """
   end
 
   defp render_output(%{type: :tabs, outputs: outputs, labels: labels}, %{
@@ -226,13 +265,24 @@ defmodule LivebookWeb.Output do
          session_pid: session_pid,
          client_id: client_id
        }) do
-    live_component(Output.InputComponent,
+    assigns = %{
       id: id,
       input: input,
       input_views: input_views,
       session_pid: session_pid,
       client_id: client_id
-    )
+    }
+
+    ~H"""
+    <.live_component
+      module={Output.InputComponent}
+      id={@id}
+      input={@input}
+      input_views={@input_views}
+      session_pid={@session_pid}
+      client_id={@client_id}
+    />
+    """
   end
 
   defp render_output(%{type: :control} = control, %{
@@ -242,14 +292,26 @@ defmodule LivebookWeb.Output do
          client_id: client_id,
          cell_id: cell_id
        }) do
-    live_component(Output.ControlComponent,
+    assigns = %{
       id: id,
       control: control,
       input_views: input_views,
       session_pid: session_pid,
       client_id: client_id,
       cell_id: cell_id
-    )
+    }
+
+    ~H"""
+    <.live_component
+      module={Output.ControlComponent}
+      id={@id}
+      control={@control}
+      input_views={@input_views}
+      session_pid={@session_pid}
+      client_id={@client_id}
+      cell_id={@cell_id}
+    />
+    """
   end
 
   defp render_output(

lib/livebook_web/live/output/audio_input_component.ex

@@ -3,7 +3,21 @@ defmodule LivebookWeb.Output.AudioInputComponent do
 
   @impl true
   def mount(socket) do
-    {:ok, assign(socket, endianness: System.endianness(), value: nil)}
+    {:ok,
+     socket
+     |> assign(
+       endianness: System.endianness(),
+       value: nil,
+       audio_url: nil,
+       decoding?: false
+     )
+     |> allow_upload(:file,
+       accept: :any,
+       max_entries: 1,
+       max_file_size: 100_000_000_000,
+       progress: &handle_progress/3,
+       auto_upload: true
+     )}
   end
 
   @impl true
@@ -13,30 +27,36 @@ defmodule LivebookWeb.Output.AudioInputComponent do
     socket = assign(socket, assigns)
 
     socket =
-      if value == socket.assigns.value do
+      cond do
+        value == socket.assigns.value ->
           socket
-      else
+
-        audio_info =
+        value == nil ->
-          if value do
+          assign(socket, value: value, audio_url: nil)
-            %{
+
-              data: Base.encode64(value.data),
+        true ->
-              num_channels: value.num_channels,
+          assign(socket, value: value, audio_url: audio_url(socket.assigns.input_id))
-              sampling_rate: value.sampling_rate
-            }
       end
 
-        push_event(socket, "audio_input_change:#{socket.assigns.id}", %{audio_info: audio_info})
+    {:ok, socket}
   end
 
-    {:ok, assign(socket, value: value)}
+  defp audio_url(input_id) do
+    # For the client-side audio preview, we serve audio encoded as WAV
+    # from a separate endpoint. To do that, we encode information in a
+    # token and then the controller fetches input value from the LV.
+    # This is especially important for client-specific inputs in forms.
+    token = LivebookWeb.SessionHelpers.generate_input_token(self(), input_id)
+    ~p"/sessions/audio-input/#{token}"
   end
 
   @impl true
   def render(assigns) do
     ~H"""
+    <div class="inline-flex flex-col gap-4 p-4 border-2 border-dashed border-gray-200 rounded-lg">
       <div
+        class="inline-flex flex-col gap-4"
         id={"#{@id}-root"}
-      class="inline-flex flex-col gap-4 p-4 border-2 border-dashed border-gray-200 rounded-lg"
         phx-hook="AudioInput"
         phx-update="ignore"
         data-id={@id}
@@ -44,8 +64,16 @@ defmodule LivebookWeb.Output.AudioInputComponent do
         data-format={@format}
         data-sampling-rate={@sampling_rate}
         data-endianness={@endianness}
+        data-audio-url={@audio_url}
       >
-      <input type="file" data-input class="hidden" name="html_value" accept="audio/*" capture="user" />
+        <input
+          type="file"
+          data-input
+          class="hidden"
+          name="html_value"
+          accept="audio/*"
+          capture="user"
+        />
         <audio controls data-preview></audio>
         <div class="flex items-center justify-center gap-4">
           <button
@@ -82,15 +110,57 @@ defmodule LivebookWeb.Output.AudioInputComponent do
           </button>
         </div>
       </div>
+      <form phx-change="validate" class="hidden" phx-target={@myself}>
+        <.live_file_input upload={@uploads.file} />
+      </form>
+      <div
+        :if={@uploads.file.entries == [] and @decoding?}
+        class="delay-200 flex justify-center items-center gap-2"
+      >
+        <span class="text-sm font-medium text-gray-500">Decoding</span> <.spinner />
+      </div>
+      <div :for={entry <- @uploads.file.entries} class="delay-200 flex flex-col gap-1">
+        <.file_entry name="Audio" entry={entry} on_clear={JS.push("clear_file", target: @myself)} />
+      </div>
+    </div>
     """
   end
 
   @impl true
-  def handle_event("change", params, socket) do
+  def handle_event("decoding", %{}, socket) do
+    {:noreply, assign(socket, decoding?: true)}
+  end
+
+  def handle_event("validate", %{}, socket) do
+    {:noreply, socket}
+  end
+
+  def handle_event("clear_file", %{"ref" => ref}, socket) do
+    {:noreply, cancel_upload(socket, :file, ref)}
+  end
+
+  defp handle_progress(:file, entry, socket) do
+    if entry.done? do
+      file_ref =
+        consume_uploaded_entry(socket, entry, fn %{path: path} ->
+          {:ok, file_ref} =
+            LivebookWeb.SessionHelpers.register_input_file(
+              socket.assigns.session_pid,
+              path,
+              socket.assigns.input_id,
+              socket.assigns.local,
+              socket.assigns.client_id
+            )
+
+          {:ok, file_ref}
+        end)
+
+      %{"num_channels" => num_channels, "sampling_rate" => sampling_rate} = entry.client_meta
+
       value = %{
-      data: Base.decode64!(params["data"]),
+        file_ref: file_ref,
-      num_channels: params["num_channels"],
+        num_channels: num_channels,
-      sampling_rate: params["sampling_rate"],
+        sampling_rate: sampling_rate,
         format: socket.assigns.format
       }
 
@@ -99,7 +169,8 @@ defmodule LivebookWeb.Output.AudioInputComponent do
         event: :change,
         value: value
       )
+    end
 
-    {:noreply, socket}
+    {:noreply, assign(socket, decoding?: false)}
   end
 end

lib/livebook_web/live/output/file_input_component.ex

@@ -71,24 +71,20 @@ defmodule LivebookWeb.Output.FileInputComponent do
 
   defp handle_progress(:file, entry, socket) do
     if entry.done? do
-      socket
+      {file_ref, client_name} =
-      |> consume_uploaded_entries(:file, fn %{path: path}, entry ->
+        consume_uploaded_entry(socket, entry, fn %{path: path} ->
           {:ok, file_ref} =
-          if socket.assigns.local do
+            LivebookWeb.SessionHelpers.register_input_file(
-            key = "#{socket.assigns.input_id}-#{socket.assigns.client_id}"
+              socket.assigns.session_pid,
-
+              path,
-            Livebook.Session.register_file(socket.assigns.session_pid, path, key,
+              socket.assigns.input_id,
-              linked_client_id: socket.assigns.client_id
+              socket.assigns.local,
+              socket.assigns.client_id
             )
-          else
-            key = "#{socket.assigns.input_id}-global"
-            Livebook.Session.register_file(socket.assigns.session_pid, path, key)
-          end
 
           {:ok, {file_ref, entry.client_name}}
         end)
-      |> case do
+
-        [{file_ref, client_name}] ->
       value = %{file_ref: file_ref, client_name: client_name}
 
       send_update(LivebookWeb.Output.InputComponent,
@@ -96,10 +92,6 @@ defmodule LivebookWeb.Output.FileInputComponent do
         event: :change,
         value: value
       )
-
-        [] ->
-          :ok
-      end
     end
 
     {:noreply, socket}

lib/livebook_web/live/output/image_input_component.ex

@@ -3,7 +3,16 @@ defmodule LivebookWeb.Output.ImageInputComponent do
 
   @impl true
   def mount(socket) do
-    {:ok, assign(socket, value: nil)}
+    {:ok,
+     socket
+     |> assign(value: nil, value: nil, image_url: nil)
+     |> allow_upload(:file,
+       accept: :any,
+       max_entries: 1,
+       max_file_size: 100_000_000_000,
+       progress: &handle_progress/3,
+       auto_upload: true
+     )}
   end
 
   @impl true
@@ -13,26 +22,37 @@ defmodule LivebookWeb.Output.ImageInputComponent do
     socket = assign(socket, assigns)
 
     socket =
-      if value == socket.assigns.value do
+      cond do
+        value == socket.assigns.value ->
           socket
-      else
+
-        image_info =
+        value == nil ->
-          if value do
+          assign(socket, value: value, image_url: nil)
-            %{data: Base.encode64(value.data), height: value.height, width: value.width}
+
+        true ->
+          assign(socket, value: value, image_url: image_url(socket.assigns.input_id))
       end
 
-        push_event(socket, "image_input_change:#{socket.assigns.id}", %{image_info: image_info})
+    {:ok, socket}
   end
 
-    {:ok, assign(socket, value: value)}
+  defp image_url(input_id) do
+    # For the client-side image preview, we serve the original binary
+    # value from a separate endpoint. To do that, we encode information
+    # in a token and then the controller fetches input value from the
+    # LV. This is especially important for client-specific inputs in
+    # forms.
+    token = LivebookWeb.SessionHelpers.generate_input_token(self(), input_id)
+    ~p"/sessions/image-input/#{token}"
   end
 
   @impl true
   def render(assigns) do
     ~H"""
+    <div class="inline-flex flex-col gap-4 p-4 border-2 border-dashed border-gray-200 rounded-lg">
       <div
         id={"#{@id}-root"}
-      class="inline-flex flex-col p-4 border-2 border-dashed border-gray-200 rounded-lg"
+        class="inline-flex flex-col"
         phx-hook="ImageInput"
         phx-update="ignore"
         data-id={@id}
@@ -41,8 +61,18 @@ defmodule LivebookWeb.Output.ImageInputComponent do
         data-width={@width}
         data-format={@format}
         data-fit={@fit}
+        data-image-url={@image_url}
+        data-value-height={@value[:height]}
+        data-value-width={@value[:width]}
       >
-      <input type="file" data-input class="hidden" name="html_value" accept="image/*" capture="user" />
+        <input
+          type="file"
+          data-input
+          class="hidden"
+          name="html_value"
+          accept="image/*"
+          capture="user"
+        />
         <div class="flex justify-center" data-preview>
           <div class="flex justify-center text-gray-500">
             Drag an image file
@@ -91,15 +121,47 @@ defmodule LivebookWeb.Output.ImageInputComponent do
           </button>
         </div>
       </div>
+      <form phx-change="validate" class="hidden" phx-target={@myself}>
+        <.live_file_input upload={@uploads.file} />
+      </form>
+      <div :for={entry <- @uploads.file.entries} class="delay-200 flex flex-col gap-1">
+        <.file_entry name="Audio" entry={entry} on_clear={JS.push("clear_file", target: @myself)} />
+      </div>
+    </div>
     """
   end
 
   @impl true
-  def handle_event("change", params, socket) do
+  def handle_event("validate", %{}, socket) do
+    {:noreply, socket}
+  end
+
+  def handle_event("clear_file", %{"ref" => ref}, socket) do
+    {:noreply, cancel_upload(socket, :file, ref)}
+  end
+
+  defp handle_progress(:file, entry, socket) do
+    if entry.done? do
+      file_ref =
+        consume_uploaded_entry(socket, entry, fn %{path: path} ->
+          {:ok, file_ref} =
+            LivebookWeb.SessionHelpers.register_input_file(
+              socket.assigns.session_pid,
+              path,
+              socket.assigns.input_id,
+              socket.assigns.local,
+              socket.assigns.client_id
+            )
+
+          {:ok, file_ref}
+        end)
+
+      %{"height" => height, "width" => width} = entry.client_meta
+
       value = %{
-      data: Base.decode64!(params["data"]),
+        file_ref: file_ref,
-      height: params["height"],
+        height: height,
-      width: params["width"],
+        width: width,
         format: socket.assigns.format
       }
 
@@ -108,6 +170,7 @@ defmodule LivebookWeb.Output.ImageInputComponent do
         event: :change,
         value: value
       )
+    end
 
     {:noreply, socket}
   end

lib/livebook_web/live/output/input_component.ex

@@ -36,6 +36,10 @@ defmodule LivebookWeb.Output.InputComponent do
         width={@input.attrs.size && elem(@input.attrs.size, 1)}
         format={@input.attrs.format}
         fit={@input.attrs.fit}
+        input_id={@input.id}
+        session_pid={@session_pid}
+        client_id={@client_id}
+        local={@local}
       />
     </div>
     """
@@ -52,6 +56,10 @@ defmodule LivebookWeb.Output.InputComponent do
         value={@value}
         format={@input.attrs.format}
         sampling_rate={@input.attrs.sampling_rate}
+        input_id={@input.id}
+        session_pid={@session_pid}
+        client_id={@client_id}
+        local={@local}
       />
     </div>
     """

lib/livebook_web/live/session_helpers.ex

@@ -272,4 +272,42 @@ defmodule LivebookWeb.SessionHelpers do
         end
     end
   end
+
+  @doc """
+  Generates a token for the given input.
+  """
+  @spec generate_input_token(pid(), String.t()) :: String.t()
+  def generate_input_token(live_view_pid, input_id) do
+    Phoenix.Token.sign(LivebookWeb.Endpoint, "session-input", %{
+      live_view_pid: live_view_pid,
+      input_id: input_id
+    })
+  end
+
+  @doc """
+  Verifies token from `generate_input_token/2` and extracts the encoded
+  data.
+  """
+  @spec verify_input_token!(String.t()) :: {pid(), String.t()}
+  def verify_input_token!(token) do
+    {:ok, %{live_view_pid: live_view_pid, input_id: input_id}} =
+      Phoenix.Token.verify(LivebookWeb.Endpoint, "session-input", token)
+
+    {live_view_pid, input_id}
+  end
+
+  @doc """
+  Registers an uploaded input file in session.
+  """
+  @spec register_input_file(pid(), String.t(), String.t(), boolean(), String.t()) ::
+          {:ok, Livebook.Runtime.file_ref()}
+  def register_input_file(session_pid, path, input_id, local, client_id) do
+    if local do
+      key = "#{input_id}-#{client_id}"
+      Livebook.Session.register_file(session_pid, path, key, linked_client_id: client_id)
+    else
+      key = "#{input_id}-global"
+      Livebook.Session.register_file(session_pid, path, key)
+    end
+  end
 end

lib/livebook_web/live/session_live.ex

@@ -1730,6 +1730,17 @@ defmodule LivebookWeb.SessionLive do
     {:noreply, socket}
   end
 
+  @impl true
+  def handle_call({:get_input_value, input_id}, _from, socket) do
+    reply =
+      case socket.private.data.input_infos do
+        %{^input_id => %{value: value}} -> {:ok, socket.assigns.session.id, value}
+        %{} -> :error
+      end
+
+    {:reply, reply, socket}
+  end
+
   @impl true
   def handle_info({:operation, operation}, socket) do
     {:noreply, handle_operation(socket, operation)}

lib/livebook_web/router.ex

@@ -100,6 +100,8 @@ defmodule LivebookWeb.Router do
       get "/sessions/:id/files/:name", SessionController, :show_file
       get "/sessions/:id/images/:name", SessionController, :show_image
       get "/sessions/:id/download/files/:name", SessionController, :download_file
+      get "/sessions/audio-input/:token", SessionController, :show_input_audio
+      get "/sessions/image-input/:token", SessionController, :show_input_image
       live "/sessions/:id/settings/custom-view", SessionLive, :custom_view_settings
       live "/sessions/:id/*path_parts", SessionLive, :catch_all
     end

mix.exs

@@ -94,9 +94,9 @@ defmodule Livebook.MixProject do
   #
   defp deps do
     [
-      {:phoenix, "~> 1.7.0"},
+      {:phoenix, "~> 1.7.7"},
       {:phoenix_html, "~> 3.0"},
-      # {:phoenix_live_view, "~> 0.19.0"},
+      # {:phoenix_live_view, "~> 0.20.0"},
       {:phoenix_live_view, github: "phoenixframework/phoenix_live_view", override: true},
       {:phoenix_live_dashboard, "~> 0.8.0"},
       {:telemetry_metrics, "~> 0.4"},

mix.lock

@@ -25,12 +25,12 @@
   "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"},
   "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
-  "phoenix": {:hex, :phoenix, "1.7.5", "3234bc87185e6a2103a15a3b1399f19775b093a6923c4064436e49cdab8ce5d2", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.1", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "5abad1789f06a3572ee5e5d5151993ed35b9e2711537904cc457a40229587979"},
+  "phoenix": {:hex, :phoenix, "1.7.7", "4cc501d4d823015007ba3cdd9c41ecaaf2ffb619d6fb283199fa8ddba89191e0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "8966e15c395e5e37591b6ed0bd2ae7f48e961f0f60ac4c733f9566b519453085"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.2", "b21bd01fdeffcfe2fab49e4942aa938b6d3e89e93a480d4aee58085560a0bc0d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "70242edd4601d50b69273b057ecf7b684644c19ee750989fd555625ae4ce8f5d"},
-  "phoenix_html": {:hex, :phoenix_html, "3.3.1", "4788757e804a30baac6b3fc9695bf5562465dd3f1da8eb8460ad5b404d9a2178", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "bed1906edd4906a15fd7b412b85b05e521e1f67c9a85418c55999277e553d0d3"},
+  "phoenix_html": {:hex, :phoenix_html, "3.3.2", "d6ce982c6d8247d2fc0defe625255c721fb8d5f1942c5ac051f6177bffa5973f", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "44adaf8e667c1c20fb9d284b6b0fa8dc7946ce29e81ce621860aa7e96de9a11d"},
-  "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.0", "0b3158b5b198aa444473c91d23d79f52fb077e807ffad80dacf88ce078fa8df2", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "87785a54474fed91a67a1227a741097eb1a42c2e49d3c0d098b588af65cd410d"},
+  "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.2", "b9e33c950d1ed98494bfbde1c34c6e51c8a4214f3bea3f07ca9a510643ee1387", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "67a598441b5f583d301a77e0298719f9654887d3d8bf14e80ff0b6acf887ef90"},
   "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"},
-  "phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view.git", "64e22999c2900e2f9266a030ca7a135a042f0645", []},
+  "phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view.git", "9b267285cfce96072e9ecd8fbe065dc64a7ad004", []},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
   "phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"},
   "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
@@ -44,5 +44,5 @@
   "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"},
   "thousand_island": {:hex, :thousand_island, "0.6.7", "3a91a7e362ca407036c6691e8a4f6e01ac8e901db3598875863a149279ac8571", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "541a5cb26b88adf8d8180b6b96a90f09566b4aad7a6b3608dcac969648cf6765"},
   "websock": {:hex, :websock, "0.5.1", "c496036ce95bc26d08ba086b2a827b212c67e7cabaa1c06473cd26b40ed8cf10", [:mix], [], "hexpm", "b9f785108b81cd457b06e5f5dabe5f65453d86a99118b2c0a515e1e296dc2d2c"},
-  "websock_adapter": {:hex, :websock_adapter, "0.5.1", "292e6c56724e3457e808e525af0e9bcfa088cc7b9c798218e78658c7f9b85066", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "8e2e1544bfde5f9d0442f9cec2f5235398b224f75c9e06b60557debf64248ec1"},
+  "websock_adapter": {:hex, :websock_adapter, "0.5.4", "7af8408e7ed9d56578539594d1ee7d8461e2dd5c3f57b0f2a5352d610ddde757", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d2c238c79c52cbe223fcdae22ca0bb5007a735b9e933870e241fce66afb4f4ab"},
 }

test/livebook_web/controllers/session_controller_test.exs

@@ -1,6 +1,8 @@
 defmodule LivebookWeb.SessionControllerTest do
   use LivebookWeb.ConnCase, async: true
 
+  require Phoenix.LiveViewTest
+
   alias Livebook.{Sessions, Session, Notebook, FileSystem}
 
   describe "show_file" do
@@ -356,6 +358,100 @@ defmodule LivebookWeb.SessionControllerTest do
     end
   end
 
+  describe "show_audio_image" do
+    @tag :tmp_dir
+    test "given :wav input returns the audio binary", %{conn: conn, tmp_dir: tmp_dir} do
+      {session, input_id} = start_session_with_audio_input(:wav, "wav content", tmp_dir)
+
+      {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
+
+      token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
+
+      conn = get(conn, ~p"/sessions/audio-input/#{token}")
+
+      assert conn.status == 200
+      assert conn.resp_body == "wav content"
+      assert get_resp_header(conn, "accept-ranges") == ["bytes"]
+
+      Session.close(session.pid)
+    end
+
+    @tag :tmp_dir
+    test "given :wav input supports range requests", %{conn: conn, tmp_dir: tmp_dir} do
+      {session, input_id} = start_session_with_audio_input(:wav, "wav content", tmp_dir)
+
+      {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
+
+      token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=4-")
+        |> get(~p"/sessions/audio-input/#{token}")
+
+      assert conn.status == 206
+      assert conn.resp_body == "content"
+      assert get_resp_header(conn, "content-range") == ["bytes 4-10/11"]
+
+      Session.close(session.pid)
+    end
+
+    @tag :tmp_dir
+    test "given :pcm_f32 input returns a WAV binary", %{conn: conn, tmp_dir: tmp_dir} do
+      {session, input_id} = start_session_with_audio_input(:pcm_f32, "pcm content", tmp_dir)
+
+      {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
+
+      token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
+
+      conn = get(conn, ~p"/sessions/audio-input/#{token}")
+
+      assert conn.status == 200
+      assert <<_header::44-binary, "pcm content">> = conn.resp_body
+      assert get_resp_header(conn, "accept-ranges") == ["bytes"]
+
+      Session.close(session.pid)
+    end
+
+    @tag :tmp_dir
+    test "given :pcm_f32 input supports range requests", %{conn: conn, tmp_dir: tmp_dir} do
+      {session, input_id} = start_session_with_audio_input(:pcm_f32, "pcm content", tmp_dir)
+
+      {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
+
+      token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
+
+      conn =
+        conn
+        |> put_req_header("range", "bytes=48-")
+        |> get(~p"/sessions/audio-input/#{token}")
+
+      assert conn.status == 206
+      assert conn.resp_body == "content"
+      assert get_resp_header(conn, "content-range") == ["bytes 48-54/55"]
+
+      Session.close(session.pid)
+    end
+  end
+
+  describe "show_input_image" do
+    @tag :tmp_dir
+    test "returns the image binary", %{conn: conn, tmp_dir: tmp_dir} do
+      {session, input_id} = start_session_with_image_input(:rgb, "rgb content", tmp_dir)
+
+      {:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
+
+      token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
+
+      conn = get(conn, ~p"/sessions/image-input/#{token}")
+
+      assert conn.status == 200
+      assert conn.resp_body == "rgb content"
+
+      Session.close(session.pid)
+    end
+  end
+
   defp start_session_and_request_asset(conn, notebook, hash) do
     {:ok, session} = Sessions.create_session(notebook: notebook)
     # We need runtime in place to actually copy the archive
@@ -387,4 +483,62 @@ defmodule LivebookWeb.SessionControllerTest do
 
     %{notebook: notebook, hash: hash}
   end
+
+  defp start_session_with_audio_input(format, binary, tmp_dir) do
+    input = %{
+      type: :input,
+      ref: "ref",
+      id: "input1",
+      destination: :noop,
+      attrs: %{type: :audio, default: nil, label: "Audio", format: format, sampling_rate: 16_000}
+    }
+
+    cell = %{Notebook.Cell.new(:code) | outputs: [{1, input}]}
+    notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [cell]}]}
+
+    {:ok, session} = Sessions.create_session(notebook: notebook)
+
+    source_path = Path.join(tmp_dir, "audio.bin")
+    File.write!(source_path, binary)
+
+    {:ok, file_ref} = Session.register_file(session.pid, source_path, "key")
+
+    Session.set_input_value(session.pid, "input1", %{
+      file_ref: file_ref,
+      sampling_rate: 16_000,
+      num_channels: 1,
+      format: format
+    })
+
+    {session, input.id}
+  end
+
+  defp start_session_with_image_input(format, binary, tmp_dir) do
+    input = %{
+      type: :input,
+      ref: "ref",
+      id: "input1",
+      destination: :noop,
+      attrs: %{type: :image, default: nil, label: "Image", format: :rgb, size: nil, fit: :contain}
+    }
+
+    cell = %{Notebook.Cell.new(:code) | outputs: [{1, input}]}
+    notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [cell]}]}
+
+    {:ok, session} = Sessions.create_session(notebook: notebook)
+
+    source_path = Path.join(tmp_dir, "image.bin")
+    File.write!(source_path, binary)
+
+    {:ok, file_ref} = Session.register_file(session.pid, source_path, "key")
+
+    Session.set_input_value(session.pid, "input1", %{
+      file_ref: file_ref,
+      height: 300,
+      width: 300,
+      format: format
+    })
+
+    {session, input.id}
+  end
 end