mirror of
https://github.com/livebook-dev/livebook.git
synced 2025-10-06 11:35:54 +08:00
Stream audio/image input values into and out of the server (#2249)
Co-authored-by: José Valim <jose.valim@dashbit.co>
This commit is contained in:
Jonatan Kłosko
GitHub
parent
4d412bd00d
commit
64a150eef2
22 changed files with 1145 additions and 471 deletions
|
|
@ -1,5 +1,9 @@
|
|||
import { getAttributeOrThrow, parseInteger } from "../lib/attribute";
|
||||
import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
|
||||
import {
|
||||
getAttributeOrDefault,
|
||||
getAttributeOrThrow,
|
||||
parseInteger,
|
||||
} from "../lib/attribute";
|
||||
import { encodeAnnotatedBuffer, encodePcmAsWav } from "../lib/codec";
|
||||
|
||||
const dropClasses = ["bg-yellow-100", "border-yellow-300"];
|
||||
|
||||
|
|
@ -18,6 +22,8 @@ const dropClasses = ["bg-yellow-100", "border-yellow-300"];
|
|||
*
|
||||
* * `data-endianness` - the server endianness, either `"little"` or `"big"`
|
||||
*
|
||||
* * `data-audio-url` - the URL to audio file to use for the current preview
|
||||
*
|
||||
*/
|
||||
const AudioInput = {
|
||||
mounted() {
|
||||
|
|
@ -32,21 +38,8 @@ const AudioInput = {
|
|||
|
||||
this.mediaRecorder = null;
|
||||
|
||||
// Render updated value
|
||||
this.handleEvent(
|
||||
`audio_input_change:${this.props.id}`,
|
||||
({ audio_info: audioInfo }) => {
|
||||
if (audioInfo) {
|
||||
this.updatePreview({
|
||||
data: this.decodeAudio(base64ToBuffer(audioInfo.data)),
|
||||
numChannels: audioInfo.num_channels,
|
||||
samplingRate: audioInfo.sampling_rate,
|
||||
});
|
||||
} else {
|
||||
this.clearPreview();
|
||||
}
|
||||
}
|
||||
);
|
||||
// Set the current value URL
|
||||
this.audioEl.src = this.props.audioUrl;
|
||||
|
||||
// File selection
|
||||
|
||||
|
|
@ -105,6 +98,8 @@ const AudioInput = {
|
|||
|
||||
updated() {
|
||||
this.props = this.getProps();
|
||||
|
||||
this.audioEl.src = this.props.audioUrl;
|
||||
},
|
||||
|
||||
getProps() {
|
||||
|
|
@ -118,6 +113,7 @@ const AudioInput = {
|
|||
),
|
||||
endianness: getAttributeOrThrow(this.el, "data-endianness"),
|
||||
format: getAttributeOrThrow(this.el, "data-format"),
|
||||
audioUrl: getAttributeOrDefault(this.el, "data-audio-url", null),
|
||||
};
|
||||
},
|
||||
|
||||
|
|
@ -176,6 +172,8 @@ const AudioInput = {
|
|||
},
|
||||
|
||||
loadEncodedAudio(buffer) {
|
||||
this.pushEventTo(this.props.phxTarget, "decoding", {});
|
||||
|
||||
const context = new AudioContext({ sampleRate: this.props.samplingRate });
|
||||
|
||||
context.decodeAudioData(buffer, (audioBuffer) => {
|
||||
|
|
@ -184,66 +182,31 @@ const AudioInput = {
|
|||
});
|
||||
},
|
||||
|
||||
updatePreview(audioInfo) {
|
||||
const oldUrl = this.audioEl.src;
|
||||
const blob = audioInfoToWavBlob(audioInfo);
|
||||
this.audioEl.src = URL.createObjectURL(blob);
|
||||
oldUrl && URL.revokeObjectURL(oldUrl);
|
||||
},
|
||||
|
||||
clearPreview() {
|
||||
const oldUrl = this.audioEl.src;
|
||||
this.audioEl.src = "";
|
||||
oldUrl && URL.revokeObjectURL(oldUrl);
|
||||
},
|
||||
|
||||
pushAudio(audioInfo) {
|
||||
this.pushEventTo(this.props.phxTarget, "change", {
|
||||
data: bufferToBase64(this.encodeAudio(audioInfo)),
|
||||
const meta = {
|
||||
num_channels: audioInfo.numChannels,
|
||||
sampling_rate: audioInfo.samplingRate,
|
||||
});
|
||||
};
|
||||
|
||||
const buffer = this.encodeAudio(audioInfo);
|
||||
|
||||
const blob = new Blob([buffer]);
|
||||
blob.meta = () => meta;
|
||||
|
||||
this.uploadTo(this.props.phxTarget, "file", [blob]);
|
||||
},
|
||||
|
||||
encodeAudio(audioInfo) {
|
||||
if (this.props.format === "pcm_f32") {
|
||||
return this.fixEndianness32(audioInfo.data);
|
||||
return convertEndianness32(audioInfo.data, this.props.endianness);
|
||||
} else if (this.props.format === "wav") {
|
||||
return encodeWavData(
|
||||
return encodePcmAsWav(
|
||||
audioInfo.data,
|
||||
audioInfo.numChannels,
|
||||
audioInfo.samplingRate
|
||||
);
|
||||
}
|
||||
},
|
||||
|
||||
decodeAudio(buffer) {
|
||||
if (this.props.format === "pcm_f32") {
|
||||
return this.fixEndianness32(buffer);
|
||||
} else if (this.props.format === "wav") {
|
||||
return decodeWavData(buffer);
|
||||
}
|
||||
},
|
||||
|
||||
fixEndianness32(buffer) {
|
||||
if (getEndianness() === this.props.endianness) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
// If the server uses different endianness, we swap bytes accordingly
|
||||
for (let i = 0; i < buffer.byteLength / 4; i++) {
|
||||
const b1 = buffer[i];
|
||||
const b2 = buffer[i + 1];
|
||||
const b3 = buffer[i + 2];
|
||||
const b4 = buffer[i + 3];
|
||||
buffer[i] = b4;
|
||||
buffer[i + 1] = b3;
|
||||
buffer[i + 2] = b2;
|
||||
buffer[i + 3] = b1;
|
||||
}
|
||||
|
||||
return buffer;
|
||||
},
|
||||
};
|
||||
|
||||
function audioBufferToAudioInfo(audioBuffer) {
|
||||
|
|
@ -267,88 +230,24 @@ function audioBufferToAudioInfo(audioBuffer) {
|
|||
return { data: pcmArray.buffer, numChannels, samplingRate };
|
||||
}
|
||||
|
||||
function audioInfoToWavBlob({ data, numChannels, samplingRate }) {
|
||||
const wavBytes = encodeWavData(data, numChannels, samplingRate);
|
||||
return new Blob([wavBytes], { type: "audio/wav" });
|
||||
}
|
||||
|
||||
// See http://soundfile.sapp.org/doc/WaveFormat
|
||||
function encodeWavData(buffer, numChannels, samplingRate) {
|
||||
const HEADER_SIZE = 44;
|
||||
|
||||
const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength);
|
||||
const view = new DataView(wavBuffer);
|
||||
|
||||
const numFrames = buffer.byteLength / 4;
|
||||
const bytesPerSample = 4;
|
||||
|
||||
const blockAlign = numChannels * bytesPerSample;
|
||||
const byteRate = samplingRate * blockAlign;
|
||||
const dataSize = numFrames * blockAlign;
|
||||
|
||||
let offset = 0;
|
||||
|
||||
function writeUint32Big(int) {
|
||||
view.setUint32(offset, int, false);
|
||||
offset += 4;
|
||||
function convertEndianness32(buffer, targetEndianness) {
|
||||
if (getEndianness() === targetEndianness) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
function writeUint32(int) {
|
||||
view.setUint32(offset, int, true);
|
||||
offset += 4;
|
||||
// If the server uses different endianness, we swap bytes accordingly
|
||||
for (let i = 0; i < buffer.byteLength / 4; i++) {
|
||||
const b1 = buffer[i];
|
||||
const b2 = buffer[i + 1];
|
||||
const b3 = buffer[i + 2];
|
||||
const b4 = buffer[i + 3];
|
||||
buffer[i] = b4;
|
||||
buffer[i + 1] = b3;
|
||||
buffer[i + 2] = b2;
|
||||
buffer[i + 3] = b1;
|
||||
}
|
||||
|
||||
function writeUint16(int) {
|
||||
view.setUint16(offset, int, true);
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
function writeFloat32(int) {
|
||||
view.setFloat32(offset, int, true);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
writeUint32Big(0x52494646);
|
||||
writeUint32(36 + dataSize);
|
||||
writeUint32Big(0x57415645);
|
||||
|
||||
writeUint32Big(0x666d7420);
|
||||
writeUint32(16);
|
||||
writeUint16(3); // 3 represents 32-bit float PCM
|
||||
writeUint16(numChannels);
|
||||
writeUint32(samplingRate);
|
||||
writeUint32(byteRate);
|
||||
writeUint16(blockAlign);
|
||||
writeUint16(bytesPerSample * 8);
|
||||
|
||||
writeUint32Big(0x64617461);
|
||||
writeUint32(dataSize);
|
||||
|
||||
const array = new Float32Array(buffer);
|
||||
|
||||
for (let i = 0; i < array.length; i++) {
|
||||
writeFloat32(array[i]);
|
||||
}
|
||||
|
||||
return wavBuffer;
|
||||
}
|
||||
|
||||
// We assume the exact same format as above, since we only need to
|
||||
// decode data we encoded previously
|
||||
function decodeWavData(buffer) {
|
||||
const HEADER_SIZE = 44;
|
||||
|
||||
const pcmBuffer = new ArrayBuffer(buffer.byteLength - HEADER_SIZE);
|
||||
const pcmArray = new Float32Array(pcmBuffer);
|
||||
|
||||
const view = new DataView(buffer);
|
||||
|
||||
for (let i = 0; i < pcmArray.length; i++) {
|
||||
const offset = HEADER_SIZE + i * 4;
|
||||
pcmArray[i] = view.getFloat32(offset, true);
|
||||
}
|
||||
|
||||
return pcmBuffer;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
function getEndianness() {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import {
|
|||
getAttributeOrThrow,
|
||||
parseInteger,
|
||||
} from "../lib/attribute";
|
||||
import { base64ToBuffer, bufferToBase64 } from "../lib/utils";
|
||||
import { encodeAnnotatedBuffer } from "../lib/codec";
|
||||
|
||||
const dropClasses = ["bg-yellow-100", "border-yellow-300"];
|
||||
|
||||
|
|
@ -24,6 +24,12 @@ const dropClasses = ["bg-yellow-100", "border-yellow-300"];
|
|||
*
|
||||
* * `data-fit` - the fit strategy
|
||||
*
|
||||
* * `data-image-url` - the URL to the image binary value
|
||||
*
|
||||
* * `data-value-height` - the height of the current image value
|
||||
*
|
||||
* * `data-value-width` - the width fo the current image value
|
||||
*
|
||||
*/
|
||||
const ImageInput = {
|
||||
mounted() {
|
||||
|
|
@ -50,18 +56,7 @@ const ImageInput = {
|
|||
this.cameraVideoEl = null;
|
||||
this.cameraStream = null;
|
||||
|
||||
// Render updated value
|
||||
this.handleEvent(
|
||||
`image_input_change:${this.props.id}`,
|
||||
({ image_info: imageInfo }) => {
|
||||
if (imageInfo) {
|
||||
const canvas = imageInfoToElement(imageInfo, this.props.format);
|
||||
this.setPreview(canvas);
|
||||
} else {
|
||||
this.setPreview(this.initialPreviewContentEl);
|
||||
}
|
||||
}
|
||||
);
|
||||
this.updateImagePreview();
|
||||
|
||||
// File selection
|
||||
|
||||
|
|
@ -139,6 +134,8 @@ const ImageInput = {
|
|||
|
||||
updated() {
|
||||
this.props = this.getProps();
|
||||
|
||||
this.updateImagePreview();
|
||||
},
|
||||
|
||||
getProps() {
|
||||
|
|
@ -149,9 +146,37 @@ const ImageInput = {
|
|||
width: getAttributeOrDefault(this.el, "data-width", null, parseInteger),
|
||||
format: getAttributeOrThrow(this.el, "data-format"),
|
||||
fit: getAttributeOrThrow(this.el, "data-fit"),
|
||||
imageUrl: getAttributeOrDefault(this.el, "data-image-url", null),
|
||||
valueHeight: getAttributeOrDefault(
|
||||
this.el,
|
||||
"data-value-height",
|
||||
null,
|
||||
parseInteger
|
||||
),
|
||||
valueWidth: getAttributeOrDefault(
|
||||
this.el,
|
||||
"data-value-width",
|
||||
null,
|
||||
parseInteger
|
||||
),
|
||||
};
|
||||
},
|
||||
|
||||
updateImagePreview() {
|
||||
if (this.props.imageUrl) {
|
||||
buildPreviewElement(
|
||||
this.props.imageUrl,
|
||||
this.props.valueHeight,
|
||||
this.props.valueWidth,
|
||||
this.props.format
|
||||
).then((element) => {
|
||||
this.setPreview(element);
|
||||
});
|
||||
} else {
|
||||
this.setPreview(this.initialPreviewContentEl);
|
||||
}
|
||||
},
|
||||
|
||||
loadFile(file) {
|
||||
const reader = new FileReader();
|
||||
|
||||
|
|
@ -272,10 +297,16 @@ const ImageInput = {
|
|||
},
|
||||
|
||||
pushImage(canvas) {
|
||||
this.pushEventTo(this.props.phxTarget, "change", {
|
||||
data: canvasToBase64(canvas, this.props.format),
|
||||
height: canvas.height,
|
||||
width: canvas.width,
|
||||
canvasToBuffer(canvas, this.props.format).then((buffer) => {
|
||||
const meta = {
|
||||
height: canvas.height,
|
||||
width: canvas.width,
|
||||
};
|
||||
|
||||
const blob = new Blob([buffer]);
|
||||
blob.meta = () => meta;
|
||||
|
||||
this.uploadTo(this.props.phxTarget, "file", [blob]);
|
||||
});
|
||||
},
|
||||
|
||||
|
|
@ -397,11 +428,15 @@ const ImageInput = {
|
|||
},
|
||||
};
|
||||
|
||||
function canvasToBase64(canvas, format) {
|
||||
function canvasToBuffer(canvas, format) {
|
||||
if (format === "png" || format === "jpeg") {
|
||||
const prefix = `data:image/${format};base64,`;
|
||||
const dataUrl = canvas.toDataURL(`image/${format}`);
|
||||
return dataUrl.slice(prefix.length);
|
||||
return new Promise((resolve, reject) => {
|
||||
canvas.toBlob((blob) => {
|
||||
blob.arrayBuffer().then((buffer) => {
|
||||
resolve(buffer);
|
||||
});
|
||||
}, `image/${format}`);
|
||||
});
|
||||
}
|
||||
|
||||
if (format === "rgb") {
|
||||
|
|
@ -410,7 +445,7 @@ function canvasToBase64(canvas, format) {
|
|||
.getImageData(0, 0, canvas.width, canvas.height);
|
||||
|
||||
const buffer = imageDataToRGBBuffer(imageData);
|
||||
return bufferToBase64(buffer);
|
||||
return Promise.resolve(buffer);
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected format: ${format}`);
|
||||
|
|
@ -429,26 +464,24 @@ function imageDataToRGBBuffer(imageData) {
|
|||
return bytes.buffer;
|
||||
}
|
||||
|
||||
function imageInfoToElement(imageInfo, format) {
|
||||
function buildPreviewElement(imageUrl, height, width, format) {
|
||||
if (format === "png" || format === "jpeg") {
|
||||
const src = `data:image/${format};base64,${imageInfo.data}`;
|
||||
const img = document.createElement("img");
|
||||
img.src = src;
|
||||
return img;
|
||||
img.src = imageUrl;
|
||||
return Promise.resolve(img);
|
||||
}
|
||||
|
||||
if (format === "rgb") {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.height = imageInfo.height;
|
||||
canvas.width = imageInfo.width;
|
||||
const buffer = base64ToBuffer(imageInfo.data);
|
||||
const imageData = imageDataFromRGBBuffer(
|
||||
buffer,
|
||||
imageInfo.width,
|
||||
imageInfo.height
|
||||
);
|
||||
canvas.getContext("2d").putImageData(imageData, 0, 0);
|
||||
return canvas;
|
||||
return fetch(imageUrl)
|
||||
.then((response) => response.arrayBuffer())
|
||||
.then((buffer) => {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.height = height;
|
||||
canvas.width = width;
|
||||
const imageData = imageDataFromRGBBuffer(buffer, width, height);
|
||||
canvas.getContext("2d").putImageData(imageData, 0, 0);
|
||||
return canvas;
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error(`Unexpected format: ${format}`);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { Socket } from "phoenix";
|
||||
import { decodeAnnotatedBuffer, encodeAnnotatedBuffer } from "../../lib/codec";
|
||||
|
||||
const csrfToken = document
|
||||
.querySelector("meta[name='csrf-token']")
|
||||
|
|
@ -43,7 +44,7 @@ export function transportEncode(meta, payload) {
|
|||
payload[1].constructor === ArrayBuffer
|
||||
) {
|
||||
const [info, buffer] = payload;
|
||||
return encode([meta, info], buffer);
|
||||
return encodeAnnotatedBuffer([meta, info], buffer);
|
||||
} else {
|
||||
return { root: [meta, payload] };
|
||||
}
|
||||
|
|
@ -51,7 +52,7 @@ export function transportEncode(meta, payload) {
|
|||
|
||||
export function transportDecode(raw) {
|
||||
if (raw.constructor === ArrayBuffer) {
|
||||
const [[meta, info], buffer] = decode(raw);
|
||||
const [[meta, info], buffer] = decodeAnnotatedBuffer(raw);
|
||||
return [meta, [info, buffer]];
|
||||
} else {
|
||||
const {
|
||||
|
|
@ -60,36 +61,3 @@ export function transportDecode(raw) {
|
|||
return [meta, payload];
|
||||
}
|
||||
}
|
||||
|
||||
const HEADER_LENGTH = 4;
|
||||
|
||||
function encode(meta, buffer) {
|
||||
const encoder = new TextEncoder();
|
||||
const metaArray = encoder.encode(JSON.stringify(meta));
|
||||
|
||||
const raw = new ArrayBuffer(
|
||||
HEADER_LENGTH + metaArray.byteLength + buffer.byteLength
|
||||
);
|
||||
const view = new DataView(raw);
|
||||
|
||||
view.setUint32(0, metaArray.byteLength);
|
||||
new Uint8Array(raw, HEADER_LENGTH, metaArray.byteLength).set(metaArray);
|
||||
new Uint8Array(raw, HEADER_LENGTH + metaArray.byteLength).set(
|
||||
new Uint8Array(buffer)
|
||||
);
|
||||
|
||||
return raw;
|
||||
}
|
||||
|
||||
function decode(raw) {
|
||||
const view = new DataView(raw);
|
||||
const metaArrayLength = view.getUint32(0);
|
||||
|
||||
const metaArray = new Uint8Array(raw, HEADER_LENGTH, metaArrayLength);
|
||||
const buffer = raw.slice(HEADER_LENGTH + metaArrayLength);
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
const meta = JSON.parse(decoder.decode(metaArray));
|
||||
|
||||
return [meta, buffer];
|
||||
}
|
||||
|
|
|
|||
103
assets/js/lib/codec.js
Normal file
103
assets/js/lib/codec.js
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* Encodes PCM float-32 in native endianness into a WAV binary.
|
||||
*/
|
||||
export function encodePcmAsWav(buffer, numChannels, samplingRate) {
|
||||
// See http://soundfile.sapp.org/doc/WaveFormat
|
||||
|
||||
const HEADER_SIZE = 44;
|
||||
|
||||
const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength);
|
||||
const view = new DataView(wavBuffer);
|
||||
|
||||
const numFrames = buffer.byteLength / 4;
|
||||
const bytesPerSample = 4;
|
||||
|
||||
const blockAlign = numChannels * bytesPerSample;
|
||||
const byteRate = samplingRate * blockAlign;
|
||||
const dataSize = numFrames * blockAlign;
|
||||
|
||||
let offset = 0;
|
||||
|
||||
function writeUint32Big(int) {
|
||||
view.setUint32(offset, int, false);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
function writeUint32(int) {
|
||||
view.setUint32(offset, int, true);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
function writeUint16(int) {
|
||||
view.setUint16(offset, int, true);
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
function writeFloat32(int) {
|
||||
view.setFloat32(offset, int, true);
|
||||
offset += 4;
|
||||
}
|
||||
|
||||
writeUint32Big(0x52494646);
|
||||
writeUint32(36 + dataSize);
|
||||
writeUint32Big(0x57415645);
|
||||
|
||||
writeUint32Big(0x666d7420);
|
||||
writeUint32(16);
|
||||
writeUint16(3); // 3 represents 32-bit float PCM
|
||||
writeUint16(numChannels);
|
||||
writeUint32(samplingRate);
|
||||
writeUint32(byteRate);
|
||||
writeUint16(blockAlign);
|
||||
writeUint16(bytesPerSample * 8);
|
||||
|
||||
writeUint32Big(0x64617461);
|
||||
writeUint32(dataSize);
|
||||
|
||||
const array = new Float32Array(buffer);
|
||||
|
||||
for (let i = 0; i < array.length; i++) {
|
||||
writeFloat32(array[i]);
|
||||
}
|
||||
|
||||
return wavBuffer;
|
||||
}
|
||||
|
||||
const HEADER_LENGTH = 4;
|
||||
|
||||
/**
|
||||
* Builds a single buffer with JSON-serialized `meta` and `buffer`.
|
||||
*/
|
||||
export function encodeAnnotatedBuffer(meta, buffer) {
|
||||
const encoder = new TextEncoder();
|
||||
const metaArray = encoder.encode(JSON.stringify(meta));
|
||||
|
||||
const raw = new ArrayBuffer(
|
||||
HEADER_LENGTH + metaArray.byteLength + buffer.byteLength
|
||||
);
|
||||
const view = new DataView(raw);
|
||||
|
||||
view.setUint32(0, metaArray.byteLength);
|
||||
new Uint8Array(raw, HEADER_LENGTH, metaArray.byteLength).set(metaArray);
|
||||
new Uint8Array(raw, HEADER_LENGTH + metaArray.byteLength).set(
|
||||
new Uint8Array(buffer)
|
||||
);
|
||||
|
||||
return raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes binary annotated with JSON-serialized metadata.
|
||||
*/
|
||||
export function decodeAnnotatedBuffer(raw) {
|
||||
const view = new DataView(raw);
|
||||
const metaArrayLength = view.getUint32(0);
|
||||
|
||||
const metaArray = new Uint8Array(raw, HEADER_LENGTH, metaArrayLength);
|
||||
const buffer = raw.slice(HEADER_LENGTH + metaArrayLength);
|
||||
|
||||
const decoder = new TextDecoder();
|
||||
const meta = JSON.parse(decoder.decode(metaArray));
|
||||
|
||||
return [meta, buffer];
|
||||
}
|
||||
6
assets/package-lock.json
generated
6
assets/package-lock.json
generated
|
|
@ -51,14 +51,14 @@
|
|||
}
|
||||
},
|
||||
"../deps/phoenix": {
|
||||
"version": "1.7.5",
|
||||
"version": "1.7.7",
|
||||
"license": "MIT"
|
||||
},
|
||||
"../deps/phoenix_html": {
|
||||
"version": "3.3.1"
|
||||
"version": "3.3.2"
|
||||
},
|
||||
"../deps/phoenix_live_view": {
|
||||
"version": "0.19.5",
|
||||
"version": "0.20.0",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@alloc/quick-lru": {
|
||||
|
|
|
|||
|
|
@ -104,10 +104,4 @@ defmodule Livebook.Notebook.Cell do
|
|||
"""
|
||||
@spec setup_cell_id() :: id()
|
||||
def setup_cell_id(), do: @setup_cell_id
|
||||
|
||||
@doc """
|
||||
Checks if the given term is a file input value (info map).
|
||||
"""
|
||||
defguard is_file_input_value(value)
|
||||
when is_map_key(value, :file_ref) and is_map_key(value, :client_name)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -83,8 +83,6 @@ defmodule Livebook.Session do
|
|||
|
||||
use GenServer, restart: :temporary
|
||||
|
||||
import Livebook.Notebook.Cell, only: [is_file_input_value: 1]
|
||||
|
||||
alias Livebook.NotebookManager
|
||||
alias Livebook.Session.{Data, FileGuard}
|
||||
alias Livebook.{Utils, Notebook, Delta, Runtime, LiveMarkdown, FileSystem}
|
||||
|
|
@ -1854,6 +1852,17 @@ defmodule Livebook.Session do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a local path to a session-registered file with the given
|
||||
reference.
|
||||
"""
|
||||
@spec registered_file_path(id(), Livebook.Runtime.file_ref()) :: String.t()
|
||||
def registered_file_path(session_id, file_ref) do
|
||||
{:file, file_id} = file_ref
|
||||
%{path: session_dir} = session_tmp_dir(session_id)
|
||||
Path.join([session_dir, "registered_files", file_id])
|
||||
end
|
||||
|
||||
defp encode_path_component(component) do
|
||||
String.replace(component, [".", "/", "\\", ":"], "_")
|
||||
end
|
||||
|
|
@ -2283,14 +2292,8 @@ defmodule Livebook.Session do
|
|||
end
|
||||
|
||||
defp handle_action(state, {:clean_up_input_values, input_infos}) do
|
||||
for {_input_id, %{value: value}} <- input_infos do
|
||||
case value do
|
||||
value when is_file_input_value(value) ->
|
||||
schedule_file_deletion(state, value.file_ref)
|
||||
|
||||
_ ->
|
||||
:ok
|
||||
end
|
||||
for {_input_id, %{value: %{file_ref: file_ref}}} <- input_infos do
|
||||
schedule_file_deletion(state, file_ref)
|
||||
end
|
||||
|
||||
state
|
||||
|
|
@ -2527,11 +2530,6 @@ defmodule Livebook.Session do
|
|||
end
|
||||
end
|
||||
|
||||
defp registered_file_path(session_id, {:file, file_id}) do
|
||||
%{path: session_dir} = session_tmp_dir(session_id)
|
||||
Path.join([session_dir, "registered_files", file_id])
|
||||
end
|
||||
|
||||
defp schedule_file_deletion(state, file_ref) do
|
||||
Process.send_after(
|
||||
self(),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
defmodule LivebookWeb.JSViewChannel do
|
||||
use Phoenix.Channel
|
||||
|
||||
alias LivebookWeb.Helpers.Codec
|
||||
|
||||
@impl true
|
||||
def join("js_view", %{"session_token" => session_token}, socket) do
|
||||
case Phoenix.Token.verify(LivebookWeb.Endpoint, "session", session_token) do
|
||||
|
|
@ -154,7 +156,7 @@ defmodule LivebookWeb.JSViewChannel do
|
|||
# payload accordingly
|
||||
|
||||
defp transport_encode!(meta, {:binary, info, binary}) do
|
||||
{:binary, encode!([meta, info], binary)}
|
||||
{:binary, Codec.encode_annotated_binary!([meta, info], binary)}
|
||||
end
|
||||
|
||||
defp transport_encode!(meta, payload) do
|
||||
|
|
@ -162,7 +164,7 @@ defmodule LivebookWeb.JSViewChannel do
|
|||
end
|
||||
|
||||
defp transport_decode!({:binary, raw}) do
|
||||
{[meta, info], binary} = decode!(raw)
|
||||
{[meta, info], binary} = Codec.decode_annotated_binary!(raw)
|
||||
{meta, {:binary, info, binary}}
|
||||
end
|
||||
|
||||
|
|
@ -170,16 +172,4 @@ defmodule LivebookWeb.JSViewChannel do
|
|||
%{"root" => [meta, payload]} = raw
|
||||
{meta, payload}
|
||||
end
|
||||
|
||||
defp encode!(meta, binary) do
|
||||
meta = Jason.encode!(meta)
|
||||
meta_size = byte_size(meta)
|
||||
<<meta_size::size(32), meta::binary, binary::binary>>
|
||||
end
|
||||
|
||||
defp decode!(raw) do
|
||||
<<meta_size::size(32), meta::binary-size(meta_size), binary::binary>> = raw
|
||||
meta = Jason.decode!(meta)
|
||||
{meta, binary}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -603,30 +603,51 @@ defmodule LivebookWeb.FormComponents do
|
|||
<.live_file_input upload={@upload} class="hidden" />
|
||||
<.label><%= @label %></.label>
|
||||
<div :for={entry <- @upload.entries} class="flex flex-col gap-1">
|
||||
<div class="flex flex-col gap-0.5">
|
||||
<div class="flex items-center justify-between text-gray-700">
|
||||
<span><%= entry.client_name %></span>
|
||||
<button
|
||||
type="button"
|
||||
class="ml-1 text-gray-500 hover:text-gray-900"
|
||||
phx-click={@on_clear}
|
||||
phx-value-ref={entry.ref}
|
||||
tabindex="-1"
|
||||
>
|
||||
<.remix_icon icon="close-line" />
|
||||
</button>
|
||||
<span class="flex-grow"></span>
|
||||
<span :if={entry.preflighted?} class="text-sm font-medium">
|
||||
<%= entry.progress %>%
|
||||
</span>
|
||||
</div>
|
||||
<div :if={entry.preflighted?} class="w-full h-2 rounded-lg bg-blue-200">
|
||||
<div
|
||||
class="h-full rounded-lg bg-blue-600 transition-all ease-out duration-1000"
|
||||
style={"width: #{entry.progress}%"}
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
<.file_entry entry={entry} on_clear={@on_clear} />
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
||||
@doc """
|
||||
Renders a file entry with progress.
|
||||
|
||||
## Examples
|
||||
|
||||
<.file_entry
|
||||
entry={entry}
|
||||
on_clear={JS.push("clear_file", target: @myself)}
|
||||
/>
|
||||
|
||||
"""
|
||||
attr :entry, Phoenix.LiveView.UploadEntry, required: true
|
||||
attr :on_clear, Phoenix.LiveView.JS, required: true
|
||||
attr :name, :string, default: nil
|
||||
|
||||
def file_entry(assigns) do
|
||||
~H"""
|
||||
<div class="flex flex-col gap-0.5">
|
||||
<div class="flex items-center justify-between gap-1 text-gray-700">
|
||||
<span><%= @name || @entry.client_name %></span>
|
||||
<button
|
||||
type="button"
|
||||
class="text-gray-500 hover:text-gray-900"
|
||||
phx-click={@on_clear}
|
||||
phx-value-ref={@entry.ref}
|
||||
tabindex="-1"
|
||||
>
|
||||
<.remix_icon icon="close-line" />
|
||||
</button>
|
||||
<span class="flex-grow"></span>
|
||||
<span :if={@entry.preflighted?} class="text-sm font-medium">
|
||||
<%= @entry.progress %>%
|
||||
</span>
|
||||
</div>
|
||||
<div :if={@entry.preflighted?} class="w-full h-2 rounded-lg bg-blue-200">
|
||||
<div
|
||||
class="h-full rounded-lg bg-blue-600 transition-all ease-out duration-1000"
|
||||
style={"width: #{@entry.progress}%"}
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ defmodule LivebookWeb.SessionController do
|
|||
use LivebookWeb, :controller
|
||||
|
||||
alias Livebook.{Sessions, Session, FileSystem}
|
||||
alias LivebookWeb.Helpers.Codec
|
||||
|
||||
def show_file(conn, %{"id" => id, "name" => name}) do
|
||||
with {:ok, session} <- Sessions.fetch_session(id),
|
||||
|
|
@ -173,6 +174,91 @@ defmodule LivebookWeb.SessionController do
|
|||
end
|
||||
end
|
||||
|
||||
def show_input_audio(conn, %{"token" => token}) do
|
||||
{live_view_pid, input_id} = LivebookWeb.SessionHelpers.verify_input_token!(token)
|
||||
|
||||
case GenServer.call(live_view_pid, {:get_input_value, input_id}) do
|
||||
{:ok, session_id, value} ->
|
||||
path = Livebook.Session.registered_file_path(session_id, value.file_ref)
|
||||
|
||||
conn =
|
||||
conn
|
||||
|> cache_permanently()
|
||||
|> put_resp_header("accept-ranges", "bytes")
|
||||
|
||||
case value.format do
|
||||
:pcm_f32 ->
|
||||
%{size: file_size} = File.stat!(path)
|
||||
|
||||
total_size = Codec.pcm_as_wav_size(file_size)
|
||||
|
||||
case parse_byte_range(conn, total_size) do
|
||||
{range_start, range_end} when range_start > 0 or range_end < total_size - 1 ->
|
||||
stream =
|
||||
Codec.encode_pcm_as_wav_stream!(
|
||||
path,
|
||||
file_size,
|
||||
value.num_channels,
|
||||
value.sampling_rate,
|
||||
range_start,
|
||||
range_end - range_start + 1
|
||||
)
|
||||
|
||||
conn
|
||||
|> put_content_range(range_start, range_end, total_size)
|
||||
|> send_stream(206, stream)
|
||||
|
||||
_ ->
|
||||
stream =
|
||||
Codec.encode_pcm_as_wav_stream!(
|
||||
path,
|
||||
file_size,
|
||||
value.num_channels,
|
||||
value.sampling_rate,
|
||||
0,
|
||||
total_size
|
||||
)
|
||||
|
||||
conn
|
||||
|> put_resp_header("content-length", Integer.to_string(total_size))
|
||||
|> send_stream(200, stream)
|
||||
end
|
||||
|
||||
:wav ->
|
||||
%{size: total_size} = File.stat!(path)
|
||||
|
||||
case parse_byte_range(conn, total_size) do
|
||||
{range_start, range_end} when range_start > 0 or range_end < total_size - 1 ->
|
||||
conn
|
||||
|> put_content_range(range_start, range_end, total_size)
|
||||
|> send_file(206, path, range_start, range_end - range_start + 1)
|
||||
|
||||
_ ->
|
||||
send_file(conn, 200, path)
|
||||
end
|
||||
end
|
||||
|
||||
:error ->
|
||||
send_resp(conn, 404, "Not found")
|
||||
end
|
||||
end
|
||||
|
||||
def show_input_image(conn, %{"token" => token}) do
|
||||
{live_view_pid, input_id} = LivebookWeb.SessionHelpers.verify_input_token!(token)
|
||||
|
||||
case GenServer.call(live_view_pid, {:get_input_value, input_id}) do
|
||||
{:ok, session_id, value} ->
|
||||
path = Livebook.Session.registered_file_path(session_id, value.file_ref)
|
||||
|
||||
conn
|
||||
|> cache_permanently()
|
||||
|> send_file(200, path)
|
||||
|
||||
:error ->
|
||||
send_resp(conn, 404, "Not found")
|
||||
end
|
||||
end
|
||||
|
||||
defp accept_encoding?(conn, encoding) do
|
||||
encoding? = &String.contains?(&1, [encoding, "*"])
|
||||
|
||||
|
|
@ -217,4 +303,55 @@ defmodule LivebookWeb.SessionController do
|
|||
content_type = MIME.from_path(path)
|
||||
put_resp_header(conn, "content-type", content_type)
|
||||
end
|
||||
|
||||
defp parse_byte_range(conn, total_size) do
|
||||
with [range] <- get_req_header(conn, "range"),
|
||||
%{"bytes" => bytes} <- Plug.Conn.Utils.params(range),
|
||||
{range_start, range_end} <- start_and_end(bytes, total_size) do
|
||||
{range_start, range_end}
|
||||
else
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
defp start_and_end("-" <> rest, total_size) do
|
||||
case Integer.parse(rest) do
|
||||
{last, ""} when last > 0 and last <= total_size -> {total_size - last, total_size - 1}
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
defp start_and_end(range, total_size) do
|
||||
case Integer.parse(range) do
|
||||
{first, "-"} when first >= 0 ->
|
||||
{first, total_size - 1}
|
||||
|
||||
{first, "-" <> rest} when first >= 0 ->
|
||||
case Integer.parse(rest) do
|
||||
{last, ""} when last >= first -> {first, min(last, total_size - 1)}
|
||||
_ -> :error
|
||||
end
|
||||
|
||||
_ ->
|
||||
:error
|
||||
end
|
||||
end
|
||||
|
||||
defp put_content_range(conn, range_start, range_end, total_size) do
|
||||
put_resp_header(conn, "content-range", "bytes #{range_start}-#{range_end}/#{total_size}")
|
||||
end
|
||||
|
||||
defp send_stream(conn, status, stream) do
|
||||
conn = send_chunked(conn, status)
|
||||
|
||||
Enum.reduce_while(stream, conn, fn chunk, conn ->
|
||||
case Plug.Conn.chunk(conn, chunk) do
|
||||
{:ok, conn} ->
|
||||
{:cont, conn}
|
||||
|
||||
{:error, :closed} ->
|
||||
{:halt, conn}
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
130
lib/livebook_web/helpers/codec.ex
Normal file
130
lib/livebook_web/helpers/codec.ex
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
defmodule LivebookWeb.Helpers.Codec do
|
||||
@wav_header_size 44
|
||||
|
||||
@doc """
|
||||
Returns the size of a WAV binary that would wrap PCM data of the
|
||||
given size.
|
||||
|
||||
This size matches the result of `encode_pcm_as_wav_stream!/6`.
|
||||
"""
|
||||
@spec pcm_as_wav_size(pos_integer()) :: pos_integer()
|
||||
def pcm_as_wav_size(pcm_size) do
|
||||
@wav_header_size + pcm_size
|
||||
end
|
||||
|
||||
@doc """
|
||||
Encodes PCM float-32 in native endianness into a WAV binary.
|
||||
|
||||
Accepts a range of the WAV binary that should be returned. Returns
|
||||
a stream, where the PCM binary is streamed from the given file.
|
||||
"""
|
||||
@spec encode_pcm_as_wav_stream!(
|
||||
Path.t(),
|
||||
non_neg_integer(),
|
||||
pos_integer(),
|
||||
pos_integer(),
|
||||
non_neg_integer(),
|
||||
pos_integer()
|
||||
) :: Enumerable.t()
|
||||
def encode_pcm_as_wav_stream!(path, file_size, num_channels, sampling_rate, offset, length) do
|
||||
{header_enum, file_length} =
|
||||
if offset < @wav_header_size do
|
||||
header = encode_pcm_as_wav_header(file_size, num_channels, sampling_rate)
|
||||
header_length = min(@wav_header_size - offset, length)
|
||||
header_slice = binary_slice(header, offset, header_length)
|
||||
{[header_slice], length - header_length}
|
||||
else
|
||||
{[], length}
|
||||
end
|
||||
|
||||
file_offset = max(offset - @wav_header_size, 0)
|
||||
|
||||
file_stream = raw_file_range_stream!(path, file_offset, file_length)
|
||||
|
||||
file_stream =
|
||||
case System.endianness() do
|
||||
:little ->
|
||||
file_stream
|
||||
|
||||
:big ->
|
||||
Stream.map(file_stream, fn binary ->
|
||||
for <<x::32-float-big <- binary>>, reduce: <<>> do
|
||||
acc -> <<acc::binary, x::32-float-little>>
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
Stream.concat(header_enum, file_stream)
|
||||
end
|
||||
|
||||
defp encode_pcm_as_wav_header(pcm_size, num_channels, sampling_rate) do
|
||||
# See http://soundfile.sapp.org/doc/WaveFormat
|
||||
|
||||
num_frames = div(pcm_size, 4)
|
||||
bytes_per_sample = 4
|
||||
|
||||
block_align = num_channels * bytes_per_sample
|
||||
byte_rate = sampling_rate * block_align
|
||||
data_size = num_frames * block_align
|
||||
|
||||
<<
|
||||
"RIFF",
|
||||
36 + data_size::32-unsigned-integer-little,
|
||||
"WAVE",
|
||||
"fmt ",
|
||||
16::32-unsigned-integer-little,
|
||||
# 3 indicates 32-bit float PCM
|
||||
3::16-unsigned-integer-little,
|
||||
num_channels::16-unsigned-integer-little,
|
||||
sampling_rate::32-unsigned-integer-little,
|
||||
byte_rate::32-unsigned-integer-little,
|
||||
block_align::16-unsigned-integer-little,
|
||||
bytes_per_sample * 8::16-unsigned-integer-little,
|
||||
"data",
|
||||
data_size::32-unsigned-integer-little
|
||||
>>
|
||||
end
|
||||
|
||||
# We assume a local path and open a raw file for efficiency
|
||||
defp raw_file_range_stream!(path, offset, length) do
|
||||
chunk_size = 64_000
|
||||
|
||||
Stream.resource(
|
||||
fn ->
|
||||
{:ok, fd} = :file.open(path, [:raw, :binary, :read, :read_ahead])
|
||||
{:ok, _} = :file.position(fd, offset)
|
||||
{fd, length}
|
||||
end,
|
||||
fn
|
||||
{fd, 0} ->
|
||||
{:halt, {fd, 0}}
|
||||
|
||||
{fd, length} ->
|
||||
size = min(chunk_size, length)
|
||||
{:ok, chunk} = :file.read(fd, size)
|
||||
{[chunk], {fd, length - size}}
|
||||
end,
|
||||
fn {fd, _} -> :file.close(fd) end
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Builds a single binary with JSON-serialized `meta` and `binary`.
|
||||
"""
|
||||
@spec encode_annotated_binary!(term(), binary()) :: binary()
|
||||
def encode_annotated_binary!(meta, binary) do
|
||||
meta = Jason.encode!(meta)
|
||||
meta_size = byte_size(meta)
|
||||
<<meta_size::size(32), meta::binary, binary::binary>>
|
||||
end
|
||||
|
||||
@doc """
|
||||
Decodes binary annotated with JSON-serialized metadata.
|
||||
"""
|
||||
@spec decode_annotated_binary!(binary()) :: {term(), binary()}
|
||||
def decode_annotated_binary!(raw) do
|
||||
<<meta_size::size(32), meta::binary-size(meta_size), binary::binary>> = raw
|
||||
meta = Jason.decode!(meta)
|
||||
{meta, binary}
|
||||
end
|
||||
end
|
||||
|
|
@ -45,17 +45,32 @@ defmodule LivebookWeb.Output do
|
|||
|
||||
defp render_output(%{type: :terminal_text, text: text}, %{id: id}) do
|
||||
text = if(text == :__pruned__, do: nil, else: text)
|
||||
live_component(Output.TerminalTextComponent, id: id, text: text)
|
||||
|
||||
assigns = %{id: id, text: text}
|
||||
|
||||
~H"""
|
||||
<.live_component module={Output.TerminalTextComponent} id={@id} text={@text} />
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(%{type: :plain_text, text: text}, %{id: id}) do
|
||||
text = if(text == :__pruned__, do: nil, else: text)
|
||||
live_component(Output.PlainTextComponent, id: id, text: text)
|
||||
|
||||
assigns = %{id: id, text: text}
|
||||
|
||||
~H"""
|
||||
<.live_component module={Output.PlainTextComponent} id={@id} text={@text} />
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(%{type: :markdown, text: text}, %{id: id, session_id: session_id}) do
|
||||
text = if(text == :__pruned__, do: nil, else: text)
|
||||
live_component(Output.MarkdownComponent, id: id, session_id: session_id, text: text)
|
||||
|
||||
assigns = %{id: id, session_id: session_id, text: text}
|
||||
|
||||
~H"""
|
||||
<.live_component module={Output.MarkdownComponent} id={@id} session_id={@session_id} text={@text} />
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(%{type: :image} = output, %{id: id}) do
|
||||
|
|
@ -71,13 +86,23 @@ defmodule LivebookWeb.Output do
|
|||
session_id: session_id,
|
||||
client_id: client_id
|
||||
}) do
|
||||
live_component(LivebookWeb.JSViewComponent,
|
||||
assigns = %{
|
||||
id: id,
|
||||
js_view: output.js_view,
|
||||
session_id: session_id,
|
||||
client_id: client_id,
|
||||
timeout_message: "Output data no longer available, please reevaluate this cell"
|
||||
)
|
||||
client_id: client_id
|
||||
}
|
||||
|
||||
~H"""
|
||||
<.live_component
|
||||
module={LivebookWeb.JSViewComponent}
|
||||
id={@id}
|
||||
js_view={@js_view}
|
||||
session_id={@session_id}
|
||||
client_id={@client_id}
|
||||
timeout_message="Output data no longer available, please reevaluate this cell"
|
||||
/>
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(%{type: :frame} = output, %{
|
||||
|
|
@ -88,7 +113,7 @@ defmodule LivebookWeb.Output do
|
|||
client_id: client_id,
|
||||
cell_id: cell_id
|
||||
}) do
|
||||
live_component(Output.FrameComponent,
|
||||
assigns = %{
|
||||
id: id,
|
||||
outputs: output.outputs,
|
||||
placeholder: output.placeholder,
|
||||
|
|
@ -97,7 +122,21 @@ defmodule LivebookWeb.Output do
|
|||
input_views: input_views,
|
||||
client_id: client_id,
|
||||
cell_id: cell_id
|
||||
)
|
||||
}
|
||||
|
||||
~H"""
|
||||
<.live_component
|
||||
module={Output.FrameComponent}
|
||||
id={@id}
|
||||
outputs={@outputs}
|
||||
placeholder={@placeholder}
|
||||
session_id={@session_id}
|
||||
session_pid={@session_pid}
|
||||
input_views={@input_views}
|
||||
client_id={@client_id}
|
||||
cell_id={@cell_id}
|
||||
/>
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(%{type: :tabs, outputs: outputs, labels: labels}, %{
|
||||
|
|
@ -226,13 +265,24 @@ defmodule LivebookWeb.Output do
|
|||
session_pid: session_pid,
|
||||
client_id: client_id
|
||||
}) do
|
||||
live_component(Output.InputComponent,
|
||||
assigns = %{
|
||||
id: id,
|
||||
input: input,
|
||||
input_views: input_views,
|
||||
session_pid: session_pid,
|
||||
client_id: client_id
|
||||
)
|
||||
}
|
||||
|
||||
~H"""
|
||||
<.live_component
|
||||
module={Output.InputComponent}
|
||||
id={@id}
|
||||
input={@input}
|
||||
input_views={@input_views}
|
||||
session_pid={@session_pid}
|
||||
client_id={@client_id}
|
||||
/>
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(%{type: :control} = control, %{
|
||||
|
|
@ -242,14 +292,26 @@ defmodule LivebookWeb.Output do
|
|||
client_id: client_id,
|
||||
cell_id: cell_id
|
||||
}) do
|
||||
live_component(Output.ControlComponent,
|
||||
assigns = %{
|
||||
id: id,
|
||||
control: control,
|
||||
input_views: input_views,
|
||||
session_pid: session_pid,
|
||||
client_id: client_id,
|
||||
cell_id: cell_id
|
||||
)
|
||||
}
|
||||
|
||||
~H"""
|
||||
<.live_component
|
||||
module={Output.ControlComponent}
|
||||
id={@id}
|
||||
control={@control}
|
||||
input_views={@input_views}
|
||||
session_pid={@session_pid}
|
||||
client_id={@client_id}
|
||||
cell_id={@cell_id}
|
||||
/>
|
||||
"""
|
||||
end
|
||||
|
||||
defp render_output(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,21 @@ defmodule LivebookWeb.Output.AudioInputComponent do
|
|||
|
||||
@impl true
|
||||
def mount(socket) do
|
||||
{:ok, assign(socket, endianness: System.endianness(), value: nil)}
|
||||
{:ok,
|
||||
socket
|
||||
|> assign(
|
||||
endianness: System.endianness(),
|
||||
value: nil,
|
||||
audio_url: nil,
|
||||
decoding?: false
|
||||
)
|
||||
|> allow_upload(:file,
|
||||
accept: :any,
|
||||
max_entries: 1,
|
||||
max_file_size: 100_000_000_000,
|
||||
progress: &handle_progress/3,
|
||||
auto_upload: true
|
||||
)}
|
||||
end
|
||||
|
||||
@impl true
|
||||
|
|
@ -13,93 +27,150 @@ defmodule LivebookWeb.Output.AudioInputComponent do
|
|||
socket = assign(socket, assigns)
|
||||
|
||||
socket =
|
||||
if value == socket.assigns.value do
|
||||
socket
|
||||
else
|
||||
audio_info =
|
||||
if value do
|
||||
%{
|
||||
data: Base.encode64(value.data),
|
||||
num_channels: value.num_channels,
|
||||
sampling_rate: value.sampling_rate
|
||||
}
|
||||
end
|
||||
cond do
|
||||
value == socket.assigns.value ->
|
||||
socket
|
||||
|
||||
push_event(socket, "audio_input_change:#{socket.assigns.id}", %{audio_info: audio_info})
|
||||
value == nil ->
|
||||
assign(socket, value: value, audio_url: nil)
|
||||
|
||||
true ->
|
||||
assign(socket, value: value, audio_url: audio_url(socket.assigns.input_id))
|
||||
end
|
||||
|
||||
{:ok, assign(socket, value: value)}
|
||||
{:ok, socket}
|
||||
end
|
||||
|
||||
defp audio_url(input_id) do
|
||||
# For the client-side audio preview, we serve audio encoded as WAV
|
||||
# from a separate endpoint. To do that, we encode information in a
|
||||
# token and then the controller fetches input value from the LV.
|
||||
# This is especially important for client-specific inputs in forms.
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(self(), input_id)
|
||||
~p"/sessions/audio-input/#{token}"
|
||||
end
|
||||
|
||||
@impl true
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
<div
|
||||
id={"#{@id}-root"}
|
||||
class="inline-flex flex-col gap-4 p-4 border-2 border-dashed border-gray-200 rounded-lg"
|
||||
phx-hook="AudioInput"
|
||||
phx-update="ignore"
|
||||
data-id={@id}
|
||||
data-phx-target={@myself}
|
||||
data-format={@format}
|
||||
data-sampling-rate={@sampling_rate}
|
||||
data-endianness={@endianness}
|
||||
>
|
||||
<input type="file" data-input class="hidden" name="html_value" accept="audio/*" capture="user" />
|
||||
<audio controls data-preview></audio>
|
||||
<div class="flex items-center justify-center gap-4">
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-record
|
||||
>
|
||||
<.remix_icon icon="mic-line" class="text-lg leading-none mr-2" />
|
||||
<span>Record</span>
|
||||
</button>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500 items-center"
|
||||
data-btn-stop
|
||||
>
|
||||
<span class="mr-2 flex h-3 w-3 relative">
|
||||
<span class="animate-ping absolute inline-flex h-full w-full rounded-full bg-gray-400 opacity-75">
|
||||
<div class="inline-flex flex-col gap-4 p-4 border-2 border-dashed border-gray-200 rounded-lg">
|
||||
<div
|
||||
class="inline-flex flex-col gap-4"
|
||||
id={"#{@id}-root"}
|
||||
phx-hook="AudioInput"
|
||||
phx-update="ignore"
|
||||
data-id={@id}
|
||||
data-phx-target={@myself}
|
||||
data-format={@format}
|
||||
data-sampling-rate={@sampling_rate}
|
||||
data-endianness={@endianness}
|
||||
data-audio-url={@audio_url}
|
||||
>
|
||||
<input
|
||||
type="file"
|
||||
data-input
|
||||
class="hidden"
|
||||
name="html_value"
|
||||
accept="audio/*"
|
||||
capture="user"
|
||||
/>
|
||||
<audio controls data-preview></audio>
|
||||
<div class="flex items-center justify-center gap-4">
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-record
|
||||
>
|
||||
<.remix_icon icon="mic-line" class="text-lg leading-none mr-2" />
|
||||
<span>Record</span>
|
||||
</button>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500 items-center"
|
||||
data-btn-stop
|
||||
>
|
||||
<span class="mr-2 flex h-3 w-3 relative">
|
||||
<span class="animate-ping absolute inline-flex h-full w-full rounded-full bg-gray-400 opacity-75">
|
||||
</span>
|
||||
<span class="relative inline-flex rounded-full h-3 w-3 bg-gray-500"></span>
|
||||
</span>
|
||||
<span class="relative inline-flex rounded-full h-3 w-3 bg-gray-500"></span>
|
||||
</span>
|
||||
<span>Stop recording</span>
|
||||
</button>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-cancel
|
||||
>
|
||||
<.remix_icon icon="close-circle-line" class="text-lg leading-none mr-2" />
|
||||
<span>Cancel</span>
|
||||
</button>
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-upload
|
||||
>
|
||||
<.remix_icon icon="upload-2-line" class="text-lg leading-none mr-2" />
|
||||
<span>Upload</span>
|
||||
</button>
|
||||
<span>Stop recording</span>
|
||||
</button>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-cancel
|
||||
>
|
||||
<.remix_icon icon="close-circle-line" class="text-lg leading-none mr-2" />
|
||||
<span>Cancel</span>
|
||||
</button>
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-upload
|
||||
>
|
||||
<.remix_icon icon="upload-2-line" class="text-lg leading-none mr-2" />
|
||||
<span>Upload</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<form phx-change="validate" class="hidden" phx-target={@myself}>
|
||||
<.live_file_input upload={@uploads.file} />
|
||||
</form>
|
||||
<div
|
||||
:if={@uploads.file.entries == [] and @decoding?}
|
||||
class="delay-200 flex justify-center items-center gap-2"
|
||||
>
|
||||
<span class="text-sm font-medium text-gray-500">Decoding</span> <.spinner />
|
||||
</div>
|
||||
<div :for={entry <- @uploads.file.entries} class="delay-200 flex flex-col gap-1">
|
||||
<.file_entry name="Audio" entry={entry} on_clear={JS.push("clear_file", target: @myself)} />
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_event("change", params, socket) do
|
||||
value = %{
|
||||
data: Base.decode64!(params["data"]),
|
||||
num_channels: params["num_channels"],
|
||||
sampling_rate: params["sampling_rate"],
|
||||
format: socket.assigns.format
|
||||
}
|
||||
|
||||
send_update(LivebookWeb.Output.InputComponent,
|
||||
id: socket.assigns.input_component_id,
|
||||
event: :change,
|
||||
value: value
|
||||
)
|
||||
def handle_event("decoding", %{}, socket) do
|
||||
{:noreply, assign(socket, decoding?: true)}
|
||||
end
|
||||
|
||||
def handle_event("validate", %{}, socket) do
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
def handle_event("clear_file", %{"ref" => ref}, socket) do
|
||||
{:noreply, cancel_upload(socket, :file, ref)}
|
||||
end
|
||||
|
||||
defp handle_progress(:file, entry, socket) do
|
||||
if entry.done? do
|
||||
file_ref =
|
||||
consume_uploaded_entry(socket, entry, fn %{path: path} ->
|
||||
{:ok, file_ref} =
|
||||
LivebookWeb.SessionHelpers.register_input_file(
|
||||
socket.assigns.session_pid,
|
||||
path,
|
||||
socket.assigns.input_id,
|
||||
socket.assigns.local,
|
||||
socket.assigns.client_id
|
||||
)
|
||||
|
||||
{:ok, file_ref}
|
||||
end)
|
||||
|
||||
%{"num_channels" => num_channels, "sampling_rate" => sampling_rate} = entry.client_meta
|
||||
|
||||
value = %{
|
||||
file_ref: file_ref,
|
||||
num_channels: num_channels,
|
||||
sampling_rate: sampling_rate,
|
||||
format: socket.assigns.format
|
||||
}
|
||||
|
||||
send_update(LivebookWeb.Output.InputComponent,
|
||||
id: socket.assigns.input_component_id,
|
||||
event: :change,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
{:noreply, assign(socket, decoding?: false)}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -71,35 +71,27 @@ defmodule LivebookWeb.Output.FileInputComponent do
|
|||
|
||||
defp handle_progress(:file, entry, socket) do
|
||||
if entry.done? do
|
||||
socket
|
||||
|> consume_uploaded_entries(:file, fn %{path: path}, entry ->
|
||||
{:ok, file_ref} =
|
||||
if socket.assigns.local do
|
||||
key = "#{socket.assigns.input_id}-#{socket.assigns.client_id}"
|
||||
|
||||
Livebook.Session.register_file(socket.assigns.session_pid, path, key,
|
||||
linked_client_id: socket.assigns.client_id
|
||||
{file_ref, client_name} =
|
||||
consume_uploaded_entry(socket, entry, fn %{path: path} ->
|
||||
{:ok, file_ref} =
|
||||
LivebookWeb.SessionHelpers.register_input_file(
|
||||
socket.assigns.session_pid,
|
||||
path,
|
||||
socket.assigns.input_id,
|
||||
socket.assigns.local,
|
||||
socket.assigns.client_id
|
||||
)
|
||||
else
|
||||
key = "#{socket.assigns.input_id}-global"
|
||||
Livebook.Session.register_file(socket.assigns.session_pid, path, key)
|
||||
end
|
||||
|
||||
{:ok, {file_ref, entry.client_name}}
|
||||
end)
|
||||
|> case do
|
||||
[{file_ref, client_name}] ->
|
||||
value = %{file_ref: file_ref, client_name: client_name}
|
||||
{:ok, {file_ref, entry.client_name}}
|
||||
end)
|
||||
|
||||
send_update(LivebookWeb.Output.InputComponent,
|
||||
id: socket.assigns.input_component_id,
|
||||
event: :change,
|
||||
value: value
|
||||
)
|
||||
value = %{file_ref: file_ref, client_name: client_name}
|
||||
|
||||
[] ->
|
||||
:ok
|
||||
end
|
||||
send_update(LivebookWeb.Output.InputComponent,
|
||||
id: socket.assigns.input_component_id,
|
||||
event: :change,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
{:noreply, socket}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,16 @@ defmodule LivebookWeb.Output.ImageInputComponent do
|
|||
|
||||
@impl true
|
||||
def mount(socket) do
|
||||
{:ok, assign(socket, value: nil)}
|
||||
{:ok,
|
||||
socket
|
||||
|> assign(value: nil, value: nil, image_url: nil)
|
||||
|> allow_upload(:file,
|
||||
accept: :any,
|
||||
max_entries: 1,
|
||||
max_file_size: 100_000_000_000,
|
||||
progress: &handle_progress/3,
|
||||
auto_upload: true
|
||||
)}
|
||||
end
|
||||
|
||||
@impl true
|
||||
|
|
@ -13,101 +22,155 @@ defmodule LivebookWeb.Output.ImageInputComponent do
|
|||
socket = assign(socket, assigns)
|
||||
|
||||
socket =
|
||||
if value == socket.assigns.value do
|
||||
socket
|
||||
else
|
||||
image_info =
|
||||
if value do
|
||||
%{data: Base.encode64(value.data), height: value.height, width: value.width}
|
||||
end
|
||||
cond do
|
||||
value == socket.assigns.value ->
|
||||
socket
|
||||
|
||||
push_event(socket, "image_input_change:#{socket.assigns.id}", %{image_info: image_info})
|
||||
value == nil ->
|
||||
assign(socket, value: value, image_url: nil)
|
||||
|
||||
true ->
|
||||
assign(socket, value: value, image_url: image_url(socket.assigns.input_id))
|
||||
end
|
||||
|
||||
{:ok, assign(socket, value: value)}
|
||||
{:ok, socket}
|
||||
end
|
||||
|
||||
defp image_url(input_id) do
|
||||
# For the client-side image preview, we serve the original binary
|
||||
# value from a separate endpoint. To do that, we encode information
|
||||
# in a token and then the controller fetches input value from the
|
||||
# LV. This is especially important for client-specific inputs in
|
||||
# forms.
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(self(), input_id)
|
||||
~p"/sessions/image-input/#{token}"
|
||||
end
|
||||
|
||||
@impl true
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
<div
|
||||
id={"#{@id}-root"}
|
||||
class="inline-flex flex-col p-4 border-2 border-dashed border-gray-200 rounded-lg"
|
||||
phx-hook="ImageInput"
|
||||
phx-update="ignore"
|
||||
data-id={@id}
|
||||
data-phx-target={@myself}
|
||||
data-height={@height}
|
||||
data-width={@width}
|
||||
data-format={@format}
|
||||
data-fit={@fit}
|
||||
>
|
||||
<input type="file" data-input class="hidden" name="html_value" accept="image/*" capture="user" />
|
||||
<div class="flex justify-center" data-preview>
|
||||
<div class="flex justify-center text-gray-500">
|
||||
Drag an image file
|
||||
<div class="inline-flex flex-col gap-4 p-4 border-2 border-dashed border-gray-200 rounded-lg">
|
||||
<div
|
||||
id={"#{@id}-root"}
|
||||
class="inline-flex flex-col"
|
||||
phx-hook="ImageInput"
|
||||
phx-update="ignore"
|
||||
data-id={@id}
|
||||
data-phx-target={@myself}
|
||||
data-height={@height}
|
||||
data-width={@width}
|
||||
data-format={@format}
|
||||
data-fit={@fit}
|
||||
data-image-url={@image_url}
|
||||
data-value-height={@value[:height]}
|
||||
data-value-width={@value[:width]}
|
||||
>
|
||||
<input
|
||||
type="file"
|
||||
data-input
|
||||
class="hidden"
|
||||
name="html_value"
|
||||
accept="image/*"
|
||||
capture="user"
|
||||
/>
|
||||
<div class="flex justify-center" data-preview>
|
||||
<div class="flex justify-center text-gray-500">
|
||||
Drag an image file
|
||||
</div>
|
||||
</div>
|
||||
<div class="hidden flex justify-center" data-camera-preview></div>
|
||||
<div class="mt-4 flex items-center justify-center gap-4">
|
||||
<.menu id={"#{@id}-camera-select-menu"} position={:bottom_left}>
|
||||
<:toggle>
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-open-camera
|
||||
>
|
||||
<.remix_icon icon="camera-line" class="text-lg leading-none mr-2" />
|
||||
<span>Open camera</span>
|
||||
</button>
|
||||
</:toggle>
|
||||
<div data-camera-list>
|
||||
<.menu_item>
|
||||
<button role="menuitem" data-camera-id>
|
||||
<span class="font-medium" data-label></span>
|
||||
</button>
|
||||
</.menu_item>
|
||||
</div>
|
||||
</.menu>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-capture-camera
|
||||
>
|
||||
<.remix_icon icon="camera-line" class="text-lg leading-none mr-2" />
|
||||
<span>Take photo</span>
|
||||
</button>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-cancel
|
||||
>
|
||||
<.remix_icon icon="close-circle-line" class="text-lg leading-none mr-2" />
|
||||
<span>Cancel</span>
|
||||
</button>
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-upload
|
||||
>
|
||||
<.remix_icon icon="upload-2-line" class="text-lg leading-none mr-2" />
|
||||
<span>Upload</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hidden flex justify-center" data-camera-preview></div>
|
||||
<div class="mt-4 flex items-center justify-center gap-4">
|
||||
<.menu id={"#{@id}-camera-select-menu"} position={:bottom_left}>
|
||||
<:toggle>
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-open-camera
|
||||
>
|
||||
<.remix_icon icon="camera-line" class="text-lg leading-none mr-2" />
|
||||
<span>Open camera</span>
|
||||
</button>
|
||||
</:toggle>
|
||||
<div data-camera-list>
|
||||
<.menu_item>
|
||||
<button role="menuitem" data-camera-id>
|
||||
<span class="font-medium" data-label></span>
|
||||
</button>
|
||||
</.menu_item>
|
||||
</div>
|
||||
</.menu>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-capture-camera
|
||||
>
|
||||
<.remix_icon icon="camera-line" class="text-lg leading-none mr-2" />
|
||||
<span>Take photo</span>
|
||||
</button>
|
||||
<button
|
||||
class="hidden button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-cancel
|
||||
>
|
||||
<.remix_icon icon="close-circle-line" class="text-lg leading-none mr-2" />
|
||||
<span>Cancel</span>
|
||||
</button>
|
||||
<button
|
||||
class="button-base button-gray border-transparent py-2 px-4 inline-flex text-gray-500"
|
||||
data-btn-upload
|
||||
>
|
||||
<.remix_icon icon="upload-2-line" class="text-lg leading-none mr-2" />
|
||||
<span>Upload</span>
|
||||
</button>
|
||||
<form phx-change="validate" class="hidden" phx-target={@myself}>
|
||||
<.live_file_input upload={@uploads.file} />
|
||||
</form>
|
||||
<div :for={entry <- @uploads.file.entries} class="delay-200 flex flex-col gap-1">
|
||||
<.file_entry name="Audio" entry={entry} on_clear={JS.push("clear_file", target: @myself)} />
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_event("change", params, socket) do
|
||||
value = %{
|
||||
data: Base.decode64!(params["data"]),
|
||||
height: params["height"],
|
||||
width: params["width"],
|
||||
format: socket.assigns.format
|
||||
}
|
||||
def handle_event("validate", %{}, socket) do
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
||||
send_update(LivebookWeb.Output.InputComponent,
|
||||
id: socket.assigns.input_component_id,
|
||||
event: :change,
|
||||
value: value
|
||||
)
|
||||
def handle_event("clear_file", %{"ref" => ref}, socket) do
|
||||
{:noreply, cancel_upload(socket, :file, ref)}
|
||||
end
|
||||
|
||||
defp handle_progress(:file, entry, socket) do
|
||||
if entry.done? do
|
||||
file_ref =
|
||||
consume_uploaded_entry(socket, entry, fn %{path: path} ->
|
||||
{:ok, file_ref} =
|
||||
LivebookWeb.SessionHelpers.register_input_file(
|
||||
socket.assigns.session_pid,
|
||||
path,
|
||||
socket.assigns.input_id,
|
||||
socket.assigns.local,
|
||||
socket.assigns.client_id
|
||||
)
|
||||
|
||||
{:ok, file_ref}
|
||||
end)
|
||||
|
||||
%{"height" => height, "width" => width} = entry.client_meta
|
||||
|
||||
value = %{
|
||||
file_ref: file_ref,
|
||||
height: height,
|
||||
width: width,
|
||||
format: socket.assigns.format
|
||||
}
|
||||
|
||||
send_update(LivebookWeb.Output.InputComponent,
|
||||
id: socket.assigns.input_component_id,
|
||||
event: :change,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
|
|
|||
|
|
@ -36,6 +36,10 @@ defmodule LivebookWeb.Output.InputComponent do
|
|||
width={@input.attrs.size && elem(@input.attrs.size, 1)}
|
||||
format={@input.attrs.format}
|
||||
fit={@input.attrs.fit}
|
||||
input_id={@input.id}
|
||||
session_pid={@session_pid}
|
||||
client_id={@client_id}
|
||||
local={@local}
|
||||
/>
|
||||
</div>
|
||||
"""
|
||||
|
|
@ -52,6 +56,10 @@ defmodule LivebookWeb.Output.InputComponent do
|
|||
value={@value}
|
||||
format={@input.attrs.format}
|
||||
sampling_rate={@input.attrs.sampling_rate}
|
||||
input_id={@input.id}
|
||||
session_pid={@session_pid}
|
||||
client_id={@client_id}
|
||||
local={@local}
|
||||
/>
|
||||
</div>
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -272,4 +272,42 @@ defmodule LivebookWeb.SessionHelpers do
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Generates a token for the given input.
|
||||
"""
|
||||
@spec generate_input_token(pid(), String.t()) :: String.t()
|
||||
def generate_input_token(live_view_pid, input_id) do
|
||||
Phoenix.Token.sign(LivebookWeb.Endpoint, "session-input", %{
|
||||
live_view_pid: live_view_pid,
|
||||
input_id: input_id
|
||||
})
|
||||
end
|
||||
|
||||
@doc """
|
||||
Verifies token from `generate_input_token/2` and extracts the encoded
|
||||
data.
|
||||
"""
|
||||
@spec verify_input_token!(String.t()) :: {pid(), String.t()}
|
||||
def verify_input_token!(token) do
|
||||
{:ok, %{live_view_pid: live_view_pid, input_id: input_id}} =
|
||||
Phoenix.Token.verify(LivebookWeb.Endpoint, "session-input", token)
|
||||
|
||||
{live_view_pid, input_id}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Registers an uploaded input file in session.
|
||||
"""
|
||||
@spec register_input_file(pid(), String.t(), String.t(), boolean(), String.t()) ::
|
||||
{:ok, Livebook.Runtime.file_ref()}
|
||||
def register_input_file(session_pid, path, input_id, local, client_id) do
|
||||
if local do
|
||||
key = "#{input_id}-#{client_id}"
|
||||
Livebook.Session.register_file(session_pid, path, key, linked_client_id: client_id)
|
||||
else
|
||||
key = "#{input_id}-global"
|
||||
Livebook.Session.register_file(session_pid, path, key)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1730,6 +1730,17 @@ defmodule LivebookWeb.SessionLive do
|
|||
{:noreply, socket}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call({:get_input_value, input_id}, _from, socket) do
|
||||
reply =
|
||||
case socket.private.data.input_infos do
|
||||
%{^input_id => %{value: value}} -> {:ok, socket.assigns.session.id, value}
|
||||
%{} -> :error
|
||||
end
|
||||
|
||||
{:reply, reply, socket}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:operation, operation}, socket) do
|
||||
{:noreply, handle_operation(socket, operation)}
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ defmodule LivebookWeb.Router do
|
|||
get "/sessions/:id/files/:name", SessionController, :show_file
|
||||
get "/sessions/:id/images/:name", SessionController, :show_image
|
||||
get "/sessions/:id/download/files/:name", SessionController, :download_file
|
||||
get "/sessions/audio-input/:token", SessionController, :show_input_audio
|
||||
get "/sessions/image-input/:token", SessionController, :show_input_image
|
||||
live "/sessions/:id/settings/custom-view", SessionLive, :custom_view_settings
|
||||
live "/sessions/:id/*path_parts", SessionLive, :catch_all
|
||||
end
|
||||
|
|
|
|||
4
mix.exs
4
mix.exs
|
|
@ -94,9 +94,9 @@ defmodule Livebook.MixProject do
|
|||
#
|
||||
defp deps do
|
||||
[
|
||||
{:phoenix, "~> 1.7.0"},
|
||||
{:phoenix, "~> 1.7.7"},
|
||||
{:phoenix_html, "~> 3.0"},
|
||||
# {:phoenix_live_view, "~> 0.19.0"},
|
||||
# {:phoenix_live_view, "~> 0.20.0"},
|
||||
{:phoenix_live_view, github: "phoenixframework/phoenix_live_view", override: true},
|
||||
{:phoenix_live_dashboard, "~> 0.8.0"},
|
||||
{:telemetry_metrics, "~> 0.4"},
|
||||
|
|
|
|||
10
mix.lock
10
mix.lock
|
|
@ -25,12 +25,12 @@
|
|||
"nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"},
|
||||
"nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"},
|
||||
"nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"},
|
||||
"phoenix": {:hex, :phoenix, "1.7.5", "3234bc87185e6a2103a15a3b1399f19775b093a6923c4064436e49cdab8ce5d2", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.1", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "5abad1789f06a3572ee5e5d5151993ed35b9e2711537904cc457a40229587979"},
|
||||
"phoenix": {:hex, :phoenix, "1.7.7", "4cc501d4d823015007ba3cdd9c41ecaaf2ffb619d6fb283199fa8ddba89191e0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "8966e15c395e5e37591b6ed0bd2ae7f48e961f0f60ac4c733f9566b519453085"},
|
||||
"phoenix_ecto": {:hex, :phoenix_ecto, "4.4.2", "b21bd01fdeffcfe2fab49e4942aa938b6d3e89e93a480d4aee58085560a0bc0d", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "70242edd4601d50b69273b057ecf7b684644c19ee750989fd555625ae4ce8f5d"},
|
||||
"phoenix_html": {:hex, :phoenix_html, "3.3.1", "4788757e804a30baac6b3fc9695bf5562465dd3f1da8eb8460ad5b404d9a2178", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "bed1906edd4906a15fd7b412b85b05e521e1f67c9a85418c55999277e553d0d3"},
|
||||
"phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.0", "0b3158b5b198aa444473c91d23d79f52fb077e807ffad80dacf88ce078fa8df2", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "87785a54474fed91a67a1227a741097eb1a42c2e49d3c0d098b588af65cd410d"},
|
||||
"phoenix_html": {:hex, :phoenix_html, "3.3.2", "d6ce982c6d8247d2fc0defe625255c721fb8d5f1942c5ac051f6177bffa5973f", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "44adaf8e667c1c20fb9d284b6b0fa8dc7946ce29e81ce621860aa7e96de9a11d"},
|
||||
"phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.8.2", "b9e33c950d1ed98494bfbde1c34c6e51c8a4214f3bea3f07ca9a510643ee1387", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:ecto_sqlite3_extras, "~> 1.1.7 or ~> 1.2.0", [hex: :ecto_sqlite3_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.19 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "67a598441b5f583d301a77e0298719f9654887d3d8bf14e80ff0b6acf887ef90"},
|
||||
"phoenix_live_reload": {:hex, :phoenix_live_reload, "1.4.1", "2aff698f5e47369decde4357ba91fc9c37c6487a512b41732818f2204a8ef1d3", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "9bffb834e7ddf08467fe54ae58b5785507aaba6255568ae22b4d46e2bb3615ab"},
|
||||
"phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view.git", "64e22999c2900e2f9266a030ca7a135a042f0645", []},
|
||||
"phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view.git", "9b267285cfce96072e9ecd8fbe065dc64a7ad004", []},
|
||||
"phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
|
||||
"phoenix_template": {:hex, :phoenix_template, "1.0.1", "85f79e3ad1b0180abb43f9725973e3b8c2c3354a87245f91431eec60553ed3ef", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "157dc078f6226334c91cb32c1865bf3911686f8bcd6bcff86736f6253e6993ee"},
|
||||
"plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"},
|
||||
|
|
@ -44,5 +44,5 @@
|
|||
"telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"},
|
||||
"thousand_island": {:hex, :thousand_island, "0.6.7", "3a91a7e362ca407036c6691e8a4f6e01ac8e901db3598875863a149279ac8571", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "541a5cb26b88adf8d8180b6b96a90f09566b4aad7a6b3608dcac969648cf6765"},
|
||||
"websock": {:hex, :websock, "0.5.1", "c496036ce95bc26d08ba086b2a827b212c67e7cabaa1c06473cd26b40ed8cf10", [:mix], [], "hexpm", "b9f785108b81cd457b06e5f5dabe5f65453d86a99118b2c0a515e1e296dc2d2c"},
|
||||
"websock_adapter": {:hex, :websock_adapter, "0.5.1", "292e6c56724e3457e808e525af0e9bcfa088cc7b9c798218e78658c7f9b85066", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "8e2e1544bfde5f9d0442f9cec2f5235398b224f75c9e06b60557debf64248ec1"},
|
||||
"websock_adapter": {:hex, :websock_adapter, "0.5.4", "7af8408e7ed9d56578539594d1ee7d8461e2dd5c3f57b0f2a5352d610ddde757", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d2c238c79c52cbe223fcdae22ca0bb5007a735b9e933870e241fce66afb4f4ab"},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
defmodule LivebookWeb.SessionControllerTest do
|
||||
use LivebookWeb.ConnCase, async: true
|
||||
|
||||
require Phoenix.LiveViewTest
|
||||
|
||||
alias Livebook.{Sessions, Session, Notebook, FileSystem}
|
||||
|
||||
describe "show_file" do
|
||||
|
|
@ -356,6 +358,100 @@ defmodule LivebookWeb.SessionControllerTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "show_audio_image" do
|
||||
@tag :tmp_dir
|
||||
test "given :wav input returns the audio binary", %{conn: conn, tmp_dir: tmp_dir} do
|
||||
{session, input_id} = start_session_with_audio_input(:wav, "wav content", tmp_dir)
|
||||
|
||||
{:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
|
||||
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
|
||||
|
||||
conn = get(conn, ~p"/sessions/audio-input/#{token}")
|
||||
|
||||
assert conn.status == 200
|
||||
assert conn.resp_body == "wav content"
|
||||
assert get_resp_header(conn, "accept-ranges") == ["bytes"]
|
||||
|
||||
Session.close(session.pid)
|
||||
end
|
||||
|
||||
@tag :tmp_dir
|
||||
test "given :wav input supports range requests", %{conn: conn, tmp_dir: tmp_dir} do
|
||||
{session, input_id} = start_session_with_audio_input(:wav, "wav content", tmp_dir)
|
||||
|
||||
{:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
|
||||
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
|
||||
|
||||
conn =
|
||||
conn
|
||||
|> put_req_header("range", "bytes=4-")
|
||||
|> get(~p"/sessions/audio-input/#{token}")
|
||||
|
||||
assert conn.status == 206
|
||||
assert conn.resp_body == "content"
|
||||
assert get_resp_header(conn, "content-range") == ["bytes 4-10/11"]
|
||||
|
||||
Session.close(session.pid)
|
||||
end
|
||||
|
||||
@tag :tmp_dir
|
||||
test "given :pcm_f32 input returns a WAV binary", %{conn: conn, tmp_dir: tmp_dir} do
|
||||
{session, input_id} = start_session_with_audio_input(:pcm_f32, "pcm content", tmp_dir)
|
||||
|
||||
{:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
|
||||
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
|
||||
|
||||
conn = get(conn, ~p"/sessions/audio-input/#{token}")
|
||||
|
||||
assert conn.status == 200
|
||||
assert <<_header::44-binary, "pcm content">> = conn.resp_body
|
||||
assert get_resp_header(conn, "accept-ranges") == ["bytes"]
|
||||
|
||||
Session.close(session.pid)
|
||||
end
|
||||
|
||||
@tag :tmp_dir
|
||||
test "given :pcm_f32 input supports range requests", %{conn: conn, tmp_dir: tmp_dir} do
|
||||
{session, input_id} = start_session_with_audio_input(:pcm_f32, "pcm content", tmp_dir)
|
||||
|
||||
{:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
|
||||
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
|
||||
|
||||
conn =
|
||||
conn
|
||||
|> put_req_header("range", "bytes=48-")
|
||||
|> get(~p"/sessions/audio-input/#{token}")
|
||||
|
||||
assert conn.status == 206
|
||||
assert conn.resp_body == "content"
|
||||
assert get_resp_header(conn, "content-range") == ["bytes 48-54/55"]
|
||||
|
||||
Session.close(session.pid)
|
||||
end
|
||||
end
|
||||
|
||||
describe "show_input_image" do
|
||||
@tag :tmp_dir
|
||||
test "returns the image binary", %{conn: conn, tmp_dir: tmp_dir} do
|
||||
{session, input_id} = start_session_with_image_input(:rgb, "rgb content", tmp_dir)
|
||||
|
||||
{:ok, view, _} = Phoenix.LiveViewTest.live(conn, ~p"/sessions/#{session.id}")
|
||||
|
||||
token = LivebookWeb.SessionHelpers.generate_input_token(view.pid, input_id)
|
||||
|
||||
conn = get(conn, ~p"/sessions/image-input/#{token}")
|
||||
|
||||
assert conn.status == 200
|
||||
assert conn.resp_body == "rgb content"
|
||||
|
||||
Session.close(session.pid)
|
||||
end
|
||||
end
|
||||
|
||||
defp start_session_and_request_asset(conn, notebook, hash) do
|
||||
{:ok, session} = Sessions.create_session(notebook: notebook)
|
||||
# We need runtime in place to actually copy the archive
|
||||
|
|
@ -387,4 +483,62 @@ defmodule LivebookWeb.SessionControllerTest do
|
|||
|
||||
%{notebook: notebook, hash: hash}
|
||||
end
|
||||
|
||||
defp start_session_with_audio_input(format, binary, tmp_dir) do
|
||||
input = %{
|
||||
type: :input,
|
||||
ref: "ref",
|
||||
id: "input1",
|
||||
destination: :noop,
|
||||
attrs: %{type: :audio, default: nil, label: "Audio", format: format, sampling_rate: 16_000}
|
||||
}
|
||||
|
||||
cell = %{Notebook.Cell.new(:code) | outputs: [{1, input}]}
|
||||
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [cell]}]}
|
||||
|
||||
{:ok, session} = Sessions.create_session(notebook: notebook)
|
||||
|
||||
source_path = Path.join(tmp_dir, "audio.bin")
|
||||
File.write!(source_path, binary)
|
||||
|
||||
{:ok, file_ref} = Session.register_file(session.pid, source_path, "key")
|
||||
|
||||
Session.set_input_value(session.pid, "input1", %{
|
||||
file_ref: file_ref,
|
||||
sampling_rate: 16_000,
|
||||
num_channels: 1,
|
||||
format: format
|
||||
})
|
||||
|
||||
{session, input.id}
|
||||
end
|
||||
|
||||
defp start_session_with_image_input(format, binary, tmp_dir) do
|
||||
input = %{
|
||||
type: :input,
|
||||
ref: "ref",
|
||||
id: "input1",
|
||||
destination: :noop,
|
||||
attrs: %{type: :image, default: nil, label: "Image", format: :rgb, size: nil, fit: :contain}
|
||||
}
|
||||
|
||||
cell = %{Notebook.Cell.new(:code) | outputs: [{1, input}]}
|
||||
notebook = %{Notebook.new() | sections: [%{Notebook.Section.new() | cells: [cell]}]}
|
||||
|
||||
{:ok, session} = Sessions.create_session(notebook: notebook)
|
||||
|
||||
source_path = Path.join(tmp_dir, "image.bin")
|
||||
File.write!(source_path, binary)
|
||||
|
||||
{:ok, file_ref} = Session.register_file(session.pid, source_path, "key")
|
||||
|
||||
Session.set_input_value(session.pid, "input1", %{
|
||||
file_ref: file_ref,
|
||||
height: 300,
|
||||
width: 300,
|
||||
format: format
|
||||
})
|
||||
|
||||
{session, input.id}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue