livebook/assets/js/hooks/audio_input.js

import { getAttributeOrThrow, parseInteger } from "../lib/attribute";
import { base64ToBuffer, bufferToBase64 } from "../lib/utils";

const dropClasses = ["bg-yellow-100", "border-yellow-300"];

/**
 * A hook for client-preprocessed audio input.
 *
 * ## Configuration
 *
 *   * `data-id` - a unique id
 *
 *   * `data-phx-target` - the component to send the `"change"` event to
 *
 *   * `data-format` - the desired audio format
 *
 *   * `data-sampling-rate` - the audio sampling rate for
 *
 *   * `data-endianness` - the server endianness, either `"little"` or `"big"`
 *
 */
const AudioInput = {
  mounted() {
    this.props = this.getProps();

    this.inputEl = this.el.querySelector(`[data-input]`);
    this.audioEl = this.el.querySelector(`[data-preview]`);
    this.uploadButton = this.el.querySelector(`[data-btn-upload]`);
    this.recordButton = this.el.querySelector(`[data-btn-record]`);
    this.stopButton = this.el.querySelector(`[data-btn-stop]`);
    this.cancelButton = this.el.querySelector(`[data-btn-cancel]`);

    this.mediaRecorder = null;

    // Render updated value
    this.handleEvent(
      `audio_input_change:${this.props.id}`,
      ({ audio_info: audioInfo }) => {
        if (audioInfo) {
          this.updatePreview({
            data: this.decodeAudio(base64ToBuffer(audioInfo.data)),
            numChannels: audioInfo.num_channels,
            samplingRate: audioInfo.sampling_rate,
          });
        } else {
          this.clearPreview();
        }
      }
    );

    // File selection

    this.uploadButton.addEventListener("click", (event) => {
      this.inputEl.click();
    });

    this.inputEl.addEventListener("change", (event) => {
      const [file] = event.target.files;
      file && this.loadFile(file);
    });

    // Drag and drop

    this.el.addEventListener("dragover", (event) => {
      event.stopPropagation();
      event.preventDefault();
      event.dataTransfer.dropEffect = "copy";
    });

    this.el.addEventListener("drop", (event) => {
      event.stopPropagation();
      event.preventDefault();
      const [file] = event.dataTransfer.files;
      file && !this.isRecording() && this.loadFile(file);
    });

    this.el.addEventListener("dragenter", (event) => {
      this.el.classList.add(...dropClasses);
    });

    this.el.addEventListener("dragleave", (event) => {
      if (!this.el.contains(event.relatedTarget)) {
        this.el.classList.remove(...dropClasses);
      }
    });

    this.el.addEventListener("drop", (event) => {
      this.el.classList.remove(...dropClasses);
    });

    // Microphone capture

    this.recordButton.addEventListener("click", (event) => {
      this.startRecording();
    });

    this.stopButton.addEventListener("click", (event) => {
      this.stopRecording();
    });

    this.cancelButton.addEventListener("click", (event) => {
      this.stopRecording(false);
    });
  },

  updated() {
    this.props = this.getProps();
  },

  getProps() {
    return {
      id: getAttributeOrThrow(this.el, "data-id"),
      phxTarget: getAttributeOrThrow(this.el, "data-phx-target", parseInteger),
      samplingRate: getAttributeOrThrow(
        this.el,
        "data-sampling-rate",
        parseInteger
      ),
      endianness: getAttributeOrThrow(this.el, "data-endianness"),
      format: getAttributeOrThrow(this.el, "data-format"),
    };
  },

  startRecording() {
    this.audioEl.classList.add("hidden");
    this.uploadButton.classList.add("hidden");
    this.recordButton.classList.add("hidden");
    this.stopButton.classList.remove("hidden");
    this.cancelButton.classList.remove("hidden");

    this.audioChunks = [];

    navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
      this.mediaRecorder = new MediaRecorder(stream);

      this.mediaRecorder.addEventListener("dataavailable", (event) => {
        this.audioChunks.push(event.data);
      });

      this.mediaRecorder.start();
    });
  },

  stopRecording(load = true) {
    this.audioEl.classList.remove("hidden");
    this.uploadButton.classList.remove("hidden");
    this.recordButton.classList.remove("hidden");
    this.stopButton.classList.add("hidden");
    this.cancelButton.classList.add("hidden");

    if (load) {
      this.mediaRecorder.addEventListener("stop", (event) => {
        const audioBlob = new Blob(this.audioChunks);

        audioBlob.arrayBuffer().then((buffer) => {
          this.loadEncodedAudio(buffer);
        });
      });
    }

    this.mediaRecorder.stop();
  },

  isRecording() {
    return this.mediaRecorder && this.mediaRecorder.state === "recording";
  },

  loadFile(file) {
    const reader = new FileReader();

    reader.onload = (readerEvent) => {
      this.loadEncodedAudio(readerEvent.target.result);
    };

    reader.readAsArrayBuffer(file);
  },

  loadEncodedAudio(buffer) {
    const context = new AudioContext({ sampleRate: this.props.samplingRate });

    context.decodeAudioData(buffer, (audioBuffer) => {
      const audioInfo = audioBufferToAudioInfo(audioBuffer);
      this.pushAudio(audioInfo);
    });
  },

  updatePreview(audioInfo) {
    const oldUrl = this.audioEl.src;
    const blob = audioInfoToWavBlob(audioInfo);
    this.audioEl.src = URL.createObjectURL(blob);
    oldUrl && URL.revokeObjectURL(oldUrl);
  },

  clearPreview() {
    const oldUrl = this.audioEl.src;
    this.audioEl.src = "";
    oldUrl && URL.revokeObjectURL(oldUrl);
  },

  pushAudio(audioInfo) {
    this.pushEventTo(this.props.phxTarget, "change", {
      data: bufferToBase64(this.encodeAudio(audioInfo)),
      num_channels: audioInfo.numChannels,
      sampling_rate: audioInfo.samplingRate,
    });
  },

  encodeAudio(audioInfo) {
    if (this.props.format === "pcm_f32") {
      return this.fixEndianness32(audioInfo.data);
    } else if (this.props.format === "wav") {
      return encodeWavData(
        audioInfo.data,
        audioInfo.numChannels,
        audioInfo.samplingRate
      );
    }
  },

  decodeAudio(buffer) {
    if (this.props.format === "pcm_f32") {
      return this.fixEndianness32(buffer);
    } else if (this.props.format === "wav") {
      return decodeWavData(buffer);
    }
  },

  fixEndianness32(buffer) {
    if (getEndianness() === this.props.endianness) {
      return buffer;
    }

    // If the server uses different endianness, we swap bytes accordingly
    for (const i = 0; i < buffer.byteLength / 4; i++) {
      const b1 = buffer[i];
      const b2 = buffer[i + 1];
      const b3 = buffer[i + 2];
      const b4 = buffer[i + 3];
      buffer[i] = b4;
      buffer[i + 1] = b3;
      buffer[i + 2] = b2;
      buffer[i + 3] = b1;
    }

    return buffer;
  },
};

function audioBufferToAudioInfo(audioBuffer) {
  const numChannels = audioBuffer.numberOfChannels;
  const samplingRate = audioBuffer.sampleRate;
  const length = audioBuffer.length;

  const size = 4 * numChannels * length;
  const buffer = new ArrayBuffer(size);

  const pcmArray = new Float32Array(buffer);

  for (let channelIdx = 0; channelIdx < numChannels; channelIdx++) {
    const channelArray = audioBuffer.getChannelData(channelIdx);

    for (let i = 0; i < channelArray.length; i++) {
      pcmArray[numChannels * i + channelIdx] = channelArray[i];
    }
  }

  return { data: pcmArray.buffer, numChannels, samplingRate };
}

function audioInfoToWavBlob({ data, numChannels, samplingRate }) {
  const wavBytes = encodeWavData(data, numChannels, samplingRate);
  return new Blob([wavBytes], { type: "audio/wav" });
}

// See http://soundfile.sapp.org/doc/WaveFormat
function encodeWavData(buffer, numChannels, samplingRate) {
  const HEADER_SIZE = 44;

  const wavBuffer = new ArrayBuffer(HEADER_SIZE + buffer.byteLength);
  const view = new DataView(wavBuffer);

  const numFrames = buffer.byteLength / 4;
  const bytesPerSample = 4;

  const blockAlign = numChannels * bytesPerSample;
  const byteRate = samplingRate * blockAlign;
  const dataSize = numFrames * blockAlign;

  let offset = 0;

  function writeUint32Big(int) {
    view.setUint32(offset, int, false);
    offset += 4;
  }

  function writeUint32(int) {
    view.setUint32(offset, int, true);
    offset += 4;
  }

  function writeUint16(int) {
    view.setUint16(offset, int, true);
    offset += 2;
  }

  function writeFloat32(int) {
    view.setFloat32(offset, int, true);
    offset += 4;
  }

  writeUint32Big(0x52494646);
  writeUint32(36 + dataSize);
  writeUint32Big(0x57415645);

  writeUint32Big(0x666d7420);
  writeUint32(16);
  writeUint16(3); // 3 represents 32-bit float PCM
  writeUint16(numChannels);
  writeUint32(samplingRate);
  writeUint32(byteRate);
  writeUint16(blockAlign);
  writeUint16(bytesPerSample * 8);

  writeUint32Big(0x64617461);
  writeUint32(dataSize);

  const array = new Float32Array(buffer);

  for (let i = 0; i < array.length; i++) {
    writeFloat32(array[i]);
  }

  return wavBuffer;
}

// We assume the exact same format as above, since we only need to
// decode data we encoded previously
function decodeWavData(buffer) {
  const HEADER_SIZE = 44;

  const pcmBuffer = new ArrayBuffer(buffer.byteLength - HEADER_SIZE);
  const pcmArray = new Float32Array(pcmBuffer);

  const view = new DataView(buffer);

  for (let i = 0; i < pcmArray.length; i++) {
    const offset = HEADER_SIZE + i * 4;
    pcmArray[i] = view.getFloat32(offset, true);
  }

  return pcmBuffer;
}

function getEndianness() {
  const buffer = new ArrayBuffer(2);
  const int16Array = new Uint16Array(buffer);
  const int8Array = new Uint8Array(buffer);

  int16Array[0] = 1;

  if (int8Array[0] === 1) {
    return "little";
  } else {
    return "big";
  }
}

export default AudioInput;