DPM/backend/static/voice-worklets/audio-playback.js

// @ts-check
/**
 * AudioWorkletProcessor that plays back Float32 mono samples received from
 * the main thread, upsampling whatever incoming rate the server uses
 * (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz).
 *
 * Lifecycle / messaging:
 *
 *   main -> worklet:
 *     { kind: "config", inputRate: 16000, muted: false }  one-shot at startup
 *     { kind: "audio", samples: Float32Array }             per chunk
 *     { kind: "clear" }                                    wipe queue (barge-in)
 *
 *   worklet -> main:
 *     { kind: "stats", queuedMs, played }                  every ~250 ms
 *     { kind: "underrun" }                                 every time the queue
 *                                                          runs dry mid-playback
 *
 * `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the
 * same audio, so the page mutes its own playback to avoid double audio. When
 * the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page
 * plays the backend audio itself.
 *
 * IMPORTANT: the read position only advances while audio is actually being
 * played (`_playing`). Advancing during idle would drift `_readIdx` far past
 * the next chunk's length and turn every read into NaN (silence) once audio
 * arrives.
 */

const STATS_INTERVAL_FRAMES = 12000;
const FADE_FRAMES = 32;

class AudioPlaybackProcessor extends AudioWorkletProcessor {
  constructor() {
    super();
    this._inputRate = 16000;
    this._stepRatio = this._inputRate / sampleRate;
    this._muted = false;
    this._queue = [];
    this._readIdx = 0;
    this._fracPos = 0;
    this._playing = false;
    this._lastSample = 0;
    this._framesSinceStats = 0;
    this._totalPlayed = 0;
    this._dbgReceived = 0;
    this._dbgOutPeak = 0;
    this._fadeIn = 0;
    this._fadeOut = 0;

    this.port.onmessage = (e) => {
      const data = e.data;
      if (!data || typeof data !== "object") return;
      switch (data.kind) {
        case "config":
          if (typeof data.inputRate === "number" && data.inputRate > 0) {
            this._inputRate = data.inputRate;
            this._stepRatio = this._inputRate / sampleRate;
          }
          if (typeof data.muted === "boolean") this._muted = data.muted;
          break;
        case "audio": {
          // Accept Float32Array, a transferred ArrayBuffer, or a plain
          // {buffer, byteOffset, length} descriptor: cross-realm structured
          // cloning can deliver the samples in any of these shapes.
          let arr = data.samples;
          let f32 = null;
          if (arr instanceof Float32Array) {
            f32 = arr;
          } else if (arr instanceof ArrayBuffer) {
            f32 = new Float32Array(arr);
          } else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) {
            f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2);
          }
          if (f32 && f32.length > 0) {
            this._queue.push(f32);
            this._dbgReceived += 1;
            if (!this._playing) {
              this._playing = true;
              this._fadeIn = FADE_FRAMES;
              this._fadeOut = 0;
            }
          }
          break;
        }
        case "clear":
          this._queue.length = 0;
          this._readIdx = 0;
          this._fracPos = 0;
          this._playing = false;
          this._lastSample = 0;
          this._fadeOut = FADE_FRAMES;
          break;
      }
    };
  }

  _queuedSamples() {
    let total = -this._readIdx;
    for (const buf of this._queue) total += buf.length;
    return Math.max(0, total);
  }

  /** Linear-interp read at the current fractional position. */
  _readInterpolated() {
    if (this._queue.length === 0) return null;
    const head = this._queue[0];
    const idx = this._readIdx;
    const frac = this._fracPos;

    let a = head[idx];
    let b;
    if (idx + 1 < head.length) {
      b = head[idx + 1];
    } else if (this._queue.length > 1) {
      b = this._queue[1][0];
    } else {
      b = a;
    }
    return a + (b - a) * frac;
  }

  /** Advance the read position by `stepRatio`; pop consumed buffers. */
  _advance() {
    this._fracPos += this._stepRatio;
    while (this._fracPos >= 1) {
      this._fracPos -= 1;
      this._readIdx += 1;
    }
    while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
      this._readIdx -= this._queue[0].length;
      this._queue.shift();
    }
  }

  process(_, outputs) {
    const channels = outputs[0];
    if (!channels || channels.length === 0) return true;
    const out = channels[0];
    const stereo = channels.length > 1 ? channels[1] : null;

    // Muted mode (Live Avatar WebRTC carries the audio): output silence.
    if (this._muted) {
      for (let i = 0; i < out.length; i++) {
        out[i] = 0;
        if (stereo) stereo[i] = 0;
      }
      this._framesSinceStats += out.length;
      return true;
    }

    for (let i = 0; i < out.length; i++) {
      let sample = 0;

      if (this._playing) {
        const v = this._readInterpolated();
        if (v === null) {
          // Underrun: ramp out cleanly to avoid clicks.
          sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES);
          this._lastSample = sample;
          if (Math.abs(sample) < 1e-4) {
            this._playing = false;
            this._lastSample = 0;
            this.port.postMessage({ kind: "underrun" });
          }
        } else {
          sample = v;
          this._lastSample = v;
          this._advance();
        }

        if (this._fadeIn > 0) {
          const gain = 1 - this._fadeIn / FADE_FRAMES;
          sample *= gain;
          this._fadeIn -= 1;
        }
        if (this._fadeOut > 0) {
          const gain = this._fadeOut / FADE_FRAMES;
          sample *= gain;
          this._fadeOut -= 1;
          if (this._fadeOut === 0) {
            this._playing = false;
            this._lastSample = 0;
          }
        }

        this._totalPlayed += 1;
        const abs = sample < 0 ? -sample : sample;
        if (abs > this._dbgOutPeak) this._dbgOutPeak = abs;
      }

      out[i] = sample;
      if (stereo) stereo[i] = sample;
    }

    this._framesSinceStats += out.length;
    if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) {
      this.port.postMessage({
        kind: "stats",
        queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000),
        played: this._totalPlayed,
        received: this._dbgReceived,
        outPeak: this._dbgOutPeak,
      });
      this._framesSinceStats = 0;
    }
    return true;
  }
}

registerProcessor("audio-playback", AudioPlaybackProcessor);