feat: 添加音频工作模块，支持从后端加载音频处理器

backend/static/voice-worklets/audio-playback.js

@@ -0,0 +1,211 @@
+// @ts-check
+/**
+ * AudioWorkletProcessor that plays back Float32 mono samples received from
+ * the main thread, upsampling whatever incoming rate the server uses
+ * (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz).
+ *
+ * Lifecycle / messaging:
+ *
+ *   main -> worklet:
+ *     { kind: "config", inputRate: 16000, muted: false }  one-shot at startup
+ *     { kind: "audio", samples: Float32Array }             per chunk
+ *     { kind: "clear" }                                    wipe queue (barge-in)
+ *
+ *   worklet -> main:
+ *     { kind: "stats", queuedMs, played }                  every ~250 ms
+ *     { kind: "underrun" }                                 every time the queue
+ *                                                          runs dry mid-playback
+ *
+ * `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the
+ * same audio, so the page mutes its own playback to avoid double audio. When
+ * the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page
+ * plays the backend audio itself.
+ *
+ * IMPORTANT: the read position only advances while audio is actually being
+ * played (`_playing`). Advancing during idle would drift `_readIdx` far past
+ * the next chunk's length and turn every read into NaN (silence) once audio
+ * arrives.
+ */
+
+const STATS_INTERVAL_FRAMES = 12000;
+const FADE_FRAMES = 32;
+
+class AudioPlaybackProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this._inputRate = 16000;
+    this._stepRatio = this._inputRate / sampleRate;
+    this._muted = false;
+    this._queue = [];
+    this._readIdx = 0;
+    this._fracPos = 0;
+    this._playing = false;
+    this._lastSample = 0;
+    this._framesSinceStats = 0;
+    this._totalPlayed = 0;
+    this._dbgReceived = 0;
+    this._dbgOutPeak = 0;
+    this._fadeIn = 0;
+    this._fadeOut = 0;
+
+    this.port.onmessage = (e) => {
+      const data = e.data;
+      if (!data || typeof data !== "object") return;
+      switch (data.kind) {
+        case "config":
+          if (typeof data.inputRate === "number" && data.inputRate > 0) {
+            this._inputRate = data.inputRate;
+            this._stepRatio = this._inputRate / sampleRate;
+          }
+          if (typeof data.muted === "boolean") this._muted = data.muted;
+          break;
+        case "audio": {
+          // Accept Float32Array, a transferred ArrayBuffer, or a plain
+          // {buffer, byteOffset, length} descriptor: cross-realm structured
+          // cloning can deliver the samples in any of these shapes.
+          let arr = data.samples;
+          let f32 = null;
+          if (arr instanceof Float32Array) {
+            f32 = arr;
+          } else if (arr instanceof ArrayBuffer) {
+            f32 = new Float32Array(arr);
+          } else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) {
+            f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2);
+          }
+          if (f32 && f32.length > 0) {
+            this._queue.push(f32);
+            this._dbgReceived += 1;
+            if (!this._playing) {
+              this._playing = true;
+              this._fadeIn = FADE_FRAMES;
+              this._fadeOut = 0;
+            }
+          }
+          break;
+        }
+        case "clear":
+          this._queue.length = 0;
+          this._readIdx = 0;
+          this._fracPos = 0;
+          this._playing = false;
+          this._lastSample = 0;
+          this._fadeOut = FADE_FRAMES;
+          break;
+      }
+    };
+  }
+
+  _queuedSamples() {
+    let total = -this._readIdx;
+    for (const buf of this._queue) total += buf.length;
+    return Math.max(0, total);
+  }
+
+  /** Linear-interp read at the current fractional position. */
+  _readInterpolated() {
+    if (this._queue.length === 0) return null;
+    const head = this._queue[0];
+    const idx = this._readIdx;
+    const frac = this._fracPos;
+
+    let a = head[idx];
+    let b;
+    if (idx + 1 < head.length) {
+      b = head[idx + 1];
+    } else if (this._queue.length > 1) {
+      b = this._queue[1][0];
+    } else {
+      b = a;
+    }
+    return a + (b - a) * frac;
+  }
+
+  /** Advance the read position by `stepRatio`; pop consumed buffers. */
+  _advance() {
+    this._fracPos += this._stepRatio;
+    while (this._fracPos >= 1) {
+      this._fracPos -= 1;
+      this._readIdx += 1;
+    }
+    while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
+      this._readIdx -= this._queue[0].length;
+      this._queue.shift();
+    }
+  }
+
+  process(_, outputs) {
+    const channels = outputs[0];
+    if (!channels || channels.length === 0) return true;
+    const out = channels[0];
+    const stereo = channels.length > 1 ? channels[1] : null;
+
+    // Muted mode (Live Avatar WebRTC carries the audio): output silence.
+    if (this._muted) {
+      for (let i = 0; i < out.length; i++) {
+        out[i] = 0;
+        if (stereo) stereo[i] = 0;
+      }
+      this._framesSinceStats += out.length;
+      return true;
+    }
+
+    for (let i = 0; i < out.length; i++) {
+      let sample = 0;
+
+      if (this._playing) {
+        const v = this._readInterpolated();
+        if (v === null) {
+          // Underrun: ramp out cleanly to avoid clicks.
+          sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES);
+          this._lastSample = sample;
+          if (Math.abs(sample) < 1e-4) {
+            this._playing = false;
+            this._lastSample = 0;
+            this.port.postMessage({ kind: "underrun" });
+          }
+        } else {
+          sample = v;
+          this._lastSample = v;
+          this._advance();
+        }
+
+        if (this._fadeIn > 0) {
+          const gain = 1 - this._fadeIn / FADE_FRAMES;
+          sample *= gain;
+          this._fadeIn -= 1;
+        }
+        if (this._fadeOut > 0) {
+          const gain = this._fadeOut / FADE_FRAMES;
+          sample *= gain;
+          this._fadeOut -= 1;
+          if (this._fadeOut === 0) {
+            this._playing = false;
+            this._lastSample = 0;
+          }
+        }
+
+        this._totalPlayed += 1;
+        const abs = sample < 0 ? -sample : sample;
+        if (abs > this._dbgOutPeak) this._dbgOutPeak = abs;
+      }
+
+      out[i] = sample;
+      if (stereo) stereo[i] = sample;
+    }
+
+    this._framesSinceStats += out.length;
+    if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) {
+      this.port.postMessage({
+        kind: "stats",
+        queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000),
+        played: this._totalPlayed,
+        received: this._dbgReceived,
+        outPeak: this._dbgOutPeak,
+      });
+      this._framesSinceStats = 0;
+    }
+    return true;
+  }
+}
+
+registerProcessor("audio-playback", AudioPlaybackProcessor);

backend/static/voice-worklets/mic-capture.js

@@ -0,0 +1,159 @@
+// @ts-check
+/**
+ * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz)
+ * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it
+ * back to the main thread in fixed-size chunks.
+ *
+ * The Hugging Face speech-to-speech WebSocket route expects the
+ * `input_audio_buffer.append` payload at 16 kHz PCM16 mono.
+ *
+ * Design notes:
+ *   - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a
+ *     cheap low-pass before decimating. Good enough for voice STT; we lose
+ *     a tiny bit of >8 kHz content which the pipeline discards anyway.
+ *   - Output frames are emitted at the cadence dictated by `chunkMs`
+ *     (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime
+ *     server batches incoming audio so the cadence is flexible; 20-100 ms
+ *     is the sweet spot.
+ *   - Float -> Int16 saturates to [-1, 1] before scaling.
+ *   - Optional noise gate: per-chunk RMS decides open/closed against a
+ *     threshold; the gain ramps (fast attack, hold, slow release) so word
+ *     onsets aren't clipped and quiet tails don't click. The gate only
+ *     affects the audio we SEND; the main-thread visualiser taps the raw
+ *     mic separately. We post the chunk RMS up every frame so the Settings
+ *     mic meter can show the live level against the threshold.
+ */
+
+const TARGET_RATE = 16000;
+const DEFAULT_CHUNK_MS = 40;
+// Gate envelope timing (fixed; only the threshold is user-tunable).
+const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive
+const GATE_HOLD_MS = 250; // stay open this long after the level drops back under
+const GATE_RELEASE_MS = 80; // then fade closed over this long (no click)
+
+class MicCaptureProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS;
+    this._inputRate = sampleRate;
+    this._ratio = this._inputRate / TARGET_RATE;
+    this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000);
+    this._scratch = new Float32Array(0);
+    this._decimated = new Float32Array(this._chunkSamples16k);
+    this._enabled = true;
+
+    // Noise gate state. Disabled by default (pure passthrough).
+    this._gateEnabled = false;
+    this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open
+    this._gateGain = 1; // smoothed gain currently applied
+    this._holdRemaining = 0; // samples left before the gate may start closing
+    this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE));
+    this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE));
+    this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE);
+
+    this.port.onmessage = (e) => {
+      const data = e.data;
+      if (data?.kind === "enable") this._enabled = !!data.value;
+      else if (data?.kind === "gate") {
+        this._gateEnabled = !!data.enabled;
+        // dB -> linear amplitude. When off, threshold 0 keeps the gate open.
+        this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0;
+      }
+    };
+  }
+
+  /**
+   * Append `incoming` to the internal scratch buffer, then emit as many
+   * full output chunks as we have material for.
+   * @param {Float32Array} incoming
+   */
+  _ingest(incoming) {
+    if (incoming.length === 0) return;
+    const next = new Float32Array(this._scratch.length + incoming.length);
+    next.set(this._scratch, 0);
+    next.set(incoming, this._scratch.length);
+    this._scratch = next;
+    this._maybeEmit();
+  }
+
+  _maybeEmit() {
+    const r = this._ratio;
+    const n = this._chunkSamples16k;
+    const needIn = Math.ceil(n * r);
+    const dec = this._decimated;
+    while (this._scratch.length >= needIn) {
+      // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter.
+      let sumSq = 0;
+      if (Math.abs(r - 3) < 1e-6) {
+        // 48 kHz -> 16 kHz fast path with boxcar lowpass.
+        for (let i = 0; i < n; i++) {
+          const idx = i * 3;
+          const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3;
+          dec[i] = s;
+          sumSq += s * s;
+        }
+      } else {
+        // Generic path: linear interpolation. Slower but works at any rate
+        // (e.g. some Windows boxes report sampleRate=44100).
+        for (let i = 0; i < n; i++) {
+          const srcPos = i * r;
+          const idx = Math.floor(srcPos);
+          const frac = srcPos - idx;
+          const a = this._scratch[idx];
+          const b = this._scratch[idx + 1] ?? a;
+          const s = a + (b - a) * frac;
+          dec[i] = s;
+          sumSq += s * s;
+        }
+      }
+      const rms = Math.sqrt(sumSq / n);
+
+      // 2. Decide the gate target for this chunk, then ramp sample-by-sample.
+      let target = 1;
+      if (this._gateEnabled) {
+        if (rms >= this._thresholdLin) {
+          this._holdRemaining = this._holdSamples; // re-arm the hold
+        } else if (this._holdRemaining > 0) {
+          this._holdRemaining -= n; // coasting through the hold window
+        } else {
+          target = 0;
+        }
+      }
+
+      // 3. Apply the (smoothed) gain and pack to Int16.
+      const out = new Int16Array(n);
+      let gain = this._gateGain;
+      for (let i = 0; i < n; i++) {
+        const coef = target > gain ? this._attackCoef : this._releaseCoef;
+        gain = target + (gain - target) * coef;
+        const s = dec[i] * gain;
+        const clamped = s < -1 ? -1 : s > 1 ? 1 : s;
+        out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
+      }
+      this._gateGain = gain;
+
+      // Shift the scratch buffer to keep only the trailing unused samples.
+      const consumed = Math.floor(n * r);
+      this._scratch = this._scratch.slice(consumed);
+
+      // Live input level for the Settings meter (raw RMS, pre-gate).
+      this.port.postMessage({ kind: "level", rms });
+
+      if (this._enabled) {
+        this.port.postMessage(out.buffer, [out.buffer]);
+      }
+      // When disabled (mic muted) we silently consume input so the worklet
+      // stays alive and the buffer never grows unbounded.
+    }
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || input.length === 0 || !input[0]) return true;
+    const mono = input[0];
+    if (mono.length > 0) this._ingest(mono);
+    return true;
+  }
+}
+
+registerProcessor("mic-capture", MicCaptureProcessor);

src/main.jsx

@@ -31,6 +31,12 @@ async function bootstrapRuntimeConfig() {
     /* 浏览器开发环境或未注册命令：跳过 */
   }
 
+  // AudioWorklet 兼容：打包环境（WebView2）无法从 tauri.localhost 自定义协议加载
+  // worklet 模块，统一改从后端真实 HTTP 服务加载（backend/static/voice-worklets/）
+  if (!window.VOICE_WORKLETS_BASE) {
+    window.VOICE_WORKLETS_BASE = `${getApiBase()}/static/voice-worklets/`;
+  }
+
   // 2) 后端 /api/config（MQTT 地址/账号；API 基址以前端实际可达地址为准）
   //    优先级保护：若构建期已用 .env.local 烘焙了 VITE_MQTT_URL，
   //    则后端 /api/config 不再覆盖（避免后端默认 localhost 冲掉正确地址）。

src/voice/s2s-ws-client.js

@@ -481,9 +481,27 @@ export class S2sWsRealtimeClient extends EventTarget {
     // The worklets live at the repo root, one level up from this module.
     // ?v=2 busts the browser cache so the fixed (unmuted) playback worklet
     // always loads instead of a stale muted copy.
-    const base = window.VOICE_WORKLETS_BASE || "/voice-worklets/";
+    // 多基址回退：打包环境优先后端 http（tauri.localhost 无法加载 worklet 模块），
-    await ctx.audioWorklet.addModule(base + "mic-capture.js?v=3");
+    // 浏览器开发回退页面源 / 相对路径。
-    await ctx.audioWorklet.addModule(base + "audio-playback.js?v=3");
+    const candidates = [
+      window.VOICE_WORKLETS_BASE,
+      `${window.location.origin}/voice-worklets/`,
+      "/voice-worklets/",
+    ].filter(Boolean);
+    let workletErr = null;
+    let workletOk = false;
+    for (const base of candidates) {
+      try {
+        await ctx.audioWorklet.addModule(base + "mic-capture.js?v=3");
+        await ctx.audioWorklet.addModule(base + "audio-playback.js?v=3");
+        workletOk = true;
+        break;
+      } catch (err) {
+        workletErr = err;
+        console.warn("[ws] worklet 加载失败，尝试备用地址:", base, err);
+      }
+    }
+    if (!workletOk) throw workletErr || new Error("AudioWorklet 模块加载失败");
 
     const captureNode = new AudioWorkletNode(ctx, "mic-capture", {
       numberOfInputs: 1,