diff --git a/backend/static/voice-worklets/audio-playback.js b/backend/static/voice-worklets/audio-playback.js new file mode 100644 index 0000000..469b904 --- /dev/null +++ b/backend/static/voice-worklets/audio-playback.js @@ -0,0 +1,211 @@ +// @ts-check +/** + * AudioWorkletProcessor that plays back Float32 mono samples received from + * the main thread, upsampling whatever incoming rate the server uses + * (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz). + * + * Lifecycle / messaging: + * + * main -> worklet: + * { kind: "config", inputRate: 16000, muted: false } one-shot at startup + * { kind: "audio", samples: Float32Array } per chunk + * { kind: "clear" } wipe queue (barge-in) + * + * worklet -> main: + * { kind: "stats", queuedMs, played } every ~250 ms + * { kind: "underrun" } every time the queue + * runs dry mid-playback + * + * `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the + * same audio, so the page mutes its own playback to avoid double audio. When + * the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page + * plays the backend audio itself. + * + * IMPORTANT: the read position only advances while audio is actually being + * played (`_playing`). Advancing during idle would drift `_readIdx` far past + * the next chunk's length and turn every read into NaN (silence) once audio + * arrives. + */ + +const STATS_INTERVAL_FRAMES = 12000; +const FADE_FRAMES = 32; + +class AudioPlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._inputRate = 16000; + this._stepRatio = this._inputRate / sampleRate; + this._muted = false; + this._queue = []; + this._readIdx = 0; + this._fracPos = 0; + this._playing = false; + this._lastSample = 0; + this._framesSinceStats = 0; + this._totalPlayed = 0; + this._dbgReceived = 0; + this._dbgOutPeak = 0; + this._fadeIn = 0; + this._fadeOut = 0; + + this.port.onmessage = (e) => { + const data = e.data; + if (!data || typeof data !== "object") return; + switch (data.kind) { + case "config": + if (typeof data.inputRate === "number" && data.inputRate > 0) { + this._inputRate = data.inputRate; + this._stepRatio = this._inputRate / sampleRate; + } + if (typeof data.muted === "boolean") this._muted = data.muted; + break; + case "audio": { + // Accept Float32Array, a transferred ArrayBuffer, or a plain + // {buffer, byteOffset, length} descriptor: cross-realm structured + // cloning can deliver the samples in any of these shapes. + let arr = data.samples; + let f32 = null; + if (arr instanceof Float32Array) { + f32 = arr; + } else if (arr instanceof ArrayBuffer) { + f32 = new Float32Array(arr); + } else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) { + f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2); + } + if (f32 && f32.length > 0) { + this._queue.push(f32); + this._dbgReceived += 1; + if (!this._playing) { + this._playing = true; + this._fadeIn = FADE_FRAMES; + this._fadeOut = 0; + } + } + break; + } + case "clear": + this._queue.length = 0; + this._readIdx = 0; + this._fracPos = 0; + this._playing = false; + this._lastSample = 0; + this._fadeOut = FADE_FRAMES; + break; + } + }; + } + + _queuedSamples() { + let total = -this._readIdx; + for (const buf of this._queue) total += buf.length; + return Math.max(0, total); + } + + /** Linear-interp read at the current fractional position. */ + _readInterpolated() { + if (this._queue.length === 0) return null; + const head = this._queue[0]; + const idx = this._readIdx; + const frac = this._fracPos; + + let a = head[idx]; + let b; + if (idx + 1 < head.length) { + b = head[idx + 1]; + } else if (this._queue.length > 1) { + b = this._queue[1][0]; + } else { + b = a; + } + return a + (b - a) * frac; + } + + /** Advance the read position by `stepRatio`; pop consumed buffers. */ + _advance() { + this._fracPos += this._stepRatio; + while (this._fracPos >= 1) { + this._fracPos -= 1; + this._readIdx += 1; + } + while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) { + this._readIdx -= this._queue[0].length; + this._queue.shift(); + } + } + + process(_, outputs) { + const channels = outputs[0]; + if (!channels || channels.length === 0) return true; + const out = channels[0]; + const stereo = channels.length > 1 ? channels[1] : null; + + // Muted mode (Live Avatar WebRTC carries the audio): output silence. + if (this._muted) { + for (let i = 0; i < out.length; i++) { + out[i] = 0; + if (stereo) stereo[i] = 0; + } + this._framesSinceStats += out.length; + return true; + } + + for (let i = 0; i < out.length; i++) { + let sample = 0; + + if (this._playing) { + const v = this._readInterpolated(); + if (v === null) { + // Underrun: ramp out cleanly to avoid clicks. + sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES); + this._lastSample = sample; + if (Math.abs(sample) < 1e-4) { + this._playing = false; + this._lastSample = 0; + this.port.postMessage({ kind: "underrun" }); + } + } else { + sample = v; + this._lastSample = v; + this._advance(); + } + + if (this._fadeIn > 0) { + const gain = 1 - this._fadeIn / FADE_FRAMES; + sample *= gain; + this._fadeIn -= 1; + } + if (this._fadeOut > 0) { + const gain = this._fadeOut / FADE_FRAMES; + sample *= gain; + this._fadeOut -= 1; + if (this._fadeOut === 0) { + this._playing = false; + this._lastSample = 0; + } + } + + this._totalPlayed += 1; + const abs = sample < 0 ? -sample : sample; + if (abs > this._dbgOutPeak) this._dbgOutPeak = abs; + } + + out[i] = sample; + if (stereo) stereo[i] = sample; + } + + this._framesSinceStats += out.length; + if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) { + this.port.postMessage({ + kind: "stats", + queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000), + played: this._totalPlayed, + received: this._dbgReceived, + outPeak: this._dbgOutPeak, + }); + this._framesSinceStats = 0; + } + return true; + } +} + +registerProcessor("audio-playback", AudioPlaybackProcessor); diff --git a/backend/static/voice-worklets/mic-capture.js b/backend/static/voice-worklets/mic-capture.js new file mode 100644 index 0000000..a48860b --- /dev/null +++ b/backend/static/voice-worklets/mic-capture.js @@ -0,0 +1,159 @@ +// @ts-check +/** + * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz) + * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it + * back to the main thread in fixed-size chunks. + * + * The Hugging Face speech-to-speech WebSocket route expects the + * `input_audio_buffer.append` payload at 16 kHz PCM16 mono. + * + * Design notes: + * - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a + * cheap low-pass before decimating. Good enough for voice STT; we lose + * a tiny bit of >8 kHz content which the pipeline discards anyway. + * - Output frames are emitted at the cadence dictated by `chunkMs` + * (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime + * server batches incoming audio so the cadence is flexible; 20-100 ms + * is the sweet spot. + * - Float -> Int16 saturates to [-1, 1] before scaling. + * - Optional noise gate: per-chunk RMS decides open/closed against a + * threshold; the gain ramps (fast attack, hold, slow release) so word + * onsets aren't clipped and quiet tails don't click. The gate only + * affects the audio we SEND; the main-thread visualiser taps the raw + * mic separately. We post the chunk RMS up every frame so the Settings + * mic meter can show the live level against the threshold. + */ + +const TARGET_RATE = 16000; +const DEFAULT_CHUNK_MS = 40; +// Gate envelope timing (fixed; only the threshold is user-tunable). +const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive +const GATE_HOLD_MS = 250; // stay open this long after the level drops back under +const GATE_RELEASE_MS = 80; // then fade closed over this long (no click) + +class MicCaptureProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS; + this._inputRate = sampleRate; + this._ratio = this._inputRate / TARGET_RATE; + this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000); + this._scratch = new Float32Array(0); + this._decimated = new Float32Array(this._chunkSamples16k); + this._enabled = true; + + // Noise gate state. Disabled by default (pure passthrough). + this._gateEnabled = false; + this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open + this._gateGain = 1; // smoothed gain currently applied + this._holdRemaining = 0; // samples left before the gate may start closing + this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE)); + this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE)); + this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE); + + this.port.onmessage = (e) => { + const data = e.data; + if (data?.kind === "enable") this._enabled = !!data.value; + else if (data?.kind === "gate") { + this._gateEnabled = !!data.enabled; + // dB -> linear amplitude. When off, threshold 0 keeps the gate open. + this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0; + } + }; + } + + /** + * Append `incoming` to the internal scratch buffer, then emit as many + * full output chunks as we have material for. + * @param {Float32Array} incoming + */ + _ingest(incoming) { + if (incoming.length === 0) return; + const next = new Float32Array(this._scratch.length + incoming.length); + next.set(this._scratch, 0); + next.set(incoming, this._scratch.length); + this._scratch = next; + this._maybeEmit(); + } + + _maybeEmit() { + const r = this._ratio; + const n = this._chunkSamples16k; + const needIn = Math.ceil(n * r); + const dec = this._decimated; + while (this._scratch.length >= needIn) { + // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter. + let sumSq = 0; + if (Math.abs(r - 3) < 1e-6) { + // 48 kHz -> 16 kHz fast path with boxcar lowpass. + for (let i = 0; i < n; i++) { + const idx = i * 3; + const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3; + dec[i] = s; + sumSq += s * s; + } + } else { + // Generic path: linear interpolation. Slower but works at any rate + // (e.g. some Windows boxes report sampleRate=44100). + for (let i = 0; i < n; i++) { + const srcPos = i * r; + const idx = Math.floor(srcPos); + const frac = srcPos - idx; + const a = this._scratch[idx]; + const b = this._scratch[idx + 1] ?? a; + const s = a + (b - a) * frac; + dec[i] = s; + sumSq += s * s; + } + } + const rms = Math.sqrt(sumSq / n); + + // 2. Decide the gate target for this chunk, then ramp sample-by-sample. + let target = 1; + if (this._gateEnabled) { + if (rms >= this._thresholdLin) { + this._holdRemaining = this._holdSamples; // re-arm the hold + } else if (this._holdRemaining > 0) { + this._holdRemaining -= n; // coasting through the hold window + } else { + target = 0; + } + } + + // 3. Apply the (smoothed) gain and pack to Int16. + const out = new Int16Array(n); + let gain = this._gateGain; + for (let i = 0; i < n; i++) { + const coef = target > gain ? this._attackCoef : this._releaseCoef; + gain = target + (gain - target) * coef; + const s = dec[i] * gain; + const clamped = s < -1 ? -1 : s > 1 ? 1 : s; + out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + } + this._gateGain = gain; + + // Shift the scratch buffer to keep only the trailing unused samples. + const consumed = Math.floor(n * r); + this._scratch = this._scratch.slice(consumed); + + // Live input level for the Settings meter (raw RMS, pre-gate). + this.port.postMessage({ kind: "level", rms }); + + if (this._enabled) { + this.port.postMessage(out.buffer, [out.buffer]); + } + // When disabled (mic muted) we silently consume input so the worklet + // stays alive and the buffer never grows unbounded. + } + } + + process(inputs) { + const input = inputs[0]; + if (!input || input.length === 0 || !input[0]) return true; + const mono = input[0]; + if (mono.length > 0) this._ingest(mono); + return true; + } +} + +registerProcessor("mic-capture", MicCaptureProcessor); diff --git a/src/main.jsx b/src/main.jsx index 9218d66..5b58b95 100644 --- a/src/main.jsx +++ b/src/main.jsx @@ -31,6 +31,12 @@ async function bootstrapRuntimeConfig() { /* 浏览器开发环境或未注册命令:跳过 */ } + // AudioWorklet 兼容:打包环境(WebView2)无法从 tauri.localhost 自定义协议加载 + // worklet 模块,统一改从后端真实 HTTP 服务加载(backend/static/voice-worklets/) + if (!window.VOICE_WORKLETS_BASE) { + window.VOICE_WORKLETS_BASE = `${getApiBase()}/static/voice-worklets/`; + } + // 2) 后端 /api/config(MQTT 地址/账号;API 基址以前端实际可达地址为准) // 优先级保护:若构建期已用 .env.local 烘焙了 VITE_MQTT_URL, // 则后端 /api/config 不再覆盖(避免后端默认 localhost 冲掉正确地址)。 diff --git a/src/voice/s2s-ws-client.js b/src/voice/s2s-ws-client.js index a0d938e..d254c32 100644 --- a/src/voice/s2s-ws-client.js +++ b/src/voice/s2s-ws-client.js @@ -481,9 +481,27 @@ export class S2sWsRealtimeClient extends EventTarget { // The worklets live at the repo root, one level up from this module. // ?v=2 busts the browser cache so the fixed (unmuted) playback worklet // always loads instead of a stale muted copy. - const base = window.VOICE_WORKLETS_BASE || "/voice-worklets/"; - await ctx.audioWorklet.addModule(base + "mic-capture.js?v=3"); - await ctx.audioWorklet.addModule(base + "audio-playback.js?v=3"); + // 多基址回退:打包环境优先后端 http(tauri.localhost 无法加载 worklet 模块), + // 浏览器开发回退页面源 / 相对路径。 + const candidates = [ + window.VOICE_WORKLETS_BASE, + `${window.location.origin}/voice-worklets/`, + "/voice-worklets/", + ].filter(Boolean); + let workletErr = null; + let workletOk = false; + for (const base of candidates) { + try { + await ctx.audioWorklet.addModule(base + "mic-capture.js?v=3"); + await ctx.audioWorklet.addModule(base + "audio-playback.js?v=3"); + workletOk = true; + break; + } catch (err) { + workletErr = err; + console.warn("[ws] worklet 加载失败,尝试备用地址:", base, err); + } + } + if (!workletOk) throw workletErr || new Error("AudioWorklet 模块加载失败"); const captureNode = new AudioWorkletNode(ctx, "mic-capture", { numberOfInputs: 1,