From 94faf14f07b96d81c228a1fd47664380ead788df Mon Sep 17 00:00:00 2001 From: Pine Date: Tue, 18 Aug 2026 08:31:53 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E5=8A=A9=E6=89=8B=E5=90=8D=E7=A7=B0=E4=B8=BA=E6=98=86=E5=B0=8F?= =?UTF-8?q?=E5=88=9B=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=AC=A2=E8=BF=8E=E8=AF=AD?= =?UTF-8?q?=E5=92=8C=E6=8F=90=E7=A4=BA=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/config.py | 2 +- src/components/AiChatPanel.jsx | 6 +- src/voice/s2s-ws-client.js | 70 +++------ src/voice/worklets/audio-playback.js | 211 +++++++++++++++++++++++++++ src/voice/worklets/mic-capture.js | 159 ++++++++++++++++++++ 5 files changed, 396 insertions(+), 52 deletions(-) create mode 100644 src/voice/worklets/audio-playback.js create mode 100644 src/voice/worklets/mic-capture.js diff --git a/backend/app/config.py b/backend/app/config.py index 559ac2b..f8c4942 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -90,7 +90,7 @@ class Settings: # 语音助手基础提示词（.env 设 DPM_S2S_INSTRUCTIONS 可覆盖；知识库内容由 rag 自动拼接） S2S_INSTRUCTIONS = _env( "DPM_S2S_INSTRUCTIONS", - "你是昆小园，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、" + "你是昆小创，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、" "query_companies（企业名录）、control_display（大屏控制）、get_time（时间）。" "涉及园区数据/企业/大屏控制时务必调用工具获取准确信息。请用简洁专业的中文回答，不超过三句话。", ) diff --git a/src/components/AiChatPanel.jsx b/src/components/AiChatPanel.jsx index 6255369..9bc23d5 100644 --- a/src/components/AiChatPanel.jsx +++ b/src/components/AiChatPanel.jsx @@ -33,7 +33,7 @@ const pick = (arr) => arr[Math.floor(Math.random() * arr.length)]; const VOICE_SAMPLES = QUICK_QUESTIONS; /* 默认欢迎语（新对话第一条消息，与初始一致） */ -const WELCOME_TEXT = '您好，我是「问问小园」，昆明市大学生创业园的专属 AI 助手（通义千问）。\n\n可以为您解答入驻申请、创业政策、创业担保贷款、OPC 概念、AI 工具等问题，也可以介绍园区入驻企业。试试右侧的预设问题吧。'; +const WELCOME_TEXT = '您好，我是昆小创，昆明市大学生创业园的专属 AI 助手。\n\n可以为您解答入驻申请、创业政策、创业担保贷款、OPC 概念、AI 工具等问题，也可以介绍园区入驻企业。试试右侧的预设问题吧。'; /* ---------- 消息 ---------- */ let msgId = 0; @@ -320,9 +320,9 @@ export default function AiChatPanel() {
问问小园 - 昆明市大学生创业园 · 通义千问 + 昆明市大学生创业园 · 昆小创
- 通义千问 + 昆小创 diff --git a/src/voice/s2s-ws-client.js b/src/voice/s2s-ws-client.js index 29c862a..5915f9f 100644 --- a/src/voice/s2s-ws-client.js +++ b/src/voice/s2s-ws-client.js @@ -1,4 +1,10 @@ // @ts-check +// AudioWorklet processor 源码（构建期内联，运行时直接 Blob → addModule）。 +// 打包环境（Tauri v2 + WebView2）无法可靠地从 tauri.localhost 自定义协议或 +// 跨域后端 URL 用 addModule 加载 worklet 模块；源码内联后零网络、零 CORS、零后端依赖。 +import micWorkletSrc from "./worklets/mic-capture.js?raw"; +import playbackWorkletSrc from "./worklets/audio-playback.js?raw"; + /** * Minimal WebSocket client for the Hugging Face speech-to-speech load balancer. * @@ -479,57 +485,25 @@ export class S2sWsRealtimeClient extends EventTarget { } // AudioWorklet 加载（WebView2/打包兼容）： - // 1) 先直接 addModule(http 地址) —— 浏览器/常规环境快路径 - // 2) 失败则 fetch 源码 → Blob URL → addModule —— 打包 WebView2 下 - // tauri.localhost 自定义协议与跨域均受限，Blob 同源最稳 - // 3) 多个候选基址（后端 http / 页面源 / 相对路径） - const candidates = [ - window.VOICE_WORKLETS_BASE, - `${window.location.origin}/voice-worklets/`, - "/voice-worklets/", - ].filter(Boolean); - let workletErr = null; - let workletOk = false; - - const addViaBlob = async (name) => { - for (const base of candidates) { - try { - const res = await fetch(base + name, { cache: "no-store" }); - if (!res.ok) continue; - const src = await res.text(); - const url = URL.createObjectURL(new Blob([src], { type: "application/javascript" })); - try { - await ctx.audioWorklet.addModule(url); - return true; - } finally { - URL.revokeObjectURL(url); - } - } catch (err) { - workletErr = err; - } - } - return false; - }; - - for (const base of candidates) { + // 源码已由 Vite `?raw` 内联进 bundle，运行时直接从内存字符串建 Blob → addModule。 + // 彻底摆脱运行时 fetch/网络/CORS/后端可达性依赖，任何 WebView2 都能加载。 + const addWorkletFromSrc = async (name, src) => { + const url = URL.createObjectURL(new Blob([src], { type: "text/javascript" })); try { - await ctx.audioWorklet.addModule(base + "mic-capture.js?v=3"); - await ctx.audioWorklet.addModule(base + "audio-playback.js?v=3"); - workletOk = true; - break; - } catch (err) { - workletErr = err; - console.warn("[ws] worklet 直接加载失败，改走 Blob 方案:", base, err); + await ctx.audioWorklet.addModule(url); + return true; + } finally { + URL.revokeObjectURL(url); } + }; + const workletOk = + (await addWorkletFromSrc("mic-capture", micWorkletSrc)) && + (await addWorkletFromSrc("audio-playback", playbackWorkletSrc)); + if (workletOk) { + console.info("[ws] AudioWorklet 已通过内联源码 Blob URL 加载（WebView2 兼容模式）"); + } else { + throw new Error("AudioWorklet 模块加载失败（内联源码 Blob addModule 失败）"); } - if (!workletOk) { - // 回退：fetch → Blob → addModule（WebView2 打包环境最稳） - const micOk = await addViaBlob("mic-capture.js?v=3"); - const playOk = await addViaBlob("audio-playback.js?v=3"); - workletOk = micOk && playOk; - if (workletOk) console.info("[ws] AudioWorklet 已通过 Blob URL 加载（WebView2 兼容模式）"); - } - if (!workletOk) throw workletErr || new Error("AudioWorklet 模块加载失败"); const captureNode = new AudioWorkletNode(ctx, "mic-capture", { numberOfInputs: 1, diff --git a/src/voice/worklets/audio-playback.js b/src/voice/worklets/audio-playback.js new file mode 100644 index 0000000..469b904 --- /dev/null +++ b/src/voice/worklets/audio-playback.js @@ -0,0 +1,211 @@ +// @ts-check +/** + * AudioWorkletProcessor that plays back Float32 mono samples received from + * the main thread, upsampling whatever incoming rate the server uses + * (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz). + * + * Lifecycle / messaging: + * + * main -> worklet: + * { kind: "config", inputRate: 16000, muted: false } one-shot at startup + * { kind: "audio", samples: Float32Array } per chunk + * { kind: "clear" } wipe queue (barge-in) + * + * worklet -> main: + * { kind: "stats", queuedMs, played } every ~250 ms + * { kind: "underrun" } every time the queue + * runs dry mid-playback + * + * `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the + * same audio, so the page mutes its own playback to avoid double audio. When + * the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page + * plays the backend audio itself. + * + * IMPORTANT: the read position only advances while audio is actually being + * played (`_playing`). Advancing during idle would drift `_readIdx` far past + * the next chunk's length and turn every read into NaN (silence) once audio + * arrives. + */ + +const STATS_INTERVAL_FRAMES = 12000; +const FADE_FRAMES = 32; + +class AudioPlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._inputRate = 16000; + this._stepRatio = this._inputRate / sampleRate; + this._muted = false; + this._queue = []; + this._readIdx = 0; + this._fracPos = 0; + this._playing = false; + this._lastSample = 0; + this._framesSinceStats = 0; + this._totalPlayed = 0; + this._dbgReceived = 0; + this._dbgOutPeak = 0; + this._fadeIn = 0; + this._fadeOut = 0; + + this.port.onmessage = (e) => { + const data = e.data; + if (!data || typeof data !== "object") return; + switch (data.kind) { + case "config": + if (typeof data.inputRate === "number" && data.inputRate > 0) { + this._inputRate = data.inputRate; + this._stepRatio = this._inputRate / sampleRate; + } + if (typeof data.muted === "boolean") this._muted = data.muted; + break; + case "audio": { + // Accept Float32Array, a transferred ArrayBuffer, or a plain + // {buffer, byteOffset, length} descriptor: cross-realm structured + // cloning can deliver the samples in any of these shapes. + let arr = data.samples; + let f32 = null; + if (arr instanceof Float32Array) { + f32 = arr; + } else if (arr instanceof ArrayBuffer) { + f32 = new Float32Array(arr); + } else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) { + f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2); + } + if (f32 && f32.length > 0) { + this._queue.push(f32); + this._dbgReceived += 1; + if (!this._playing) { + this._playing = true; + this._fadeIn = FADE_FRAMES; + this._fadeOut = 0; + } + } + break; + } + case "clear": + this._queue.length = 0; + this._readIdx = 0; + this._fracPos = 0; + this._playing = false; + this._lastSample = 0; + this._fadeOut = FADE_FRAMES; + break; + } + }; + } + + _queuedSamples() { + let total = -this._readIdx; + for (const buf of this._queue) total += buf.length; + return Math.max(0, total); + } + + /** Linear-interp read at the current fractional position. */ + _readInterpolated() { + if (this._queue.length === 0) return null; + const head = this._queue[0]; + const idx = this._readIdx; + const frac = this._fracPos; + + let a = head[idx]; + let b; + if (idx + 1 < head.length) { + b = head[idx + 1]; + } else if (this._queue.length > 1) { + b = this._queue[1][0]; + } else { + b = a; + } + return a + (b - a) * frac; + } + + /** Advance the read position by `stepRatio`; pop consumed buffers. */ + _advance() { + this._fracPos += this._stepRatio; + while (this._fracPos >= 1) { + this._fracPos -= 1; + this._readIdx += 1; + } + while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) { + this._readIdx -= this._queue[0].length; + this._queue.shift(); + } + } + + process(_, outputs) { + const channels = outputs[0]; + if (!channels || channels.length === 0) return true; + const out = channels[0]; + const stereo = channels.length > 1 ? channels[1] : null; + + // Muted mode (Live Avatar WebRTC carries the audio): output silence. + if (this._muted) { + for (let i = 0; i < out.length; i++) { + out[i] = 0; + if (stereo) stereo[i] = 0; + } + this._framesSinceStats += out.length; + return true; + } + + for (let i = 0; i < out.length; i++) { + let sample = 0; + + if (this._playing) { + const v = this._readInterpolated(); + if (v === null) { + // Underrun: ramp out cleanly to avoid clicks. + sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES); + this._lastSample = sample; + if (Math.abs(sample) < 1e-4) { + this._playing = false; + this._lastSample = 0; + this.port.postMessage({ kind: "underrun" }); + } + } else { + sample = v; + this._lastSample = v; + this._advance(); + } + + if (this._fadeIn > 0) { + const gain = 1 - this._fadeIn / FADE_FRAMES; + sample *= gain; + this._fadeIn -= 1; + } + if (this._fadeOut > 0) { + const gain = this._fadeOut / FADE_FRAMES; + sample *= gain; + this._fadeOut -= 1; + if (this._fadeOut === 0) { + this._playing = false; + this._lastSample = 0; + } + } + + this._totalPlayed += 1; + const abs = sample < 0 ? -sample : sample; + if (abs > this._dbgOutPeak) this._dbgOutPeak = abs; + } + + out[i] = sample; + if (stereo) stereo[i] = sample; + } + + this._framesSinceStats += out.length; + if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) { + this.port.postMessage({ + kind: "stats", + queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000), + played: this._totalPlayed, + received: this._dbgReceived, + outPeak: this._dbgOutPeak, + }); + this._framesSinceStats = 0; + } + return true; + } +} + +registerProcessor("audio-playback", AudioPlaybackProcessor); diff --git a/src/voice/worklets/mic-capture.js b/src/voice/worklets/mic-capture.js new file mode 100644 index 0000000..a48860b --- /dev/null +++ b/src/voice/worklets/mic-capture.js @@ -0,0 +1,159 @@ +// @ts-check +/** + * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz) + * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it + * back to the main thread in fixed-size chunks. + * + * The Hugging Face speech-to-speech WebSocket route expects the + * `input_audio_buffer.append` payload at 16 kHz PCM16 mono. + * + * Design notes: + * - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a + * cheap low-pass before decimating. Good enough for voice STT; we lose + * a tiny bit of >8 kHz content which the pipeline discards anyway. + * - Output frames are emitted at the cadence dictated by `chunkMs` + * (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime + * server batches incoming audio so the cadence is flexible; 20-100 ms + * is the sweet spot. + * - Float -> Int16 saturates to [-1, 1] before scaling. + * - Optional noise gate: per-chunk RMS decides open/closed against a + * threshold; the gain ramps (fast attack, hold, slow release) so word + * onsets aren't clipped and quiet tails don't click. The gate only + * affects the audio we SEND; the main-thread visualiser taps the raw + * mic separately. We post the chunk RMS up every frame so the Settings + * mic meter can show the live level against the threshold. + */ + +const TARGET_RATE = 16000; +const DEFAULT_CHUNK_MS = 40; +// Gate envelope timing (fixed; only the threshold is user-tunable). +const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive +const GATE_HOLD_MS = 250; // stay open this long after the level drops back under +const GATE_RELEASE_MS = 80; // then fade closed over this long (no click) + +class MicCaptureProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS; + this._inputRate = sampleRate; + this._ratio = this._inputRate / TARGET_RATE; + this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000); + this._scratch = new Float32Array(0); + this._decimated = new Float32Array(this._chunkSamples16k); + this._enabled = true; + + // Noise gate state. Disabled by default (pure passthrough). + this._gateEnabled = false; + this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open + this._gateGain = 1; // smoothed gain currently applied + this._holdRemaining = 0; // samples left before the gate may start closing + this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE)); + this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE)); + this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE); + + this.port.onmessage = (e) => { + const data = e.data; + if (data?.kind === "enable") this._enabled = !!data.value; + else if (data?.kind === "gate") { + this._gateEnabled = !!data.enabled; + // dB -> linear amplitude. When off, threshold 0 keeps the gate open. + this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0; + } + }; + } + + /** + * Append `incoming` to the internal scratch buffer, then emit as many + * full output chunks as we have material for. + * @param {Float32Array} incoming + */ + _ingest(incoming) { + if (incoming.length === 0) return; + const next = new Float32Array(this._scratch.length + incoming.length); + next.set(this._scratch, 0); + next.set(incoming, this._scratch.length); + this._scratch = next; + this._maybeEmit(); + } + + _maybeEmit() { + const r = this._ratio; + const n = this._chunkSamples16k; + const needIn = Math.ceil(n * r); + const dec = this._decimated; + while (this._scratch.length >= needIn) { + // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter. + let sumSq = 0; + if (Math.abs(r - 3) < 1e-6) { + // 48 kHz -> 16 kHz fast path with boxcar lowpass. + for (let i = 0; i < n; i++) { + const idx = i * 3; + const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3; + dec[i] = s; + sumSq += s * s; + } + } else { + // Generic path: linear interpolation. Slower but works at any rate + // (e.g. some Windows boxes report sampleRate=44100). + for (let i = 0; i < n; i++) { + const srcPos = i * r; + const idx = Math.floor(srcPos); + const frac = srcPos - idx; + const a = this._scratch[idx]; + const b = this._scratch[idx + 1] ?? a; + const s = a + (b - a) * frac; + dec[i] = s; + sumSq += s * s; + } + } + const rms = Math.sqrt(sumSq / n); + + // 2. Decide the gate target for this chunk, then ramp sample-by-sample. + let target = 1; + if (this._gateEnabled) { + if (rms >= this._thresholdLin) { + this._holdRemaining = this._holdSamples; // re-arm the hold + } else if (this._holdRemaining > 0) { + this._holdRemaining -= n; // coasting through the hold window + } else { + target = 0; + } + } + + // 3. Apply the (smoothed) gain and pack to Int16. + const out = new Int16Array(n); + let gain = this._gateGain; + for (let i = 0; i < n; i++) { + const coef = target > gain ? this._attackCoef : this._releaseCoef; + gain = target + (gain - target) * coef; + const s = dec[i] * gain; + const clamped = s < -1 ? -1 : s > 1 ? 1 : s; + out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + } + this._gateGain = gain; + + // Shift the scratch buffer to keep only the trailing unused samples. + const consumed = Math.floor(n * r); + this._scratch = this._scratch.slice(consumed); + + // Live input level for the Settings meter (raw RMS, pre-gate). + this.port.postMessage({ kind: "level", rms }); + + if (this._enabled) { + this.port.postMessage(out.buffer, [out.buffer]); + } + // When disabled (mic muted) we silently consume input so the worklet + // stays alive and the buffer never grows unbounded. + } + } + + process(inputs) { + const input = inputs[0]; + if (!input || input.length === 0 || !input[0]) return true; + const mono = input[0]; + if (mono.length > 0) this._ingest(mono); + return true; + } +} + +registerProcessor("mic-capture", MicCaptureProcessor);