From 7e6b14c44afad3a638f9f6fb4de30b157adf3489 Mon Sep 17 00:00:00 2001 From: Pine Date: Tue, 18 Aug 2026 08:56:17 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E9=99=8D=E7=BA=A7=E5=88=B0=20ScriptProcessorNode=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=BC=BA=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/voice/s2s-ws-client.js | 238 +++++++++++++++++++++++++------------ 1 file changed, 165 insertions(+), 73 deletions(-) diff --git a/src/voice/s2s-ws-client.js b/src/voice/s2s-ws-client.js index cfb3faa..1a387cc 100644 --- a/src/voice/s2s-ws-client.js +++ b/src/voice/s2s-ws-client.js @@ -469,8 +469,8 @@ export class S2sWsRealtimeClient extends EventTarget { // Prefer a context the caller already created + resumed inside the tap // gesture (required on iOS). Fall back to creating one here for callers // that don't (desktop is lenient about the gesture timing). - // Most desktops give us 48 kHz, mobiles can give 44.1/24/16 kHz; the - // capture worklet handles any rate (linear interp fallback). + // Most desktops give us 48 kHz, mobiles can give 44.1/24/16 kHz; both the + // worklet and the ScriptProcessor fallback handle any rate. const ctx = this.options.audioContext ?? new AudioContext({ latencyHint: "interactive" }); this._ctx = ctx; @@ -484,62 +484,85 @@ export class S2sWsRealtimeClient extends EventTarget { } } - // AudioWorklet 加载（WebView2/打包兼容）——多策略 + 详细日志： - // 源码已由 Vite `?raw` 内联进 bundle。不同 WebView2 版本对 - // blob: / data: / http: 三种 URL 的 addModule 支持不一致，逐一尝试， - // 并打印每次失败的真实 name/message，便于定位真正原因。 - // 策略1 内联源码 → Blob URL（无网络，多数环境最稳） - // 策略2 内联源码 → data: URL（部分 WebView2 拒绝 blob: 但接受 data:） - // 策略3 后端真实 HTTP（IP 可达 + CORS *，直连 addModule） - const attempts = []; - const tryModule = async (label, url) => { + // ---- 音频管线：优先 AudioWorklet；环境不支持则降级 ScriptProcessorNode ---- + // 展播机 WebView2 上 addModule 整体 AbortError（机制坏，非代码问题）。此时 + // 绝不能 throw —— 改用 ScriptProcessorNode（老 API，所有 WebView 都支持， + // 无需加载任何模块），保证麦克风采集与播放一定能跑通。 + this._useScriptProcessor = false; + this._playbackQueue = []; + let audioWorkletOk = false; + try { + audioWorkletOk = + (await this._initAudioWorklet(ctx, "mic-capture", micWorkletSrc)) && + (await this._initAudioWorklet(ctx, "audio-playback", playbackWorkletSrc)); + } catch (err) { + console.warn("[ws] AudioWorklet 初始化异常，将降级 ScriptProcessor:", err); + audioWorkletOk = false; + } + if (!audioWorkletOk) { + console.warn("[ws] AudioWorklet 不可用 → 降级 ScriptProcessorNode 管线（兼容模式）"); + this._useScriptProcessor = true; + } + + // 麦克风源 + 输入分析器（两条路径共用；直接并行 tap 原始信号，不经 worklet） + const micSrc = ctx.createMediaStreamSource(this.options.micStream); + this._micSrc = micSrc; + const micAnalyser = ctx.createAnalyser(); + micAnalyser.fftSize = VIS_FFT_SIZE; + micAnalyser.smoothingTimeConstant = 0; + micSrc.connect(micAnalyser); + this._micAnalyser = micAnalyser; + + // 输出分析器（共用；置于播放节点与扬声器之间） + const outAnalyser = ctx.createAnalyser(); + outAnalyser.fftSize = VIS_FFT_SIZE; + outAnalyser.smoothingTimeConstant = 0.3; + this._outAnalyser = outAnalyser; + + if (this._useScriptProcessor) { + this._setupScriptProcessor(ctx, micSrc, outAnalyser); + } else { + this._setupWorkletNodes(ctx, micSrc, outAnalyser); + } + + this._visualiser = new OrbVisualiser(micAnalyser, outAnalyser, () => this._aiSpeaking); + this._visualiser.start(); + } + + /** 尝试用内联源码把 worklet 加载进 ctx.audioWorklet（多策略：blob/data/http）。 */ + async _initAudioWorklet(ctx, name, src) { + if (!ctx.audioWorklet || typeof ctx.audioWorklet.addModule !== "function") return false; + const tryModule = async (url) => { try { await ctx.audioWorklet.addModule(url); return true; } catch (err) { - const detail = `${err?.name ?? ""}: ${err?.message ?? String(err)}`; - attempts.push(`${label} -> ${detail}`); - console.warn(`[ws] worklet addModule 失败（${label}）:`, err); + console.warn(`[ws] addModule(${name}) 失败:`, err?.name, err?.message); return false; } }; - - const addWorkletFromSrc = async (name, src) => { - // 策略1：Blob URL - const blobUrl = URL.createObjectURL(new Blob([src], { type: "text/javascript" })); - let ok = await tryModule(`${name}@blob`, blobUrl); - URL.revokeObjectURL(blobUrl); + // 策略1：Blob URL（内联源码，无网络） + const blobUrl = URL.createObjectURL(new Blob([src], { type: "text/javascript" })); + let ok = await tryModule(blobUrl); + URL.revokeObjectURL(blobUrl); + if (ok) return true; + // 策略2：data: URL + try { + const b64 = btoa(unescape(encodeURIComponent(src))); + ok = await tryModule(`data:text/javascript;base64,${b64}`); + if (ok) return true; + } catch { /* 忽略编码错误 */ } + // 策略3：后端真实 HTTP（IP 可达 + CORS *） + const httpBase = window.VOICE_WORKLETS_BASE; + if (httpBase) { + ok = await tryModule(`${httpBase}${name}.js`); if (ok) return true; - - // 策略2：data: URL（base64，UTF-8 安全） - try { - const b64 = btoa(unescape(encodeURIComponent(src))); - ok = await tryModule(`${name}@data`, `data:text/javascript;base64,${b64}`); - if (ok) return true; - } catch (err) { - attempts.push(`${name}@data-encode -> ${err?.message ?? String(err)}`); - } - - // 策略3：后端 HTTP（IP 可达 + CORS *） - const httpBase = window.VOICE_WORKLETS_BASE; - if (httpBase) { - ok = await tryModule(`${name}@http`, `${httpBase}${name}.js`); - if (ok) return true; - } - return false; - }; - - const workletOk = - (await addWorkletFromSrc("mic-capture", micWorkletSrc)) && - (await addWorkletFromSrc("audio-playback", playbackWorkletSrc)); - if (workletOk) { - console.info("[ws] AudioWorklet 加载成功（WebView2 兼容模式）"); - } else { - throw new Error( - `AudioWorklet 模块加载失败。尝试详情：${attempts.join(" | ") || "无"}`, - ); } + return false; + } + /** 正常路径：用 AudioWorkletNode 做采集与播放。 */ + _setupWorkletNodes(ctx, micSrc, outAnalyser) { const captureNode = new AudioWorkletNode(ctx, "mic-capture", { numberOfInputs: 1, numberOfOutputs: 0, @@ -556,19 +579,8 @@ export class S2sWsRealtimeClient extends EventTarget { }; // Push the initial gate config now that the worklet exists. captureNode.port.postMessage({ kind: "gate", ...this._noiseGate }); - this._captureNode = captureNode; - - const micSrc = ctx.createMediaStreamSource(this.options.micStream); micSrc.connect(captureNode); - this._micSrc = micSrc; - - // Mic analyser: tap the mic in parallel with the worklet so we get the - // raw (un-resampled, un-clipped) signal for the visualiser. - const micAnalyser = ctx.createAnalyser(); - micAnalyser.fftSize = VIS_FFT_SIZE; - micAnalyser.smoothingTimeConstant = 0; - micSrc.connect(micAnalyser); - this._micAnalyser = micAnalyser; + this._captureNode = captureNode; const playbackNode = new AudioWorkletNode(ctx, "audio-playback", { numberOfInputs: 0, @@ -577,18 +589,88 @@ export class S2sWsRealtimeClient extends EventTarget { }); playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE, muted: this._playbackMuted === true }); playbackNode.port.onmessage = (e) => this._onPlaybackMessage(e.data); - - // Output analyser sits between the playback worklet and the speakers. - const outAnalyser = ctx.createAnalyser(); - outAnalyser.fftSize = VIS_FFT_SIZE; - outAnalyser.smoothingTimeConstant = 0.3; playbackNode.connect(outAnalyser); outAnalyser.connect(ctx.destination); - this._outAnalyser = outAnalyser; this._playbackNode = playbackNode; + } - this._visualiser = new OrbVisualiser(micAnalyser, outAnalyser, () => this._aiSpeaking); - this._visualiser.start(); + /** 降级路径：ScriptProcessorNode 重采样采集 + 播放（兼容所有 WebView，无模块加载）。 */ + _setupScriptProcessor(ctx, micSrc, outAnalyser) { + const buf = 4096; + const targetRate = OUTPUT_SAMPLE_RATE; // 16000 + const chunkSamples = Math.round((targetRate * MIC_CHUNK_MS) / 1000); // 640 (40ms) + const ratio = ctx.sampleRate / targetRate; + + // ---- 麦克风采集：累积输入 → 重采样到 16k → 打包 Int16 分块 ---- + const cap = ctx.createScriptProcessor(buf, 1, 1); + const sink = ctx.createGain(); + sink.gain.value = 0; + sink.connect(ctx.destination); // ScriptProcessor 须连到 destination 才会触发回调 + let scratch = new Float32Array(0); + cap.onaudioprocess = (e) => { + const input = e.inputBuffer.getChannelData(0); + const next = new Float32Array(scratch.length + input.length); + next.set(scratch, 0); + next.set(input, scratch.length); + scratch = next; + const need = Math.ceil(chunkSamples * ratio); + while (scratch.length >= need) { + const dec = new Float32Array(chunkSamples); + let sumSq = 0; + for (let i = 0; i < chunkSamples; i++) { + const idx = Math.floor(i * ratio); + const s = scratch[idx]; + dec[i] = s; + sumSq += s * s; + } + const out = new Int16Array(chunkSamples); + for (let i = 0; i < chunkSamples; i++) { + const s = dec[i] < -1 ? -1 : dec[i] > 1 ? 1 : dec[i]; + out[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + this.dispatchEvent( + new CustomEvent("input-level", { detail: { rms: Math.sqrt(sumSq / chunkSamples) } }), + ); + this._onMicChunk(out.buffer); + scratch = scratch.slice(Math.floor(chunkSamples * ratio)); + } + }; + micSrc.connect(cap); + cap.connect(sink); + this._captureNode = cap; + + // ---- 播放：从队列读取 Float32 16k 样本 → 升采样到 ctx.sampleRate 输出 ---- + const pb = ctx.createScriptProcessor(buf, 0, 1); + const stepRatio = targetRate / ctx.sampleRate; + let frac = 0; + let readPos = 0; + pb.onaudioprocess = (e) => { + const out = e.outputBuffer.getChannelData(0); + if (this._playbackMuted === true) { + for (let i = 0; i < out.length; i++) out[i] = 0; + return; + } + const q = this._playbackQueue; + for (let i = 0; i < out.length; i++) { + let s = 0; + if (q.length) { + const head = q[0]; + const a = head[readPos]; + let b; + if (readPos + 1 < head.length) b = head[readPos + 1]; + else if (q.length > 1) b = q[1][0]; + else b = a; + s = a + (b - a) * frac; + frac += stepRatio; + while (frac >= 1) { frac -= 1; readPos += 1; } + while (q.length && readPos >= q[0].length) { readPos -= q[0].length; q.shift(); } + } + out[i] = s; + } + }; + pb.connect(outAnalyser); + outAnalyser.connect(ctx.destination); + this._playbackNode = pb; } /** @param {string} connectUrl */ @@ -960,10 +1042,16 @@ export class S2sWsRealtimeClient extends EventTarget { if (a > peak) peak = a; } this._dbgPeak = Math.max(this._dbgPeak || 0, peak); - // Clone (no transfer list): structured cloning across the main->worklet - // boundary is more robust than transferring the buffer — some browser - // versions mis-shape the received object and the worklet would drop it. - this._playbackNode.port.postMessage({ kind: "audio", samples }); + if (this._useScriptProcessor) { + // 降级路径：把 16k Float32 样本推入主线程播放队列，由 ScriptProcessor 读取。 + if (!this._playbackQueue) this._playbackQueue = []; + this._playbackQueue.push(samples); + } else { + // Clone (no transfer list): structured cloning across the main->worklet + // boundary is more robust than transferring the buffer — some browser + // versions mis-shape the received object and the worklet would drop it. + this._playbackNode.port.postMessage({ kind: "audio", samples }); + } } /** @param {CloseEvent} ev */ @@ -1154,7 +1242,11 @@ export class S2sWsRealtimeClient extends EventTarget { setPlaybackMuted(muted) { this._playbackMuted = muted; if (this._playbackNode) { - this._playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE, muted }); + if (this._useScriptProcessor) { + // 降级路径：无需通知；onaudioprocess 内读取 _playbackMuted。 + } else { + this._playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE, muted }); + } } }