feat(voice): 实时语音对话页（React 完全复刻原版 UI）
- /voice 页面：中间圆球 + 左右按钮 + 状态图标 + 右下角气泡（进场/淡出/阶梯字号/上限8）+ 噪声门弧线电平，结构类名与原版一致 - s2s WebSocket 客户端 + worklet 音频（mic-capture/audio-playback）+ orb 可视化 - 摄像头实时预览（默认开启）+ 视觉识别状态条（后端 YOLO 推理） - 底部园区概览条（复用 ParkOverviewStrip），orb-wrap 视窗居中 - 页眉复用 ScreenLayout；左右键导航加入 /voice - 文档：docs/voice-integration-plan.md
This commit is contained in:
Pine
@@ -0,0 +1,211 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* AudioWorkletProcessor that plays back Float32 mono samples received from
|
||||
* the main thread, upsampling whatever incoming rate the server uses
|
||||
* (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz).
|
||||
*
|
||||
* Lifecycle / messaging:
|
||||
*
|
||||
* main -> worklet:
|
||||
* { kind: "config", inputRate: 16000, muted: false } one-shot at startup
|
||||
* { kind: "audio", samples: Float32Array } per chunk
|
||||
* { kind: "clear" } wipe queue (barge-in)
|
||||
*
|
||||
* worklet -> main:
|
||||
* { kind: "stats", queuedMs, played } every ~250 ms
|
||||
* { kind: "underrun" } every time the queue
|
||||
* runs dry mid-playback
|
||||
*
|
||||
* `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the
|
||||
* same audio, so the page mutes its own playback to avoid double audio. When
|
||||
* the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page
|
||||
* plays the backend audio itself.
|
||||
*
|
||||
* IMPORTANT: the read position only advances while audio is actually being
|
||||
* played (`_playing`). Advancing during idle would drift `_readIdx` far past
|
||||
* the next chunk's length and turn every read into NaN (silence) once audio
|
||||
* arrives.
|
||||
*/
|
||||
|
||||
const STATS_INTERVAL_FRAMES = 12000;
|
||||
const FADE_FRAMES = 32;
|
||||
|
||||
class AudioPlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this._inputRate = 16000;
|
||||
this._stepRatio = this._inputRate / sampleRate;
|
||||
this._muted = false;
|
||||
this._queue = [];
|
||||
this._readIdx = 0;
|
||||
this._fracPos = 0;
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this._framesSinceStats = 0;
|
||||
this._totalPlayed = 0;
|
||||
this._dbgReceived = 0;
|
||||
this._dbgOutPeak = 0;
|
||||
this._fadeIn = 0;
|
||||
this._fadeOut = 0;
|
||||
|
||||
this.port.onmessage = (e) => {
|
||||
const data = e.data;
|
||||
if (!data || typeof data !== "object") return;
|
||||
switch (data.kind) {
|
||||
case "config":
|
||||
if (typeof data.inputRate === "number" && data.inputRate > 0) {
|
||||
this._inputRate = data.inputRate;
|
||||
this._stepRatio = this._inputRate / sampleRate;
|
||||
}
|
||||
if (typeof data.muted === "boolean") this._muted = data.muted;
|
||||
break;
|
||||
case "audio": {
|
||||
// Accept Float32Array, a transferred ArrayBuffer, or a plain
|
||||
// {buffer, byteOffset, length} descriptor: cross-realm structured
|
||||
// cloning can deliver the samples in any of these shapes.
|
||||
let arr = data.samples;
|
||||
let f32 = null;
|
||||
if (arr instanceof Float32Array) {
|
||||
f32 = arr;
|
||||
} else if (arr instanceof ArrayBuffer) {
|
||||
f32 = new Float32Array(arr);
|
||||
} else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) {
|
||||
f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2);
|
||||
}
|
||||
if (f32 && f32.length > 0) {
|
||||
this._queue.push(f32);
|
||||
this._dbgReceived += 1;
|
||||
if (!this._playing) {
|
||||
this._playing = true;
|
||||
this._fadeIn = FADE_FRAMES;
|
||||
this._fadeOut = 0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "clear":
|
||||
this._queue.length = 0;
|
||||
this._readIdx = 0;
|
||||
this._fracPos = 0;
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this._fadeOut = FADE_FRAMES;
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
_queuedSamples() {
|
||||
let total = -this._readIdx;
|
||||
for (const buf of this._queue) total += buf.length;
|
||||
return Math.max(0, total);
|
||||
}
|
||||
|
||||
/** Linear-interp read at the current fractional position. */
|
||||
_readInterpolated() {
|
||||
if (this._queue.length === 0) return null;
|
||||
const head = this._queue[0];
|
||||
const idx = this._readIdx;
|
||||
const frac = this._fracPos;
|
||||
|
||||
let a = head[idx];
|
||||
let b;
|
||||
if (idx + 1 < head.length) {
|
||||
b = head[idx + 1];
|
||||
} else if (this._queue.length > 1) {
|
||||
b = this._queue[1][0];
|
||||
} else {
|
||||
b = a;
|
||||
}
|
||||
return a + (b - a) * frac;
|
||||
}
|
||||
|
||||
/** Advance the read position by `stepRatio`; pop consumed buffers. */
|
||||
_advance() {
|
||||
this._fracPos += this._stepRatio;
|
||||
while (this._fracPos >= 1) {
|
||||
this._fracPos -= 1;
|
||||
this._readIdx += 1;
|
||||
}
|
||||
while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
|
||||
this._readIdx -= this._queue[0].length;
|
||||
this._queue.shift();
|
||||
}
|
||||
}
|
||||
|
||||
process(_, outputs) {
|
||||
const channels = outputs[0];
|
||||
if (!channels || channels.length === 0) return true;
|
||||
const out = channels[0];
|
||||
const stereo = channels.length > 1 ? channels[1] : null;
|
||||
|
||||
// Muted mode (Live Avatar WebRTC carries the audio): output silence.
|
||||
if (this._muted) {
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
out[i] = 0;
|
||||
if (stereo) stereo[i] = 0;
|
||||
}
|
||||
this._framesSinceStats += out.length;
|
||||
return true;
|
||||
}
|
||||
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
let sample = 0;
|
||||
|
||||
if (this._playing) {
|
||||
const v = this._readInterpolated();
|
||||
if (v === null) {
|
||||
// Underrun: ramp out cleanly to avoid clicks.
|
||||
sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES);
|
||||
this._lastSample = sample;
|
||||
if (Math.abs(sample) < 1e-4) {
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this.port.postMessage({ kind: "underrun" });
|
||||
}
|
||||
} else {
|
||||
sample = v;
|
||||
this._lastSample = v;
|
||||
this._advance();
|
||||
}
|
||||
|
||||
if (this._fadeIn > 0) {
|
||||
const gain = 1 - this._fadeIn / FADE_FRAMES;
|
||||
sample *= gain;
|
||||
this._fadeIn -= 1;
|
||||
}
|
||||
if (this._fadeOut > 0) {
|
||||
const gain = this._fadeOut / FADE_FRAMES;
|
||||
sample *= gain;
|
||||
this._fadeOut -= 1;
|
||||
if (this._fadeOut === 0) {
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
}
|
||||
}
|
||||
|
||||
this._totalPlayed += 1;
|
||||
const abs = sample < 0 ? -sample : sample;
|
||||
if (abs > this._dbgOutPeak) this._dbgOutPeak = abs;
|
||||
}
|
||||
|
||||
out[i] = sample;
|
||||
if (stereo) stereo[i] = sample;
|
||||
}
|
||||
|
||||
this._framesSinceStats += out.length;
|
||||
if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) {
|
||||
this.port.postMessage({
|
||||
kind: "stats",
|
||||
queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000),
|
||||
played: this._totalPlayed,
|
||||
received: this._dbgReceived,
|
||||
outPeak: this._dbgOutPeak,
|
||||
});
|
||||
this._framesSinceStats = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("audio-playback", AudioPlaybackProcessor);
|
||||
Reference in New Issue
Block a user