feat: 添加语音 AI 主动问候功能,优化问候文本发送逻辑和音频处理参数
This commit is contained in:
@@ -139,17 +139,20 @@ export default function VoiceAssistant() {
|
||||
const voiceStateRef = useRef(state);
|
||||
voiceStateRef.current = state;
|
||||
const pendingGreetingRef = useRef(false); // 连接成功后待发送的问候
|
||||
const pendingGreetingTextRef = useRef(''); // 待发送的自定义问候文本(admin 测试)
|
||||
const greetingSentRef = useRef(false); // 问候已发出,等回复完成启动倒计时
|
||||
const greetingTimerRef = useRef(null); // 等待回答的倒计时句柄
|
||||
|
||||
// 主动发送问候(sendUserText → 后端 LLM 生成 → TTS 语音播放)
|
||||
const sendGreeting = useCallback(() => {
|
||||
const sendGreeting = useCallback((text) => {
|
||||
const c = clientRef.current;
|
||||
if (!c) return;
|
||||
pendingGreetingRef.current = false;
|
||||
pendingGreetingTextRef.current = '';
|
||||
greetingSentRef.current = true;
|
||||
c.sendUserText(GREETING_TEXT);
|
||||
console.warn('[vision] 自动问候已发送:', GREETING_TEXT);
|
||||
const msg = (text && String(text).trim()) || GREETING_TEXT;
|
||||
c.sendUserText(msg);
|
||||
console.warn('[vision] 主动问候已发送:', msg);
|
||||
}, []);
|
||||
|
||||
// ── 右下角气泡渲染(复刻原版 ui/chat.js)───────────────────────────────
|
||||
@@ -336,7 +339,14 @@ export default function VoiceAssistant() {
|
||||
console.warn('[vision] 用户已回答,取消挂断倒计时');
|
||||
}
|
||||
});
|
||||
client.addEventListener('input-level', (e) => paintInputLevel(e.detail?.rms));
|
||||
client.addEventListener('input-level', (e) => {
|
||||
// 诊断:麦克风电平(噪声门阈值 -50dB≈0.00316 线性值,说话时应显著高于它)
|
||||
const rms = e.detail?.rms;
|
||||
if (typeof rms === 'number' && !this?._rmsLog) {
|
||||
console.info(`[mic] 麦克风电平 rms=${rms.toFixed(4)}(说话时应 > 0.01,若长期接近 0 说明麦克风被静音/无输入)`);
|
||||
}
|
||||
paintInputLevel(rms);
|
||||
});
|
||||
client.addEventListener('toolcall', (e) => void onToolCall(e));
|
||||
client.addEventListener('response-finished', () => {
|
||||
setState('listening');
|
||||
@@ -355,8 +365,8 @@ export default function VoiceAssistant() {
|
||||
clientRef.current = client;
|
||||
await client.connect();
|
||||
setState('listening');
|
||||
// 自动问候:连接成功后发送问候(语音)
|
||||
if (pendingGreetingRef.current) sendGreeting();
|
||||
// 自动问候:连接成功后发送问候(语音;支持 admin 自定义文本)
|
||||
if (pendingGreetingRef.current) sendGreeting(pendingGreetingTextRef.current);
|
||||
} catch (err) {
|
||||
setErrorMsg(err?.message || String(err));
|
||||
setState('error');
|
||||
@@ -401,9 +411,17 @@ export default function VoiceAssistant() {
|
||||
// MQTT 调度器 → 语音页操作:启动/关闭对话、刷新页面
|
||||
useEffect(() => {
|
||||
const onVoice = (e) => {
|
||||
const { action } = e.detail || {};
|
||||
const { action, text } = e.detail || {};
|
||||
if (action === 'start') {
|
||||
if (!clientRef.current) void start();
|
||||
const greetText = (text && String(text).trim()) || sessionStorage.getItem('dpm_voice_greet') || '';
|
||||
if (greetText) sessionStorage.removeItem('dpm_voice_greet');
|
||||
if (clientRef.current) {
|
||||
sendGreeting(greetText);
|
||||
} else {
|
||||
pendingGreetingTextRef.current = greetText;
|
||||
pendingGreetingRef.current = true;
|
||||
void start();
|
||||
}
|
||||
} else if (action === 'stop') {
|
||||
void stop();
|
||||
} else if (action === 'refresh') {
|
||||
@@ -449,7 +467,7 @@ export default function VoiceAssistant() {
|
||||
}
|
||||
pendingGreetingRef.current = true;
|
||||
if (clientRef.current) {
|
||||
sendGreeting();
|
||||
sendGreeting('');
|
||||
} else {
|
||||
void start(); // start 连接成功后检查 pendingGreetingRef 再发问候
|
||||
}
|
||||
|
||||
@@ -122,6 +122,13 @@ export function useMqttControl() {
|
||||
|
||||
// ── 语音对话页操作 ──
|
||||
case 'voice_start':
|
||||
// admin 主动问候测试:携带 text;若不在 /voice 页先跳转,文本暂存 sessionStorage
|
||||
if (cmd.params?.text) sessionStorage.setItem('dpm_voice_greet', cmd.params.text);
|
||||
if (location.pathname !== '/voice') navigate('/voice');
|
||||
window.dispatchEvent(new CustomEvent('dpm:voice-control', {
|
||||
detail: { action: 'start', text: cmd.params?.text || '' },
|
||||
}));
|
||||
break;
|
||||
case 'voice_stop':
|
||||
case 'voice_refresh':
|
||||
window.dispatchEvent(new CustomEvent('dpm:voice-control', {
|
||||
|
||||
@@ -652,6 +652,14 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return;
|
||||
if (!this._sessionConfigured) return; // Server rejects audio before session.update.
|
||||
if (this._muted) return;
|
||||
// 诊断:首块音频与每 5 秒一次的发送确认(确认麦克风→WS 链路通畅)
|
||||
if (!this._sentChunks) this._sentChunks = 0;
|
||||
this._sentChunks++;
|
||||
if (this._sentChunks === 1) {
|
||||
console.info(`[mic] 首个音频块已发送(${pcm16Buffer.byteLength} 字节 PCM16@16k)→ 语音链路已打通`);
|
||||
} else if (this._sentChunks % 50 === 0) {
|
||||
console.info(`[mic] 已持续发送音频 ${this._sentChunks} 块`);
|
||||
}
|
||||
const b64 = base64FromArrayBuffer(pcm16Buffer);
|
||||
this._send({ type: "input_audio_buffer.append", audio: b64 });
|
||||
}
|
||||
@@ -706,6 +714,7 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
break;
|
||||
|
||||
case "input_audio_buffer.speech_started":
|
||||
console.info("[vad] 检测到开始说话(服务端 VAD)");
|
||||
// User started speaking — stop any audio still playing OR queued, every
|
||||
// time. We clear unconditionally (not just when `_aiSpeaking`): after a
|
||||
// reply or a tool result the worklet's ring buffer can still be draining
|
||||
@@ -717,6 +726,7 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
break;
|
||||
|
||||
case "input_audio_buffer.speech_stopped":
|
||||
console.info("[vad] 检测到说话结束(服务端 VAD)→ 发起识别调用");
|
||||
if (this._status === "user-speaking") this._setStatus("processing");
|
||||
break;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user