From f75fe7b9b33022dc91bca86bc98220b24c181380 Mon Sep 17 00:00:00 2001 From: Pine Date: Tue, 18 Aug 2026 08:20:44 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E8=AF=AD=E9=9F=B3=20?= =?UTF-8?q?AI=20=E4=B8=BB=E5=8A=A8=E9=97=AE=E5=80=99=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E9=97=AE=E5=80=99=E6=96=87=E6=9C=AC?= =?UTF-8?q?=E5=8F=91=E9=80=81=E9=80=BB=E8=BE=91=E5=92=8C=E9=9F=B3=E9=A2=91?= =?UTF-8?q?=E5=A4=84=E7=90=86=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/config.py | 2 +- backend/app/s2s_bridge.py | 4 ++-- backend/static/admin.js | 31 +++++++++++++++++++++++++++++++ backend/templates/admin.html | 19 +++++++++++++++++++ src/pages/VoiceAssistant.jsx | 36 +++++++++++++++++++++++++++--------- src/utils/useMqttControl.js | 7 +++++++ src/voice/s2s-ws-client.js | 10 ++++++++++ 7 files changed, 97 insertions(+), 12 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 3777048..559ac2b 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -90,7 +90,7 @@ class Settings: # 语音助手基础提示词（.env 设 DPM_S2S_INSTRUCTIONS 可覆盖；知识库内容由 rag 自动拼接） S2S_INSTRUCTIONS = _env( "DPM_S2S_INSTRUCTIONS", - "你是 PineSound 园区智能语音助手。可用工具：get_park_overview（园区实时数据）、" + "你是昆小园，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、" "query_companies（企业名录）、control_display（大屏控制）、get_time（时间）。" "涉及园区数据/企业/大屏控制时务必调用工具获取准确信息。请用简洁专业的中文回答，不超过三句话。", ) diff --git a/backend/app/s2s_bridge.py b/backend/app/s2s_bridge.py index 73ccf38..4cf01d4 100644 --- a/backend/app/s2s_bridge.py +++ b/backend/app/s2s_bridge.py @@ -41,8 +41,8 @@ def _build_pool(stop_event: threading.Event): "--qwen3_cloud_model", settings.S2S_TTS_MODEL, "--qwen3_cloud_voice", settings.S2S_TTS_VOICE, "--enable_live_transcription", "false", - "--thresh", "0.8", - "--min_silence_ms", "600", + "--thresh", "0.6", + "--min_silence_ms", "500", "--min_speech_ms", "500", "--log_level", settings.S2S_LOG_LEVEL, ] diff --git a/backend/static/admin.js b/backend/static/admin.js index ca8fffe..274687e 100644 --- a/backend/static/admin.js +++ b/backend/static/admin.js @@ -358,3 +358,34 @@ post('/api/delete', { path: current.path }, function () { closePreview(); reload('已删除'); }); }); })(); +/* ---------- 语音 AI 测试（主动问候） ---------- */ +(function () { + var greetBtn = document.getElementById('btnVoiceGreet'); + var greetInput = document.getElementById('voiceGreetText'); + var greetHint = document.getElementById('voiceGreetHint'); + if (!greetBtn || !greetInput) return; + greetBtn.addEventListener('click', function () { + var text = (greetInput.value || '').trim(); + if (!text) { toast('请输入问候语'); return; } + greetBtn.disabled = true; + greetBtn.textContent = '发送中…'; + fetch('/api/display/command', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ action: 'voice_start', params: { text: text } }), + }) + .then(function (r) { return r.json().catch(function () { return {}; }); }) + .then(function (d) { + greetBtn.disabled = false; + greetBtn.textContent = '播报问候'; + if (d && d.ok === false) { toast('发送失败：MQTT 未发布（' + (d.error || '未知') + '）', 'error'); return; } + if (greetHint) greetHint.textContent = '✅ 已发送：展播端将进入语音对话页并主动播报「' + text + '」，请留意展播端声音。'; + toast('语音问候已发送'); + }) + .catch(function () { + greetBtn.disabled = false; + greetBtn.textContent = '播报问候'; + toast('发送失败：后端不可达', 'error'); + }); + }); +})(); diff --git a/backend/templates/admin.html b/backend/templates/admin.html index 7357c40..30dbb15 100644 --- a/backend/templates/admin.html +++ b/backend/templates/admin.html @@ -242,6 +242,25 @@ + +
+
+
+

语音 AI 测试

+
+
+
+ + +
+
+ + +
+
+
点击后展播端将自动进入语音对话页并主动播报上述问候（LLM 生成 + TTS），用于验证语音链路。
+
+
diff --git a/src/pages/VoiceAssistant.jsx b/src/pages/VoiceAssistant.jsx index 7237f14..4d5145a 100644 --- a/src/pages/VoiceAssistant.jsx +++ b/src/pages/VoiceAssistant.jsx @@ -139,17 +139,20 @@ export default function VoiceAssistant() { const voiceStateRef = useRef(state); voiceStateRef.current = state; const pendingGreetingRef = useRef(false); // 连接成功后待发送的问候 + const pendingGreetingTextRef = useRef(''); // 待发送的自定义问候文本（admin 测试） const greetingSentRef = useRef(false); // 问候已发出，等回复完成启动倒计时 const greetingTimerRef = useRef(null); // 等待回答的倒计时句柄 // 主动发送问候（sendUserText → 后端 LLM 生成 → TTS 语音播放） - const sendGreeting = useCallback(() => { + const sendGreeting = useCallback((text) => { const c = clientRef.current; if (!c) return; pendingGreetingRef.current = false; + pendingGreetingTextRef.current = ''; greetingSentRef.current = true; - c.sendUserText(GREETING_TEXT); - console.warn('[vision] 自动问候已发送:', GREETING_TEXT); + const msg = (text && String(text).trim()) || GREETING_TEXT; + c.sendUserText(msg); + console.warn('[vision] 主动问候已发送:', msg); }, []); // ── 右下角气泡渲染（复刻原版 ui/chat.js）─────────────────────────────── @@ -336,7 +339,14 @@ export default function VoiceAssistant() { console.warn('[vision] 用户已回答，取消挂断倒计时'); } }); - client.addEventListener('input-level', (e) => paintInputLevel(e.detail?.rms)); + client.addEventListener('input-level', (e) => { + // 诊断：麦克风电平（噪声门阈值 -50dB≈0.00316 线性值，说话时应显著高于它） + const rms = e.detail?.rms; + if (typeof rms === 'number' && !this?._rmsLog) { + console.info(`[mic] 麦克风电平 rms=${rms.toFixed(4)}（说话时应 > 0.01，若长期接近 0 说明麦克风被静音/无输入）`); + } + paintInputLevel(rms); + }); client.addEventListener('toolcall', (e) => void onToolCall(e)); client.addEventListener('response-finished', () => { setState('listening'); @@ -355,8 +365,8 @@ export default function VoiceAssistant() { clientRef.current = client; await client.connect(); setState('listening'); - // 自动问候：连接成功后发送问候（语音） - if (pendingGreetingRef.current) sendGreeting(); + // 自动问候：连接成功后发送问候（语音；支持 admin 自定义文本） + if (pendingGreetingRef.current) sendGreeting(pendingGreetingTextRef.current); } catch (err) { setErrorMsg(err?.message || String(err)); setState('error'); @@ -401,9 +411,17 @@ export default function VoiceAssistant() { // MQTT 调度器 → 语音页操作：启动/关闭对话、刷新页面 useEffect(() => { const onVoice = (e) => { - const { action } = e.detail || {}; + const { action, text } = e.detail || {}; if (action === 'start') { - if (!clientRef.current) void start(); + const greetText = (text && String(text).trim()) || sessionStorage.getItem('dpm_voice_greet') || ''; + if (greetText) sessionStorage.removeItem('dpm_voice_greet'); + if (clientRef.current) { + sendGreeting(greetText); + } else { + pendingGreetingTextRef.current = greetText; + pendingGreetingRef.current = true; + void start(); + } } else if (action === 'stop') { void stop(); } else if (action === 'refresh') { @@ -449,7 +467,7 @@ export default function VoiceAssistant() { } pendingGreetingRef.current = true; if (clientRef.current) { - sendGreeting(); + sendGreeting(''); } else { void start(); // start 连接成功后检查 pendingGreetingRef 再发问候 } diff --git a/src/utils/useMqttControl.js b/src/utils/useMqttControl.js index 71dcbf8..a051936 100644 --- a/src/utils/useMqttControl.js +++ b/src/utils/useMqttControl.js @@ -122,6 +122,13 @@ export function useMqttControl() { // ── 语音对话页操作 ── case 'voice_start': + // admin 主动问候测试：携带 text；若不在 /voice 页先跳转，文本暂存 sessionStorage + if (cmd.params?.text) sessionStorage.setItem('dpm_voice_greet', cmd.params.text); + if (location.pathname !== '/voice') navigate('/voice'); + window.dispatchEvent(new CustomEvent('dpm:voice-control', { + detail: { action: 'start', text: cmd.params?.text || '' }, + })); + break; case 'voice_stop': case 'voice_refresh': window.dispatchEvent(new CustomEvent('dpm:voice-control', { diff --git a/src/voice/s2s-ws-client.js b/src/voice/s2s-ws-client.js index 17be6a2..29c862a 100644 --- a/src/voice/s2s-ws-client.js +++ b/src/voice/s2s-ws-client.js @@ -652,6 +652,14 @@ export class S2sWsRealtimeClient extends EventTarget { if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; if (!this._sessionConfigured) return; // Server rejects audio before session.update. if (this._muted) return; + // 诊断：首块音频与每 5 秒一次的发送确认（确认麦克风→WS 链路通畅） + if (!this._sentChunks) this._sentChunks = 0; + this._sentChunks++; + if (this._sentChunks === 1) { + console.info(`[mic] 首个音频块已发送（${pcm16Buffer.byteLength} 字节 PCM16@16k）→ 语音链路已打通`); + } else if (this._sentChunks % 50 === 0) { + console.info(`[mic] 已持续发送音频 ${this._sentChunks} 块`); + } const b64 = base64FromArrayBuffer(pcm16Buffer); this._send({ type: "input_audio_buffer.append", audio: b64 }); } @@ -706,6 +714,7 @@ export class S2sWsRealtimeClient extends EventTarget { break; case "input_audio_buffer.speech_started": + console.info("[vad] 检测到开始说话（服务端 VAD）"); // User started speaking — stop any audio still playing OR queued, every // time. We clear unconditionally (not just when `_aiSpeaking`): after a // reply or a tool result the worklet's ring buffer can still be draining @@ -717,6 +726,7 @@ export class S2sWsRealtimeClient extends EventTarget { break; case "input_audio_buffer.speech_stopped": + console.info("[vad] 检测到说话结束（服务端 VAD）→ 发起识别调用"); if (this._status === "user-speaking") this._setStatus("processing"); break;