feat: 添加语音 AI 主动问候功能，优化问候文本发送逻辑和音频处理参数
This commit is contained in:
Pine
@@ -90,7 +90,7 @@ class Settings:
|
||||
# 语音助手基础提示词（.env 设 DPM_S2S_INSTRUCTIONS 可覆盖；知识库内容由 rag 自动拼接）
|
||||
S2S_INSTRUCTIONS = _env(
|
||||
"DPM_S2S_INSTRUCTIONS",
|
||||
"你是 PineSound 园区智能语音助手。可用工具：get_park_overview（园区实时数据）、"
|
||||
"你是昆小园，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、"
|
||||
"query_companies（企业名录）、control_display（大屏控制）、get_time（时间）。"
|
||||
"涉及园区数据/企业/大屏控制时务必调用工具获取准确信息。请用简洁专业的中文回答，不超过三句话。",
|
||||
)
|
||||
|
||||
@@ -41,8 +41,8 @@ def _build_pool(stop_event: threading.Event):
|
||||
"--qwen3_cloud_model", settings.S2S_TTS_MODEL,
|
||||
"--qwen3_cloud_voice", settings.S2S_TTS_VOICE,
|
||||
"--enable_live_transcription", "false",
|
||||
"--thresh", "0.8",
|
||||
"--min_silence_ms", "600",
|
||||
"--thresh", "0.6",
|
||||
"--min_silence_ms", "500",
|
||||
"--min_speech_ms", "500",
|
||||
"--log_level", settings.S2S_LOG_LEVEL,
|
||||
]
|
||||
|
||||
@@ -358,3 +358,34 @@
|
||||
post('/api/delete', { path: current.path }, function () { closePreview(); reload('已删除'); });
|
||||
});
|
||||
})();
|
||||
/* ---------- 语音 AI 测试（主动问候） ---------- */
|
||||
(function () {
|
||||
var greetBtn = document.getElementById('btnVoiceGreet');
|
||||
var greetInput = document.getElementById('voiceGreetText');
|
||||
var greetHint = document.getElementById('voiceGreetHint');
|
||||
if (!greetBtn || !greetInput) return;
|
||||
greetBtn.addEventListener('click', function () {
|
||||
var text = (greetInput.value || '').trim();
|
||||
if (!text) { toast('请输入问候语'); return; }
|
||||
greetBtn.disabled = true;
|
||||
greetBtn.textContent = '发送中…';
|
||||
fetch('/api/display/command', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'voice_start', params: { text: text } }),
|
||||
})
|
||||
.then(function (r) { return r.json().catch(function () { return {}; }); })
|
||||
.then(function (d) {
|
||||
greetBtn.disabled = false;
|
||||
greetBtn.textContent = '播报问候';
|
||||
if (d && d.ok === false) { toast('发送失败：MQTT 未发布（' + (d.error || '未知') + '）', 'error'); return; }
|
||||
if (greetHint) greetHint.textContent = '✅ 已发送：展播端将进入语音对话页并主动播报「' + text + '」，请留意展播端声音。';
|
||||
toast('语音问候已发送');
|
||||
})
|
||||
.catch(function () {
|
||||
greetBtn.disabled = false;
|
||||
greetBtn.textContent = '播报问候';
|
||||
toast('发送失败：后端不可达', 'error');
|
||||
});
|
||||
});
|
||||
})();
|
||||
|
||||
@@ -242,6 +242,25 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 语音 AI 测试（admin → 前端主动问候） -->
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<div class="card-icon"><svg viewBox="0 0 24 24" width="20" height="20" fill="none" stroke="#1D5DCE" stroke-width="1.6"><path d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2M12 19v3"/></svg></div>
|
||||
<h3>语音 AI 测试</h3>
|
||||
</div>
|
||||
<div class="form-inline">
|
||||
<div class="form-group" style="flex:2;">
|
||||
<label>问候语（前端将主动播报，测试语音 AI）</label>
|
||||
<input type="text" id="voiceGreetText" value="您好呀，我是园区智能语音助手，有什么可以帮您？" style="width:100%;">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label> </label>
|
||||
<button class="btn-primary" id="btnVoiceGreet" style="width:100%;">播报问候</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="hint" id="voiceGreetHint">点击后展播端将自动进入语音对话页并主动播报上述问候（LLM 生成 + TTS），用于验证语音链路。</div>
|
||||
</div>
|
||||
|
||||
<!-- 显示控制 -->
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
|
||||
@@ -139,17 +139,20 @@ export default function VoiceAssistant() {
|
||||
const voiceStateRef = useRef(state);
|
||||
voiceStateRef.current = state;
|
||||
const pendingGreetingRef = useRef(false); // 连接成功后待发送的问候
|
||||
const pendingGreetingTextRef = useRef(''); // 待发送的自定义问候文本（admin 测试）
|
||||
const greetingSentRef = useRef(false); // 问候已发出，等回复完成启动倒计时
|
||||
const greetingTimerRef = useRef(null); // 等待回答的倒计时句柄
|
||||
|
||||
// 主动发送问候（sendUserText → 后端 LLM 生成 → TTS 语音播放）
|
||||
const sendGreeting = useCallback(() => {
|
||||
const sendGreeting = useCallback((text) => {
|
||||
const c = clientRef.current;
|
||||
if (!c) return;
|
||||
pendingGreetingRef.current = false;
|
||||
pendingGreetingTextRef.current = '';
|
||||
greetingSentRef.current = true;
|
||||
c.sendUserText(GREETING_TEXT);
|
||||
console.warn('[vision] 自动问候已发送:', GREETING_TEXT);
|
||||
const msg = (text && String(text).trim()) || GREETING_TEXT;
|
||||
c.sendUserText(msg);
|
||||
console.warn('[vision] 主动问候已发送:', msg);
|
||||
}, []);
|
||||
|
||||
// ── 右下角气泡渲染（复刻原版 ui/chat.js）───────────────────────────────
|
||||
@@ -336,7 +339,14 @@ export default function VoiceAssistant() {
|
||||
console.warn('[vision] 用户已回答，取消挂断倒计时');
|
||||
}
|
||||
});
|
||||
client.addEventListener('input-level', (e) => paintInputLevel(e.detail?.rms));
|
||||
client.addEventListener('input-level', (e) => {
|
||||
// 诊断：麦克风电平（噪声门阈值 -50dB≈0.00316 线性值，说话时应显著高于它）
|
||||
const rms = e.detail?.rms;
|
||||
if (typeof rms === 'number' && !this?._rmsLog) {
|
||||
console.info(`[mic] 麦克风电平 rms=${rms.toFixed(4)}（说话时应 > 0.01，若长期接近 0 说明麦克风被静音/无输入）`);
|
||||
}
|
||||
paintInputLevel(rms);
|
||||
});
|
||||
client.addEventListener('toolcall', (e) => void onToolCall(e));
|
||||
client.addEventListener('response-finished', () => {
|
||||
setState('listening');
|
||||
@@ -355,8 +365,8 @@ export default function VoiceAssistant() {
|
||||
clientRef.current = client;
|
||||
await client.connect();
|
||||
setState('listening');
|
||||
// 自动问候：连接成功后发送问候（语音）
|
||||
if (pendingGreetingRef.current) sendGreeting();
|
||||
// 自动问候：连接成功后发送问候（语音；支持 admin 自定义文本）
|
||||
if (pendingGreetingRef.current) sendGreeting(pendingGreetingTextRef.current);
|
||||
} catch (err) {
|
||||
setErrorMsg(err?.message || String(err));
|
||||
setState('error');
|
||||
@@ -401,9 +411,17 @@ export default function VoiceAssistant() {
|
||||
// MQTT 调度器 → 语音页操作：启动/关闭对话、刷新页面
|
||||
useEffect(() => {
|
||||
const onVoice = (e) => {
|
||||
const { action } = e.detail || {};
|
||||
const { action, text } = e.detail || {};
|
||||
if (action === 'start') {
|
||||
if (!clientRef.current) void start();
|
||||
const greetText = (text && String(text).trim()) || sessionStorage.getItem('dpm_voice_greet') || '';
|
||||
if (greetText) sessionStorage.removeItem('dpm_voice_greet');
|
||||
if (clientRef.current) {
|
||||
sendGreeting(greetText);
|
||||
} else {
|
||||
pendingGreetingTextRef.current = greetText;
|
||||
pendingGreetingRef.current = true;
|
||||
void start();
|
||||
}
|
||||
} else if (action === 'stop') {
|
||||
void stop();
|
||||
} else if (action === 'refresh') {
|
||||
@@ -449,7 +467,7 @@ export default function VoiceAssistant() {
|
||||
}
|
||||
pendingGreetingRef.current = true;
|
||||
if (clientRef.current) {
|
||||
sendGreeting();
|
||||
sendGreeting('');
|
||||
} else {
|
||||
void start(); // start 连接成功后检查 pendingGreetingRef 再发问候
|
||||
}
|
||||
|
||||
@@ -122,6 +122,13 @@ export function useMqttControl() {
|
||||
|
||||
// ── 语音对话页操作 ──
|
||||
case 'voice_start':
|
||||
// admin 主动问候测试：携带 text；若不在 /voice 页先跳转，文本暂存 sessionStorage
|
||||
if (cmd.params?.text) sessionStorage.setItem('dpm_voice_greet', cmd.params.text);
|
||||
if (location.pathname !== '/voice') navigate('/voice');
|
||||
window.dispatchEvent(new CustomEvent('dpm:voice-control', {
|
||||
detail: { action: 'start', text: cmd.params?.text || '' },
|
||||
}));
|
||||
break;
|
||||
case 'voice_stop':
|
||||
case 'voice_refresh':
|
||||
window.dispatchEvent(new CustomEvent('dpm:voice-control', {
|
||||
|
||||
@@ -652,6 +652,14 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return;
|
||||
if (!this._sessionConfigured) return; // Server rejects audio before session.update.
|
||||
if (this._muted) return;
|
||||
// 诊断：首块音频与每 5 秒一次的发送确认（确认麦克风→WS 链路通畅）
|
||||
if (!this._sentChunks) this._sentChunks = 0;
|
||||
this._sentChunks++;
|
||||
if (this._sentChunks === 1) {
|
||||
console.info(`[mic] 首个音频块已发送（${pcm16Buffer.byteLength} 字节 PCM16@16k）→ 语音链路已打通`);
|
||||
} else if (this._sentChunks % 50 === 0) {
|
||||
console.info(`[mic] 已持续发送音频 ${this._sentChunks} 块`);
|
||||
}
|
||||
const b64 = base64FromArrayBuffer(pcm16Buffer);
|
||||
this._send({ type: "input_audio_buffer.append", audio: b64 });
|
||||
}
|
||||
@@ -706,6 +714,7 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
break;
|
||||
|
||||
case "input_audio_buffer.speech_started":
|
||||
console.info("[vad] 检测到开始说话（服务端 VAD）");
|
||||
// User started speaking — stop any audio still playing OR queued, every
|
||||
// time. We clear unconditionally (not just when `_aiSpeaking`): after a
|
||||
// reply or a tool result the worklet's ring buffer can still be draining
|
||||
@@ -717,6 +726,7 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
break;
|
||||
|
||||
case "input_audio_buffer.speech_stopped":
|
||||
console.info("[vad] 检测到说话结束（服务端 VAD）→ 发起识别调用");
|
||||
if (this._status === "user-speaking") this._setStatus("processing");
|
||||
break;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user