/* ========================================================= 实时语音对话页（/voice）—— React 完全复刻 live-avatar 对话 UI · 无自带 topbar（DPM 页眉由 ScreenLayout 注入） · 中间圆球（状态机 + 指示器 + 噪声门圆环）、右下角气泡渲染、 底部文字输入 —— 结构与类名与原版一致，样式引入原版 style.css · 逻辑：本地 VAD + 云 ASR(qwen3-asr-flash-realtime) + 云 LLM(qwen-plus) + 云 TTS(qwen3-tts-flash-realtime) ========================================================= */ import { useCallback, useEffect, useRef, useState } from 'react'; import { S2sWsRealtimeClient } from '../voice/s2s-ws-client.js'; import useVisionDetection, { VISION_DWELL_MS } from '../voice/useVisionDetection.js'; import { getApiBase as API_BASE } from '../config'; import '../styles/datascreen.css'; import './voice-original.css'; import './voice-overrides.css'; const VOICE_URL = window.__DPM_VOICE_WS__ || `ws://${window.location.hostname}:8765/v1/realtime`; // 基础提示词由后端统一管理（GET /api/s2s/instructions：配置提示词 + 园区知识库）， // 前端不硬编码；此处仅为后端不可用时的兜底 const FALLBACK_INSTRUCTIONS = '你是 PineSound 园区智能语音助手，请用简洁专业的中文回答，不超过三句话。'; const DEFAULT_VOICE = 'Cherry'; const DEFAULT_GATE_DB = -50; // 工具定义（与后端 app/tools.py 的 tool_schemas 一致，经 session.update 传给 LLM） const TOOL_DEFS = [ { type: 'function', name: 'get_park_overview', description: '获取园区实时运营概览：在园项目数、累计孵化企业、带动就业、营收（今日/累计）、设备在线率、在园人数、能耗等。回答园区数据类问题时使用。', parameters: { type: 'object', properties: {} }, }, { type: 'function', name: 'query_companies', description: '查询园区入驻企业名录，支持按企业名关键词过滤。回答"有哪些企业/某企业是否入驻"时使用。', parameters: { type: 'object', properties: { keyword: { type: 'string', description: '企业名关键词，可为空字符串' } } }, }, { type: 'function', name: 'control_display', description: '控制大屏显示：switch_page 切换页面、alert 弹通知、media_play/media_pause 控制媒体播放。用户要求控制大屏时使用。', parameters: { type: 'object', properties: { action: { type: 'string', enum: ['switch_page', 'alert', 'media_play', 'media_pause'] }, target: { type: 'string', description: '目标：页面路径（如 /、/twin、/ai、/voice）或通知文本' }, }, }, }, { type: 'function', name: 'get_time', description: '获取当前日期时间。用户问"现在几点/今天几号"时使用。', parameters: { type: 'object', properties: {} }, }, ]; const STATE_VIEWS = { idle: '点击开始', connecting: '思考中', listening: '正在聆听', 'user-speaking': '', processing: '', 'ai-speaking': '', error: '点击重试', }; const STATE_CLASS = { idle: 'state-idle', connecting: 'state-connecting', listening: 'state-listening', 'user-speaking': 'state-user-speaking', processing: 'state-processing', 'ai-speaking': 'state-ai-speaking', error: 'state-error', }; const LIVE_STATES = new Set(['listening', 'user-speaking', 'processing', 'ai-speaking']); const GATE_OFF_DB = -66; const GATE_MAX_DB = -3; // ── 噪声门弧线几何（复刻原版 main.js：200° 弧，缺口朝向圆球）── const ARC_R = 40; const ARC_SPAN_DEG = 200; const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // 80°：左下起点 function arcPoint(f, r = ARC_R) { const deg = ARC_START_DEG + f * ARC_SPAN_DEG; const rad = (deg * Math.PI) / 180; return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; } function fullArcD() { const a = arcPoint(0); const b = arcPoint(1); return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 1 1 ${b.x} ${b.y}`; } function dbToFraction(db) { const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); } const icons = { micOn: , micOff: , indMic: , indError: , indVoice: , stop: , }; // 单条气泡：挂载后 rAF 加 `.in` 触发淡入（复刻原版 _spawnBubble 的进场动画） function Bubble({ m, latest }) { const [show, setShow] = useState(false); useEffect(() => { const raf = requestAnimationFrame(() => setShow(true)); return () => cancelAnimationFrame(raf); }, []); return (
{m.role === 'user' ? 'You' : 'Assistant'}
{m.text}
); } export default function VoiceAssistant() { const [state, setState] = useState('idle'); const [messages, setMessages] = useState([]); const [muted, setMuted] = useState(false); const [errorMsg, setErrorMsg] = useState(''); // 全局人物识别开关（MQTT vision_set 控制） const [visionEnabled, setVisionEnabled] = useState(true); const clientRef = useRef(null); const audioCtxRef = useRef(null); const streamRef = useRef(null); // 噪声门弧线元素 const micGateRef = useRef(null); const mgaTrackRef = useRef(null); const mgaFillRef = useRef(null); const mgaHitRef = useRef(null); const mgaHandleRef = useRef(null); // 气泡定时器：id -> { dismiss?, remove? } const bubbleTimersRef = useRef(new Map()); const messagesRef = useRef([]); const live = LIVE_STATES.has(state); // ── 自动问候生命周期状态（须在 start 之前定义：start 依赖数组引用 sendGreeting）── const GREETING_TEXT = '您好呀，需要我的帮助吗？'; const GREETING_WAIT_MS = 25000; // 问候播完后等待用户回答的超时 const voiceStateRef = useRef(state); voiceStateRef.current = state; const pendingGreetingRef = useRef(false); // 连接成功后待发送的问候 const greetingSentRef = useRef(false); // 问候已发出，等回复完成启动倒计时 const greetingTimerRef = useRef(null); // 等待回答的倒计时句柄 // 主动发送问候（sendUserText → 后端 LLM 生成 → TTS 语音播放） const sendGreeting = useCallback(() => { const c = clientRef.current; if (!c) return; pendingGreetingRef.current = false; greetingSentRef.current = true; c.sendUserText(GREETING_TEXT); console.warn('[vision] 自动问候已发送:', GREETING_TEXT); }, []); // ── 右下角气泡渲染（复刻原版 ui/chat.js）─────────────────────────────── // 流程：新气泡插入 → rAF 加 `.in`（淡入）→ 4s 后加 `.out`（淡出）→ 400ms 移除。 // 持续更新的气泡（partial 逐字流）续期 4s；栈上限 8 条，最旧优先淡出， // 但永不驱逐正在更新的用户气泡。全部动画定时器集中管理，卸载时清理。 const clearBubbleTimers = useCallback((id) => { const t = bubbleTimersRef.current.get(id); if (!t) return; if (t.dismiss) clearTimeout(t.dismiss); if (t.remove) clearTimeout(t.remove); bubbleTimersRef.current.delete(id); }, []); /** 立即把气泡标记为 `.out`，delay(默认400ms) 后从数组移除。 */ const markLeaving = useCallback((id, delay = 400) => { setMessages((prev) => { const m = prev.find((x) => x.id === id); if (!m || m.leaving) return prev; return prev.map((x) => (x.id === id ? { ...x, leaving: true } : x)); }); const remove = setTimeout(() => { setMessages((prev) => prev.filter((x) => x.id !== id)); bubbleTimersRef.current.delete(id); }, delay); bubbleTimersRef.current.set(id, { remove }); }, []); /** delay(默认4s) 后自动淡出；再次调用即续期（清掉旧计时重排）。 */ const scheduleBubbleDismiss = useCallback((id, delay = 4000) => { clearBubbleTimers(id); const dismiss = setTimeout(() => { setMessages((prev) => { const m = prev.find((x) => x.id === id); if (!m || m.leaving) return prev; return prev.map((x) => (x.id === id ? { ...x, leaving: true } : x)); }); const remove = setTimeout(() => { setMessages((prev) => prev.filter((x) => x.id !== id)); bubbleTimersRef.current.delete(id); }, 400); bubbleTimersRef.current.set(id, { remove }); }, delay); bubbleTimersRef.current.set(id, { dismiss }); }, [clearBubbleTimers]); const pushMessage = useCallback((role, text, partial = false) => { const key = role === 'user' ? 'user' : 'assistant'; let next = [...messagesRef.current]; const last = next.length ? next[next.length - 1] : null; if (partial && last && last.role === key) { // 同一条逐字流：更新文本，持续展示不消失 next[next.length - 1] = { ...last, text, partial: true }; } else { const id = `${Date.now()}-${Math.random()}`; next.push({ id, role: key, text, partial }); // 最多保留 4 条（最新大字 + 向上历史小字；正在逐字更新的消息位于末位，必然保留） next = next.slice(-4); } messagesRef.current = next; setMessages(next); }, []); // ── 工具调用执行器：LLM 请求工具 → 转发后端 /api/tools/exec → 结果回传 ── const onToolCall = useCallback(async (e) => { const { name, arguments: argsStr, callId } = e?.detail ?? {}; if (!name || !callId) return; let args = {}; try { args = JSON.parse(argsStr || '{}'); } catch { /* 忽略非法参数 */ } console.warn(`[tools] LLM 调用工具: ${name}`, args); pushMessage('assistant', `🔧 调用工具：${name}`, false); let output; try { const res = await fetch(`${API_BASE()}/api/tools/exec`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ name, args }), }); const data = await res.json(); output = data?.result ?? JSON.stringify(data); } catch (err) { output = JSON.stringify({ error: String(err) }); } const c = clientRef.current; if (c) { c.sendToolOutput(callId, output); c.requestResponse(); } }, [pushMessage]); // 结束会话：淡出全部气泡（原版 teardown 行为） const dismissAllBubbles = useCallback(() => { const ids = messagesRef.current.map((m) => m.id); messagesRef.current = []; ids.forEach((id) => { if (id !== undefined) markLeaving(id); }); }, [markLeaving]); // 卸载时清理所有气泡定时器 useEffect(() => { return () => { for (const t of bubbleTimersRef.current.values()) { if (t.dismiss) clearTimeout(t.dismiss); if (t.remove) clearTimeout(t.remove); } bubbleTimersRef.current.clear(); }; }, []); // 噪声门弧线：一次性几何（track/fill/hit 同一条 200° 弧 + 阈值圆点） useEffect(() => { const d = fullArcD(); mgaTrackRef.current?.setAttribute('d', d); mgaFillRef.current?.setAttribute('d', d); mgaHitRef.current?.setAttribute('d', d); const fill = mgaFillRef.current; if (fill) { fill.setAttribute('pathLength', '100'); fill.style.strokeDasharray = '100 100'; fill.style.strokeDashoffset = '100'; // 无电平时空弧 } const p = arcPoint(dbToFraction(DEFAULT_GATE_DB)); mgaHandleRef.current?.setAttribute('cx', String(p.x)); mgaHandleRef.current?.setAttribute('cy', String(p.y)); }, []); // 实时麦克风电平 → 弧线填充 + 过门点亮（复刻原版 paintInputLevel） const paintInputLevel = useCallback((rms) => { if (!mgaFillRef.current || !micGateRef.current) return; const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; const f = dbToFraction(db); mgaFillRef.current.style.strokeDashoffset = String(100 * (1 - f)); const enabled = DEFAULT_GATE_DB > GATE_OFF_DB; micGateRef.current.classList.toggle('gate-open', enabled && f >= dbToFraction(DEFAULT_GATE_DB)); }, []); const start = useCallback(async () => { if (clientRef.current) return; setState('connecting'); try { if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) { throw new Error('非安全上下文：请通过 http://localhost 或 HTTPS 访问，浏览器才允许麦克风'); } const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); streamRef.current = stream; if (!audioCtxRef.current) audioCtxRef.current = new AudioContext(); if (audioCtxRef.current.state === 'suspended') await audioCtxRef.current.resume(); // 基础提示词：从后端拉取（配置 + 园区知识库已由服务端组装）；失败用极简兜底 let instructions = FALLBACK_INSTRUCTIONS; try { const insRes = await fetch(`${API_BASE()}/api/s2s/instructions`, { cache: 'no-store' }); const insData = await insRes.json(); if (insData?.ok && insData.instructions) instructions = insData.instructions; } catch { /* 后端不可用不影响对话 */ } const client = new S2sWsRealtimeClient({ directUrl: VOICE_URL, voice: DEFAULT_VOICE, instructions, tools: TOOL_DEFS, noiseGate: { enabled: true, thresholdDb: DEFAULT_GATE_DB }, audioContext: audioCtxRef.current, micStream: stream, }); const statusMap = { connected: 'listening', 'user-speaking': 'user-speaking', processing: 'processing', 'ai-speaking': 'ai-speaking', idle: 'idle', }; client.addEventListener('status', (e) => { const s = e.detail?.status; setState(statusMap[s] ?? 'listening'); }); client.addEventListener('transcript', (e) => { const d = e.detail; pushMessage(d.role === 'user' ? 'user' : 'assistant', d.text, d.partial); // 用户开始说话（回答了问候）→ 取消挂断倒计时 if (d.role === 'user' && d.text.trim() && greetingTimerRef.current) { clearTimeout(greetingTimerRef.current); greetingTimerRef.current = null; console.warn('[vision] 用户已回答，取消挂断倒计时'); } }); client.addEventListener('input-level', (e) => paintInputLevel(e.detail?.rms)); client.addEventListener('toolcall', (e) => void onToolCall(e)); client.addEventListener('response-finished', () => { setState('listening'); // 自动问候的回复播放完毕 → 启动"等待用户回答"倒计时；无回答则自动挂断 if (greetingSentRef.current) { greetingSentRef.current = false; if (greetingTimerRef.current) clearTimeout(greetingTimerRef.current); greetingTimerRef.current = setTimeout(() => { console.warn('[vision] 问候后用户未回答，自动挂断回初始界面'); void stop(); }, GREETING_WAIT_MS); } }); client.addEventListener('error', () => setState('error')); clientRef.current = client; await client.connect(); setState('listening'); // 自动问候：连接成功后发送问候（语音） if (pendingGreetingRef.current) sendGreeting(); } catch (err) { setErrorMsg(err?.message || String(err)); setState('error'); } }, [pushMessage, paintInputLevel, sendGreeting, onToolCall]); const stop = useCallback(async () => { // 清理自动问候相关状态/倒计时 if (greetingTimerRef.current) { clearTimeout(greetingTimerRef.current); greetingTimerRef.current = null; } pendingGreetingRef.current = false; greetingSentRef.current = false; const c = clientRef.current; clientRef.current = null; if (c) { try { await c.close(); } catch { /* noop */ } } streamRef.current?.getTracks().forEach((t) => t.stop()); streamRef.current = null; setMuted(false); setErrorMsg(''); dismissAllBubbles(); setState('idle'); }, [dismissAllBubbles]); const toggleMute = useCallback(() => { const c = clientRef.current; if (!c) return; setMuted((m) => { c.setMuted(!m); return !m; }); }, []); // ── 视觉识别触发：主动问候 + 后端记录/广播 ───────────────────────────── // 完整生命周期： // 触发（仅 idle 状态）→ 打开对话(start) → 连接成功发问候(sendUserText→LLM→语音) // → 回复播放完(response-finished) → 等待用户回答 25s → 无回答则自动挂断回 idle // → 静默 3 分钟（hook VISION_SILENT_MS）期间不再打扰 // 循环依赖：onVisionTrigger 需读取 vision.faces，而它又要作为参数传给 useVisionDetection。 // 解法：hook 的 onTrigger/onGesture 从 ref 取最新回调；回调在 vision 声明之后再定义。 const visionTriggerRef = useRef(null); const gestureRef = useRef(null); // MQTT 调度器 → 语音页操作：启动/关闭对话、刷新页面 useEffect(() => { const onVoice = (e) => { const { action } = e.detail || {}; if (action === 'start') { if (!clientRef.current) void start(); } else if (action === 'stop') { void stop(); } else if (action === 'refresh') { window.location.reload(); } }; window.addEventListener('dpm:voice-control', onVoice); return () => window.removeEventListener('dpm:voice-control', onVoice); }, [start, stop]); // MQTT 调度器 → 全局人物识别开关 useEffect(() => { const onVision = (e) => setVisionEnabled(Boolean(e.detail?.enabled)); window.addEventListener('dpm:vision-control', onVision); return () => window.removeEventListener('dpm:vision-control', onVision); }, []); const vision = useVisionDetection({ enabled: visionEnabled, onTrigger: () => visionTriggerRef.current?.(), onGesture: (g) => gestureRef.current?.(g), }); // 手势 → 对话控制：举手 toggle —— 无对话则开始，正在对话则结束 const onGesture = useCallback((g) => { if (g !== 'raise') return; if (clientRef.current) { void stop(); } else { void start(); } }, [start, stop]); useEffect(() => { gestureRef.current = onGesture; }, [onGesture]); const onVisionTrigger = useCallback(() => { // 仅 idle 允许自动问候：对话中/连接中/其他状态一律不打扰 if (voiceStateRef.current !== 'idle') { console.warn('[vision] 自动问候跳过：当前状态', voiceStateRef.current); return; } pendingGreetingRef.current = true; if (clientRef.current) { sendGreeting(); } else { void start(); // start 连接成功后检查 pendingGreetingRef 再发问候 } fetch(`${API_BASE()}/api/vision/event`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ event: 'triggered', faces: vision.faces, dwell_ms: VISION_DWELL_MS }), }).catch(() => { /* 上报失败不影响本地问候 */ }); }, [vision.faces, start, sendGreeting]); useEffect(() => { visionTriggerRef.current = onVisionTrigger; }, [onVisionTrigger]); // 摄像头识别状态变化 → 上报后端（详细日志定位） useEffect(() => { if (!vision.status || vision.status === 'off' || vision.status === 'loading') return; fetch(`${API_BASE()}/api/vision/event`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ event: vision.status === 'model-error' ? 'model_error' : vision.status, faces: vision.faces, dwell_ms: vision.dwellMs, detail: vision.errorInfo ? `${vision.errorInfo.name}: ${vision.errorInfo.message}` : '', }), }).catch(() => { /* 静默 */ }); }, [vision.status]); // eslint-disable-line react-hooks/exhaustive-deps const visionStatusText = { off: '摄像头关闭', loading: '摄像头启动中…', detecting: `识别中 · ${vision.faces} 人${vision.faces > 0 ? `（后端 ${vision.latency ?? '-'}ms）` : ' · 未检到人脸'} · 链路${vision.modelPhase === 'ok' ? 'OK' : vision.modelPhase === 'failed' ? `失败:${(vision.modelErr || '').slice(0, 40)}` : '连接中'} · 亮度${vision.frameLum ?? '-'}${vision.detectErr ? ` · 异常:${String(vision.detectErr).slice(0, 50)}` : ''}`, facing: `面向大屏 ${(vision.dwellMs / 1000).toFixed(1)}s / ${VISION_DWELL_MS / 1000}s`, triggered: '已触发 · 主动问候', silent: '静默中', error: '摄像头不可用', }[vision.status] ?? ''; // 摄像头失败的具体原因 const visionErrorText = (() => { if (vision.status !== 'error') return null; const name = vision.errorInfo?.name ?? ''; const msg = vision.errorInfo?.message ?? ''; if (!window.isSecureContext) return '非安全上下文：请用 localhost 或 HTTPS 访问'; if (name === 'NotFoundError') return `未检测到摄像头设备：${msg || '请确认 USB 摄像头已接入本机'}`; if (name === 'NotAllowedError') return `摄像头权限被拒：请在浏览器地址栏允许 ${location.host} 使用摄像头`; if (name === 'NotReadableError') return '摄像头被其他应用占用或不可用，请关闭占用程序后刷新'; return `摄像头启动失败（${name}）：${msg}`; })(); const caption = state === 'error' ? (errorMsg || 'Tap to retry') : (STATE_VIEWS[state] ?? ''); return (
{/* 非安全上下文提示（局域网 IP 访问时浏览器禁用麦克风/摄像头） */} {!window.isSecureContext && (
⚠️ 当前页面不是安全上下文，浏览器禁止麦克风/摄像头。 请改用 http://localhost 访问（或为后端配置 HTTPS）。
)} {/* 主舞台：中间圆球 */}

{caption}

{/* 右下角气泡（复刻原版：进场 in / 4s 自动淡出 out / 阶梯字号） */}
{messages.slice(-4).map((m, i, arr) => ( ))}
{/* 视觉识别：摄像头默认开启；video 始终渲染（ref 先于启动存在），未开启时隐藏 */}
{vision.status === 'error' && !vision.cameraOn && (
⚠️ {visionErrorText}
)}
); }