Pine
18d28be943
- 引入新的`docker-compose.yml`文件，便于后端和MQTT代理（EMQX）的部署。 - 更新多个组件中的API基本检索，使用函数进行动态解析。 - 加强了多个组件中API调用的错误处理和日志记录。 - 优化了AI聊天面板离线场景的回退响应。 - 更新DataScreen和MediaScreen组件中的数据显示和统计，以反映准确的指标。 - 重构MQTT连接逻辑，以支持动态凭证和客户端ID。
610 lines
28 KiB
React
610 lines
28 KiB
React
/* =========================================================
|
||
实时语音对话页（/voice）—— React 完全复刻 live-avatar 对话 UI
|
||
· 无自带 topbar（DPM 页眉由 ScreenLayout 注入）
|
||
· 中间圆球（状态机 + 指示器 + 噪声门圆环）、右下角气泡渲染、
|
||
底部文字输入 —— 结构与类名与原版一致，样式引入原版 style.css
|
||
· 逻辑：本地 VAD + 云 ASR(qwen3-asr-flash-realtime) + 云 LLM(qwen-plus) + 云 TTS(qwen3-tts-flash-realtime)
|
||
========================================================= */
|
||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||
import { S2sWsRealtimeClient } from '../voice/s2s-ws-client.js';
|
||
import useVisionDetection, { VISION_DWELL_MS } from '../voice/useVisionDetection.js';
|
||
import { getApiBase as API_BASE } from '../config';
|
||
import '../styles/datascreen.css';
|
||
import './voice-original.css';
|
||
import './voice-overrides.css';
|
||
|
||
const VOICE_URL = window.__DPM_VOICE_WS__ || `ws://${window.location.hostname}:8765/v1/realtime`;
|
||
// 基础提示词由后端统一管理（GET /api/s2s/instructions：配置提示词 + 园区知识库），
|
||
// 前端不硬编码；此处仅为后端不可用时的兜底
|
||
const FALLBACK_INSTRUCTIONS = '你是 PineSound 园区智能语音助手，请用简洁专业的中文回答，不超过三句话。';
|
||
const DEFAULT_VOICE = 'Cherry';
|
||
const DEFAULT_GATE_DB = -50;
|
||
|
||
// 工具定义（与后端 app/tools.py 的 tool_schemas 一致，经 session.update 传给 LLM）
|
||
const TOOL_DEFS = [
|
||
{
|
||
type: 'function', name: 'get_park_overview',
|
||
description: '获取园区实时运营概览：在园项目数、累计孵化企业、带动就业、营收（今日/累计）、设备在线率、在园人数、能耗等。回答园区数据类问题时使用。',
|
||
parameters: { type: 'object', properties: {} },
|
||
},
|
||
{
|
||
type: 'function', name: 'query_companies',
|
||
description: '查询园区入驻企业名录，支持按企业名关键词过滤。回答"有哪些企业/某企业是否入驻"时使用。',
|
||
parameters: { type: 'object', properties: { keyword: { type: 'string', description: '企业名关键词，可为空字符串' } } },
|
||
},
|
||
{
|
||
type: 'function', name: 'control_display',
|
||
description: '控制大屏显示：switch_page 切换页面、alert 弹通知、media_play/media_pause 控制媒体播放。用户要求控制大屏时使用。',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
action: { type: 'string', enum: ['switch_page', 'alert', 'media_play', 'media_pause'] },
|
||
target: { type: 'string', description: '目标：页面路径（如 /、/twin、/ai、/voice）或通知文本' },
|
||
},
|
||
},
|
||
},
|
||
{
|
||
type: 'function', name: 'get_time',
|
||
description: '获取当前日期时间。用户问"现在几点/今天几号"时使用。',
|
||
parameters: { type: 'object', properties: {} },
|
||
},
|
||
];
|
||
|
||
const STATE_VIEWS = {
|
||
idle: '点击开始',
|
||
connecting: '思考中',
|
||
listening: '正在聆听',
|
||
'user-speaking': '',
|
||
processing: '',
|
||
'ai-speaking': '',
|
||
error: '点击重试',
|
||
};
|
||
const STATE_CLASS = {
|
||
idle: 'state-idle', connecting: 'state-connecting', listening: 'state-listening',
|
||
'user-speaking': 'state-user-speaking', processing: 'state-processing',
|
||
'ai-speaking': 'state-ai-speaking', error: 'state-error',
|
||
};
|
||
const LIVE_STATES = new Set(['listening', 'user-speaking', 'processing', 'ai-speaking']);
|
||
const GATE_OFF_DB = -66;
|
||
const GATE_MAX_DB = -3;
|
||
// ── 噪声门弧线几何（复刻原版 main.js：200° 弧，缺口朝向圆球）──
|
||
const ARC_R = 40;
|
||
const ARC_SPAN_DEG = 200;
|
||
const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // 80°：左下起点
|
||
function arcPoint(f, r = ARC_R) {
|
||
const deg = ARC_START_DEG + f * ARC_SPAN_DEG;
|
||
const rad = (deg * Math.PI) / 180;
|
||
return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) };
|
||
}
|
||
function fullArcD() {
|
||
const a = arcPoint(0);
|
||
const b = arcPoint(1);
|
||
return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 1 1 ${b.x} ${b.y}`;
|
||
}
|
||
function dbToFraction(db) {
|
||
const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db));
|
||
return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB);
|
||
}
|
||
|
||
const icons = { micOn: <svg className="mic-on" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"><rect x="9" y="2" width="6" height="12" rx="3" /><path d="M5 10a7 7 0 0 0 14 0" /><line x1="12" y1="19" x2="12" y2="22" /></svg>,
|
||
micOff: <svg className="mic-off" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"><line x1="2" y1="2" x2="22" y2="22" /><path d="M9 5a3 3 0 0 1 6 0v4" /><path d="M9 10v1a3 3 0 0 0 5.1 2.1" /><path d="M19 10a7 7 0 0 1-1.24 3.97" /><path d="M5 10a7 7 0 0 0 11 5.67" /><line x1="12" y1="19" x2="12" y2="22" /></svg>,
|
||
indMic: <svg className="ind ind-mic" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true"><rect x="9" y="2" width="6" height="12" rx="3" fill="currentColor" stroke="none" /><path d="M5 10a7 7 0 0 0 14 0" /><line x1="12" y1="19" x2="12" y2="22" /><line x1="8" y1="22" x2="16" y2="22" /></svg>,
|
||
indError: <svg className="ind ind-error" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><circle cx="12" cy="12" r="9" /><line x1="12" y1="8" x2="12" y2="13" /><line x1="12" y1="16" x2="12" y2="16" /></svg>,
|
||
indVoice: <svg className="ind ind-voice" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true"><path d="M3 10v4a1 1 0 0 0 1 1h3l5 4V5L7 9H4a1 1 0 0 0-1 1z" fill="currentColor" stroke="none" /><path className="wave wave-1" d="M16 8a5 5 0 0 1 0 8" /><path className="wave wave-2" d="M19 5a9 9 0 0 1 0 14" /></svg>,
|
||
stop: <svg viewBox="0 0 24 24" fill="currentColor"><rect x="6" y="6" width="12" height="12" rx="2" /></svg>,
|
||
};
|
||
|
||
// 单条气泡：挂载后 rAF 加 `.in` 触发淡入（复刻原版 _spawnBubble 的进场动画）
|
||
function Bubble({ m, latest }) {
|
||
const [show, setShow] = useState(false);
|
||
useEffect(() => {
|
||
const raf = requestAnimationFrame(() => setShow(true));
|
||
return () => cancelAnimationFrame(raf);
|
||
}, []);
|
||
return (
|
||
<div className={`bubble ${m.role}${show ? ' in' : ''}${latest ? ' latest' : ' history'}`}>
|
||
<div className="bubble-role">{m.role === 'user' ? 'You' : 'Assistant'}</div>
|
||
<div className={`bubble-body${m.partial ? ' partial' : ''}`}>{m.text}</div>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
export default function VoiceAssistant() {
|
||
const [state, setState] = useState('idle');
|
||
const [messages, setMessages] = useState([]);
|
||
const [muted, setMuted] = useState(false);
|
||
const [errorMsg, setErrorMsg] = useState('');
|
||
// 全局人物识别开关（MQTT vision_set 控制）
|
||
const [visionEnabled, setVisionEnabled] = useState(true);
|
||
|
||
const clientRef = useRef(null);
|
||
const audioCtxRef = useRef(null);
|
||
const streamRef = useRef(null);
|
||
// 噪声门弧线元素
|
||
const micGateRef = useRef(null);
|
||
const mgaTrackRef = useRef(null);
|
||
const mgaFillRef = useRef(null);
|
||
const mgaHitRef = useRef(null);
|
||
const mgaHandleRef = useRef(null);
|
||
// 气泡定时器：id -> { dismiss?, remove? }
|
||
const bubbleTimersRef = useRef(new Map());
|
||
const messagesRef = useRef([]);
|
||
|
||
const live = LIVE_STATES.has(state);
|
||
|
||
// ── 自动问候生命周期状态（须在 start 之前定义：start 依赖数组引用 sendGreeting）──
|
||
const GREETING_TEXT = '您好呀，需要我的帮助吗？';
|
||
const GREETING_WAIT_MS = 25000; // 问候播完后等待用户回答的超时
|
||
const voiceStateRef = useRef(state);
|
||
voiceStateRef.current = state;
|
||
const pendingGreetingRef = useRef(false); // 连接成功后待发送的问候
|
||
const greetingSentRef = useRef(false); // 问候已发出，等回复完成启动倒计时
|
||
const greetingTimerRef = useRef(null); // 等待回答的倒计时句柄
|
||
|
||
// 主动发送问候（sendUserText → 后端 LLM 生成 → TTS 语音播放）
|
||
const sendGreeting = useCallback(() => {
|
||
const c = clientRef.current;
|
||
if (!c) return;
|
||
pendingGreetingRef.current = false;
|
||
greetingSentRef.current = true;
|
||
c.sendUserText(GREETING_TEXT);
|
||
console.warn('[vision] 自动问候已发送:', GREETING_TEXT);
|
||
}, []);
|
||
|
||
// ── 右下角气泡渲染（复刻原版 ui/chat.js）───────────────────────────────
|
||
// 流程：新气泡插入 → rAF 加 `.in`（淡入）→ 4s 后加 `.out`（淡出）→ 400ms 移除。
|
||
// 持续更新的气泡（partial 逐字流）续期 4s；栈上限 8 条，最旧优先淡出，
|
||
// 但永不驱逐正在更新的用户气泡。全部动画定时器集中管理，卸载时清理。
|
||
const clearBubbleTimers = useCallback((id) => {
|
||
const t = bubbleTimersRef.current.get(id);
|
||
if (!t) return;
|
||
if (t.dismiss) clearTimeout(t.dismiss);
|
||
if (t.remove) clearTimeout(t.remove);
|
||
bubbleTimersRef.current.delete(id);
|
||
}, []);
|
||
|
||
/** 立即把气泡标记为 `.out`，delay(默认400ms) 后从数组移除。 */
|
||
const markLeaving = useCallback((id, delay = 400) => {
|
||
setMessages((prev) => {
|
||
const m = prev.find((x) => x.id === id);
|
||
if (!m || m.leaving) return prev;
|
||
return prev.map((x) => (x.id === id ? { ...x, leaving: true } : x));
|
||
});
|
||
const remove = setTimeout(() => {
|
||
setMessages((prev) => prev.filter((x) => x.id !== id));
|
||
bubbleTimersRef.current.delete(id);
|
||
}, delay);
|
||
bubbleTimersRef.current.set(id, { remove });
|
||
}, []);
|
||
|
||
/** delay(默认4s) 后自动淡出；再次调用即续期（清掉旧计时重排）。 */
|
||
const scheduleBubbleDismiss = useCallback((id, delay = 4000) => {
|
||
clearBubbleTimers(id);
|
||
const dismiss = setTimeout(() => {
|
||
setMessages((prev) => {
|
||
const m = prev.find((x) => x.id === id);
|
||
if (!m || m.leaving) return prev;
|
||
return prev.map((x) => (x.id === id ? { ...x, leaving: true } : x));
|
||
});
|
||
const remove = setTimeout(() => {
|
||
setMessages((prev) => prev.filter((x) => x.id !== id));
|
||
bubbleTimersRef.current.delete(id);
|
||
}, 400);
|
||
bubbleTimersRef.current.set(id, { remove });
|
||
}, delay);
|
||
bubbleTimersRef.current.set(id, { dismiss });
|
||
}, [clearBubbleTimers]);
|
||
|
||
const pushMessage = useCallback((role, text, partial = false) => {
|
||
const key = role === 'user' ? 'user' : 'assistant';
|
||
let next = [...messagesRef.current];
|
||
const last = next.length ? next[next.length - 1] : null;
|
||
if (partial && last && last.role === key) {
|
||
// 同一条逐字流：更新文本，持续展示不消失
|
||
next[next.length - 1] = { ...last, text, partial: true };
|
||
} else {
|
||
const id = `${Date.now()}-${Math.random()}`;
|
||
next.push({ id, role: key, text, partial });
|
||
// 最多保留 4 条（最新大字 + 向上历史小字；正在逐字更新的消息位于末位，必然保留）
|
||
next = next.slice(-4);
|
||
}
|
||
messagesRef.current = next;
|
||
setMessages(next);
|
||
}, []);
|
||
|
||
// ── 工具调用执行器：LLM 请求工具 → 转发后端 /api/tools/exec → 结果回传 ──
|
||
const onToolCall = useCallback(async (e) => {
|
||
const { name, arguments: argsStr, callId } = e?.detail ?? {};
|
||
if (!name || !callId) return;
|
||
let args = {};
|
||
try { args = JSON.parse(argsStr || '{}'); } catch { /* 忽略非法参数 */ }
|
||
console.warn(`[tools] LLM 调用工具: ${name}`, args);
|
||
pushMessage('assistant', `🔧 调用工具：${name}`, false);
|
||
let output;
|
||
try {
|
||
const res = await fetch(`${API_BASE()}/api/tools/exec`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({ name, args }),
|
||
});
|
||
const data = await res.json();
|
||
output = data?.result ?? JSON.stringify(data);
|
||
} catch (err) {
|
||
output = JSON.stringify({ error: String(err) });
|
||
}
|
||
const c = clientRef.current;
|
||
if (c) {
|
||
c.sendToolOutput(callId, output);
|
||
c.requestResponse();
|
||
}
|
||
}, [pushMessage]);
|
||
|
||
// 结束会话：淡出全部气泡（原版 teardown 行为）
|
||
const dismissAllBubbles = useCallback(() => {
|
||
const ids = messagesRef.current.map((m) => m.id);
|
||
messagesRef.current = [];
|
||
ids.forEach((id) => {
|
||
if (id !== undefined) markLeaving(id);
|
||
});
|
||
}, [markLeaving]);
|
||
|
||
// 卸载时清理所有气泡定时器
|
||
useEffect(() => {
|
||
return () => {
|
||
for (const t of bubbleTimersRef.current.values()) {
|
||
if (t.dismiss) clearTimeout(t.dismiss);
|
||
if (t.remove) clearTimeout(t.remove);
|
||
}
|
||
bubbleTimersRef.current.clear();
|
||
};
|
||
}, []);
|
||
|
||
// 噪声门弧线：一次性几何（track/fill/hit 同一条 200° 弧 + 阈值圆点）
|
||
useEffect(() => {
|
||
const d = fullArcD();
|
||
mgaTrackRef.current?.setAttribute('d', d);
|
||
mgaFillRef.current?.setAttribute('d', d);
|
||
mgaHitRef.current?.setAttribute('d', d);
|
||
const fill = mgaFillRef.current;
|
||
if (fill) {
|
||
fill.setAttribute('pathLength', '100');
|
||
fill.style.strokeDasharray = '100 100';
|
||
fill.style.strokeDashoffset = '100'; // 无电平时空弧
|
||
}
|
||
const p = arcPoint(dbToFraction(DEFAULT_GATE_DB));
|
||
mgaHandleRef.current?.setAttribute('cx', String(p.x));
|
||
mgaHandleRef.current?.setAttribute('cy', String(p.y));
|
||
}, []);
|
||
|
||
// 实时麦克风电平 → 弧线填充 + 过门点亮（复刻原版 paintInputLevel）
|
||
const paintInputLevel = useCallback((rms) => {
|
||
if (!mgaFillRef.current || !micGateRef.current) return;
|
||
const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB;
|
||
const f = dbToFraction(db);
|
||
mgaFillRef.current.style.strokeDashoffset = String(100 * (1 - f));
|
||
const enabled = DEFAULT_GATE_DB > GATE_OFF_DB;
|
||
micGateRef.current.classList.toggle('gate-open', enabled && f >= dbToFraction(DEFAULT_GATE_DB));
|
||
}, []);
|
||
|
||
const start = useCallback(async () => {
|
||
if (clientRef.current) return;
|
||
setState('connecting');
|
||
try {
|
||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||
throw new Error('非安全上下文：请通过 http://localhost 或 HTTPS 访问，浏览器才允许麦克风');
|
||
}
|
||
const stream = await navigator.mediaDevices.getUserMedia({
|
||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||
});
|
||
streamRef.current = stream;
|
||
if (!audioCtxRef.current) audioCtxRef.current = new AudioContext();
|
||
if (audioCtxRef.current.state === 'suspended') await audioCtxRef.current.resume();
|
||
|
||
// 基础提示词：从后端拉取（配置 + 园区知识库已由服务端组装）；失败用极简兜底
|
||
let instructions = FALLBACK_INSTRUCTIONS;
|
||
try {
|
||
const insRes = await fetch(`${API_BASE()}/api/s2s/instructions`, { cache: 'no-store' });
|
||
const insData = await insRes.json();
|
||
if (insData?.ok && insData.instructions) instructions = insData.instructions;
|
||
} catch { /* 后端不可用不影响对话 */ }
|
||
|
||
const client = new S2sWsRealtimeClient({
|
||
directUrl: VOICE_URL,
|
||
voice: DEFAULT_VOICE,
|
||
instructions,
|
||
tools: TOOL_DEFS,
|
||
noiseGate: { enabled: true, thresholdDb: DEFAULT_GATE_DB },
|
||
audioContext: audioCtxRef.current,
|
||
micStream: stream,
|
||
});
|
||
const statusMap = {
|
||
connected: 'listening', 'user-speaking': 'user-speaking',
|
||
processing: 'processing', 'ai-speaking': 'ai-speaking', idle: 'idle',
|
||
};
|
||
client.addEventListener('status', (e) => {
|
||
const s = e.detail?.status;
|
||
setState(statusMap[s] ?? 'listening');
|
||
});
|
||
client.addEventListener('transcript', (e) => {
|
||
const d = e.detail;
|
||
pushMessage(d.role === 'user' ? 'user' : 'assistant', d.text, d.partial);
|
||
// 用户开始说话（回答了问候）→ 取消挂断倒计时
|
||
if (d.role === 'user' && d.text.trim() && greetingTimerRef.current) {
|
||
clearTimeout(greetingTimerRef.current);
|
||
greetingTimerRef.current = null;
|
||
console.warn('[vision] 用户已回答，取消挂断倒计时');
|
||
}
|
||
});
|
||
client.addEventListener('input-level', (e) => paintInputLevel(e.detail?.rms));
|
||
client.addEventListener('toolcall', (e) => void onToolCall(e));
|
||
client.addEventListener('response-finished', () => {
|
||
setState('listening');
|
||
// 自动问候的回复播放完毕 → 启动"等待用户回答"倒计时；无回答则自动挂断
|
||
if (greetingSentRef.current) {
|
||
greetingSentRef.current = false;
|
||
if (greetingTimerRef.current) clearTimeout(greetingTimerRef.current);
|
||
greetingTimerRef.current = setTimeout(() => {
|
||
console.warn('[vision] 问候后用户未回答，自动挂断回初始界面');
|
||
void stop();
|
||
}, GREETING_WAIT_MS);
|
||
}
|
||
});
|
||
client.addEventListener('error', () => setState('error'));
|
||
|
||
clientRef.current = client;
|
||
await client.connect();
|
||
setState('listening');
|
||
// 自动问候：连接成功后发送问候（语音）
|
||
if (pendingGreetingRef.current) sendGreeting();
|
||
} catch (err) {
|
||
setErrorMsg(err?.message || String(err));
|
||
setState('error');
|
||
}
|
||
}, [pushMessage, paintInputLevel, sendGreeting, onToolCall]);
|
||
|
||
const stop = useCallback(async () => {
|
||
// 清理自动问候相关状态/倒计时
|
||
if (greetingTimerRef.current) {
|
||
clearTimeout(greetingTimerRef.current);
|
||
greetingTimerRef.current = null;
|
||
}
|
||
pendingGreetingRef.current = false;
|
||
greetingSentRef.current = false;
|
||
const c = clientRef.current;
|
||
clientRef.current = null;
|
||
if (c) { try { await c.close(); } catch { /* noop */ } }
|
||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||
streamRef.current = null;
|
||
setMuted(false);
|
||
setErrorMsg('');
|
||
dismissAllBubbles();
|
||
setState('idle');
|
||
}, [dismissAllBubbles]);
|
||
|
||
const toggleMute = useCallback(() => {
|
||
const c = clientRef.current;
|
||
if (!c) return;
|
||
setMuted((m) => { c.setMuted(!m); return !m; });
|
||
}, []);
|
||
|
||
// ── 视觉识别触发：主动问候 + 后端记录/广播 ─────────────────────────────
|
||
// 完整生命周期：
|
||
// 触发（仅 idle 状态）→ 打开对话(start) → 连接成功发问候(sendUserText→LLM→语音)
|
||
// → 回复播放完(response-finished) → 等待用户回答 25s → 无回答则自动挂断回 idle
|
||
// → 静默 3 分钟（hook VISION_SILENT_MS）期间不再打扰
|
||
// 循环依赖：onVisionTrigger 需读取 vision.faces，而它又要作为参数传给 useVisionDetection。
|
||
// 解法：hook 的 onTrigger/onGesture 从 ref 取最新回调；回调在 vision 声明之后再定义。
|
||
const visionTriggerRef = useRef(null);
|
||
const gestureRef = useRef(null);
|
||
|
||
// MQTT 调度器 → 语音页操作：启动/关闭对话、刷新页面
|
||
useEffect(() => {
|
||
const onVoice = (e) => {
|
||
const { action } = e.detail || {};
|
||
if (action === 'start') {
|
||
if (!clientRef.current) void start();
|
||
} else if (action === 'stop') {
|
||
void stop();
|
||
} else if (action === 'refresh') {
|
||
window.location.reload();
|
||
}
|
||
};
|
||
window.addEventListener('dpm:voice-control', onVoice);
|
||
return () => window.removeEventListener('dpm:voice-control', onVoice);
|
||
}, [start, stop]);
|
||
|
||
// MQTT 调度器 → 全局人物识别开关
|
||
useEffect(() => {
|
||
const onVision = (e) => setVisionEnabled(Boolean(e.detail?.enabled));
|
||
window.addEventListener('dpm:vision-control', onVision);
|
||
return () => window.removeEventListener('dpm:vision-control', onVision);
|
||
}, []);
|
||
|
||
const vision = useVisionDetection({
|
||
enabled: visionEnabled,
|
||
onTrigger: () => visionTriggerRef.current?.(),
|
||
onGesture: (g) => gestureRef.current?.(g),
|
||
});
|
||
|
||
// 手势 → 对话控制：举手 toggle —— 无对话则开始，正在对话则结束
|
||
const onGesture = useCallback((g) => {
|
||
if (g !== 'raise') return;
|
||
if (clientRef.current) {
|
||
void stop();
|
||
} else {
|
||
void start();
|
||
}
|
||
}, [start, stop]);
|
||
|
||
useEffect(() => {
|
||
gestureRef.current = onGesture;
|
||
}, [onGesture]);
|
||
|
||
const onVisionTrigger = useCallback(() => {
|
||
// 仅 idle 允许自动问候：对话中/连接中/其他状态一律不打扰
|
||
if (voiceStateRef.current !== 'idle') {
|
||
console.warn('[vision] 自动问候跳过：当前状态', voiceStateRef.current);
|
||
return;
|
||
}
|
||
pendingGreetingRef.current = true;
|
||
if (clientRef.current) {
|
||
sendGreeting();
|
||
} else {
|
||
void start(); // start 连接成功后检查 pendingGreetingRef 再发问候
|
||
}
|
||
fetch(`${API_BASE()}/api/vision/event`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({ event: 'triggered', faces: vision.faces, dwell_ms: VISION_DWELL_MS }),
|
||
}).catch(() => { /* 上报失败不影响本地问候 */ });
|
||
}, [vision.faces, start, sendGreeting]);
|
||
|
||
useEffect(() => {
|
||
visionTriggerRef.current = onVisionTrigger;
|
||
}, [onVisionTrigger]);
|
||
|
||
// 摄像头识别状态变化 → 上报后端（详细日志定位）
|
||
useEffect(() => {
|
||
if (!vision.status || vision.status === 'off' || vision.status === 'loading') return;
|
||
fetch(`${API_BASE()}/api/vision/event`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({
|
||
event: vision.status === 'model-error' ? 'model_error' : vision.status,
|
||
faces: vision.faces,
|
||
dwell_ms: vision.dwellMs,
|
||
detail: vision.errorInfo ? `${vision.errorInfo.name}: ${vision.errorInfo.message}` : '',
|
||
}),
|
||
}).catch(() => { /* 静默 */ });
|
||
}, [vision.status]); // eslint-disable-line react-hooks/exhaustive-deps
|
||
|
||
const visionStatusText = {
|
||
off: '摄像头关闭',
|
||
loading: '摄像头启动中…',
|
||
detecting: `识别中 · ${vision.faces} 人${vision.faces > 0 ? `（后端 ${vision.latency ?? '-'}ms）` : ' · 未检到人脸'} · 链路${vision.modelPhase === 'ok' ? 'OK' : vision.modelPhase === 'failed' ? `失败:${(vision.modelErr || '').slice(0, 40)}` : '连接中'} · 亮度${vision.frameLum ?? '-'}${vision.detectErr ? ` · 异常:${String(vision.detectErr).slice(0, 50)}` : ''}`,
|
||
facing: `面向大屏 ${(vision.dwellMs / 1000).toFixed(1)}s / ${VISION_DWELL_MS / 1000}s`,
|
||
triggered: '已触发 · 主动问候',
|
||
silent: '静默中',
|
||
error: '摄像头不可用',
|
||
}[vision.status] ?? '';
|
||
|
||
// 摄像头失败的具体原因
|
||
const visionErrorText = (() => {
|
||
if (vision.status !== 'error') return null;
|
||
const name = vision.errorInfo?.name ?? '';
|
||
const msg = vision.errorInfo?.message ?? '';
|
||
if (!window.isSecureContext) return '非安全上下文：请用 localhost 或 HTTPS 访问';
|
||
if (name === 'NotFoundError') return `未检测到摄像头设备：${msg || '请确认 USB 摄像头已接入本机'}`;
|
||
if (name === 'NotAllowedError') return `摄像头权限被拒：请在浏览器地址栏允许 ${location.host} 使用摄像头`;
|
||
if (name === 'NotReadableError') return '摄像头被其他应用占用或不可用，请关闭占用程序后刷新';
|
||
return `摄像头启动失败（${name}）：${msg}`;
|
||
})();
|
||
|
||
const caption = state === 'error' ? (errorMsg || 'Tap to retry') : (STATE_VIEWS[state] ?? '');
|
||
|
||
return (
|
||
<div id="app">
|
||
{/* 非安全上下文提示（局域网 IP 访问时浏览器禁用麦克风/摄像头） */}
|
||
{!window.isSecureContext && (
|
||
<div className="voice-secure-warn">
|
||
⚠️ 当前页面不是安全上下文，浏览器禁止麦克风/摄像头。
|
||
请改用 <b>http://localhost</b> 访问（或为后端配置 HTTPS）。
|
||
</div>
|
||
)}
|
||
|
||
{/* 主舞台：中间圆球 */}
|
||
<main className="stage">
|
||
<div className={`orb-wrap${live ? ' live' : ''}`}>
|
||
<div id="mic-gate" className="mic-gate" ref={micGateRef}>
|
||
<svg id="mic-gate-arc" className="mic-gate-arc" viewBox="0 0 100 100" aria-hidden="true">
|
||
<path id="mga-track" className="mga-track" fill="none" ref={mgaTrackRef} />
|
||
<path id="mga-fill" className="mga-fill" fill="none" ref={mgaFillRef} />
|
||
<path id="mga-hit" className="mga-hit" fill="none" ref={mgaHitRef} />
|
||
<circle id="mga-handle" className="mga-handle" r="3" ref={mgaHandleRef} />
|
||
</svg>
|
||
<button
|
||
id="mic-btn"
|
||
className={`side-btn${muted ? ' muted' : ''}`}
|
||
type="button"
|
||
aria-label={muted ? 'Unmute' : 'Mute'}
|
||
title={muted ? 'Unmute' : 'Mute'}
|
||
aria-hidden={!live}
|
||
tabIndex={live ? 0 : -1}
|
||
onClick={toggleMute}
|
||
>
|
||
{icons.micOn}
|
||
{icons.micOff}
|
||
</button>
|
||
</div>
|
||
|
||
<button
|
||
id="main-circle"
|
||
className={`circle ${STATE_CLASS[state] ?? 'state-idle'}`}
|
||
type="button"
|
||
aria-label="Start voice conversation"
|
||
disabled={state === 'connecting'}
|
||
onClick={live ? stop : start}
|
||
>
|
||
<span className="circle-glow" aria-hidden="true" />
|
||
<span className="circle-ring" aria-hidden="true" />
|
||
<span className="circle-ring-outer" aria-hidden="true" />
|
||
<span className="circle-core">
|
||
<span className="circle-indicator" aria-hidden="true">
|
||
{icons.indMic}
|
||
{icons.indError}
|
||
<span className="ind ind-spinner" />
|
||
<span className="ind ind-thinking"><span className="dot" /><span className="dot" /><span className="dot" /></span>
|
||
<span className="ind ind-bars"><span className="bar" /><span className="bar" /><span className="bar" /><span className="bar" /><span className="bar" /></span>
|
||
{icons.indVoice}
|
||
</span>
|
||
</span>
|
||
</button>
|
||
|
||
<button
|
||
id="stop-btn"
|
||
className="side-btn"
|
||
type="button"
|
||
aria-label="End"
|
||
title="End"
|
||
aria-hidden={!live}
|
||
tabIndex={live ? 0 : -1}
|
||
onClick={stop}
|
||
>
|
||
{icons.stop}
|
||
</button>
|
||
</div>
|
||
|
||
<p id="circle-caption" className={`circle-caption${caption ? '' : ' empty'}${state === 'error' ? ' error' : ''}`} role="status">{caption}</p>
|
||
<p id="circle-subcaption" className="circle-subcaption" hidden />
|
||
</main>
|
||
|
||
{/* 右下角气泡（复刻原版：进场 in / 4s 自动淡出 out / 阶梯字号） */}
|
||
<div id="bubble-stack" className="bubble-stack" aria-live="polite" aria-atomic="false">
|
||
{messages.slice(-4).map((m, i, arr) => (
|
||
<Bubble key={m.id} m={m} latest={i === arr.length - 1} />
|
||
))}
|
||
</div>
|
||
|
||
{/* 视觉识别：摄像头默认开启；video 始终渲染（ref 先于启动存在），未开启时隐藏 */}
|
||
<div className={`voice-cam-pip${vision.cameraOn ? '' : ' hidden'}`}>
|
||
<video ref={vision.videoRef} autoPlay playsInline muted />
|
||
<div className="voice-cam-bar">
|
||
<span className="voice-cam-title">
|
||
<span className={`voice-cam-dot ${vision.status === 'triggered' ? 'trigger' : vision.status === 'facing' ? 'face' : 'on'}`} />
|
||
智能识别中
|
||
</span>
|
||
<span>
|
||
{vision.gesture === 'raise' && (
|
||
<b style={{ color: '#2f6bff' }}>您好呀 · {live ? '开始对话' : '结束对话'}</b>
|
||
)}
|
||
{visionStatusText}
|
||
</span>
|
||
</div>
|
||
</div>
|
||
{vision.status === 'error' && !vision.cameraOn && (
|
||
<div className="voice-vision-status">⚠️ {visionErrorText}</div>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|