feat(voice): 实时语音对话页（React 完全复刻原版 UI）
- /voice 页面：中间圆球 + 左右按钮 + 状态图标 + 右下角气泡（进场/淡出/阶梯字号/上限8）+ 噪声门弧线电平，结构类名与原版一致 - s2s WebSocket 客户端 + worklet 音频（mic-capture/audio-playback）+ orb 可视化 - 摄像头实时预览（默认开启）+ 视觉识别状态条（后端 YOLO 推理） - 底部园区概览条（复用 ParkOverviewStrip），orb-wrap 视窗居中 - 页眉复用 ScreenLayout；左右键导航加入 /voice - 文档：docs/voice-integration-plan.md
This commit is contained in:
Pine
@@ -3,6 +3,7 @@ import MediaScreen from './pages/MediaScreen';
|
||||
import DataScreen from './pages/DataScreen';
|
||||
import DigitalTwin from './pages/DigitalTwin';
|
||||
import AiAssistant from './pages/AiAssistant';
|
||||
import VoiceAssistant from './pages/VoiceAssistant';
|
||||
import ScreenLayout from './components/ScreenLayout';
|
||||
import './styles/tokens.css';
|
||||
|
||||
@@ -16,6 +17,8 @@ export default function App() {
|
||||
<Route path="/twin" element={<DigitalTwin />} />
|
||||
<Route path="/ai" element={<AiAssistant />} />
|
||||
<Route path="/screen" element={<MediaScreen />} />
|
||||
{/* 实时语音对话页（React 原生，复用页眉） */}
|
||||
<Route path="/voice" element={<VoiceAssistant />} />
|
||||
</Route>
|
||||
{/* 管理后台已迁移至 FastAPI + Jinja（GET /admin），不由 React 路由接管 */}
|
||||
<Route path="*" element={<Navigate to="/" replace />} />
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
const env = import.meta.env || {};
|
||||
|
||||
export const API_BASE = window.__DPM_API__ || env.VITE_API_BASE || 'http://localhost:10085';
|
||||
export const API_BASE = window.__DPM_API__ || env.VITE_API_BASE || 'http://192.168.1.9:10085';
|
||||
export const MQTT_URL = window.__DPM_MQTT__ || env.VITE_MQTT_URL || 'ws://192.168.1.3:8083/mqtt';
|
||||
export const MQTT_USERNAME = window.__DPM_MQTT_USER__ || env.VITE_MQTT_USERNAME || '';
|
||||
export const MQTT_PASSWORD = window.__DPM_MQTT_PASS__ || env.VITE_MQTT_PASSWORD || '';
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
/* =========================================================
|
||||
实时语音对话页（/voice）—— React 完全复刻 live-avatar 对话 UI
|
||||
· 无自带 topbar（DPM 页眉由 ScreenLayout 注入）
|
||||
· 中间圆球（状态机 + 指示器 + 噪声门圆环）、右下角气泡渲染、
|
||||
底部文字输入 —— 结构与类名与原版一致，样式引入原版 style.css
|
||||
· 逻辑：本地 VAD + 云 ASR(qwen3-asr-flash-realtime) + 云 LLM(qwen-plus) + 云 TTS(qwen3-tts-flash-realtime)
|
||||
========================================================= */
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { S2sWsRealtimeClient } from '../voice/s2s-ws-client.js';
|
||||
import useVisionDetection, { VISION_DWELL_MS } from '../voice/useVisionDetection.js';
|
||||
import ParkOverviewStrip from '../components/ParkOverviewStrip';
|
||||
import { useParkSim } from '../utils/parkData';
|
||||
import { API_BASE } from '../config';
|
||||
import '../styles/datascreen.css';
|
||||
import './voice-original.css';
|
||||
import './voice-overrides.css';
|
||||
|
||||
const VOICE_URL = window.__DPM_VOICE_WS__ || `ws://${window.location.hostname}:8765/v1/realtime`;
|
||||
const DEFAULT_INSTRUCTIONS = '你是园区智能助手，请用简洁专业的中文回答，不超过三句话。';
|
||||
const DEFAULT_VOICE = 'Cherry';
|
||||
const DEFAULT_GATE_DB = -50;
|
||||
|
||||
const STATE_VIEWS = {
|
||||
idle: '点击开始',
|
||||
connecting: '思考中',
|
||||
listening: '正在聆听',
|
||||
'user-speaking': '',
|
||||
processing: '',
|
||||
'ai-speaking': '',
|
||||
error: '点击重试',
|
||||
};
|
||||
const STATE_CLASS = {
|
||||
idle: 'state-idle', connecting: 'state-connecting', listening: 'state-listening',
|
||||
'user-speaking': 'state-user-speaking', processing: 'state-processing',
|
||||
'ai-speaking': 'state-ai-speaking', error: 'state-error',
|
||||
};
|
||||
const LIVE_STATES = new Set(['listening', 'user-speaking', 'processing', 'ai-speaking']);
|
||||
const GATE_OFF_DB = -66;
|
||||
const GATE_MAX_DB = -3;
|
||||
// ── 噪声门弧线几何（复刻原版 main.js：200° 弧，缺口朝向圆球）──
|
||||
const ARC_R = 40;
|
||||
const ARC_SPAN_DEG = 200;
|
||||
const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // 80°：左下起点
|
||||
function arcPoint(f, r = ARC_R) {
|
||||
const deg = ARC_START_DEG + f * ARC_SPAN_DEG;
|
||||
const rad = (deg * Math.PI) / 180;
|
||||
return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) };
|
||||
}
|
||||
function fullArcD() {
|
||||
const a = arcPoint(0);
|
||||
const b = arcPoint(1);
|
||||
return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 1 1 ${b.x} ${b.y}`;
|
||||
}
|
||||
function dbToFraction(db) {
|
||||
const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db));
|
||||
return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB);
|
||||
}
|
||||
|
||||
const icons = { micOn: <svg className="mic-on" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"><rect x="9" y="2" width="6" height="12" rx="3" /><path d="M5 10a7 7 0 0 0 14 0" /><line x1="12" y1="19" x2="12" y2="22" /></svg>,
|
||||
micOff: <svg className="mic-off" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"><line x1="2" y1="2" x2="22" y2="22" /><path d="M9 5a3 3 0 0 1 6 0v4" /><path d="M9 10v1a3 3 0 0 0 5.1 2.1" /><path d="M19 10a7 7 0 0 1-1.24 3.97" /><path d="M5 10a7 7 0 0 0 11 5.67" /><line x1="12" y1="19" x2="12" y2="22" /></svg>,
|
||||
indMic: <svg className="ind ind-mic" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true"><rect x="9" y="2" width="6" height="12" rx="3" fill="currentColor" stroke="none" /><path d="M5 10a7 7 0 0 0 14 0" /><line x1="12" y1="19" x2="12" y2="22" /><line x1="8" y1="22" x2="16" y2="22" /></svg>,
|
||||
indError: <svg className="ind ind-error" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round"><circle cx="12" cy="12" r="9" /><line x1="12" y1="8" x2="12" y2="13" /><line x1="12" y1="16" x2="12" y2="16" /></svg>,
|
||||
indVoice: <svg className="ind ind-voice" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true"><path d="M3 10v4a1 1 0 0 0 1 1h3l5 4V5L7 9H4a1 1 0 0 0-1 1z" fill="currentColor" stroke="none" /><path className="wave wave-1" d="M16 8a5 5 0 0 1 0 8" /><path className="wave wave-2" d="M19 5a9 9 0 0 1 0 14" /></svg>,
|
||||
stop: <svg viewBox="0 0 24 24" fill="currentColor"><rect x="6" y="6" width="12" height="12" rx="2" /></svg>,
|
||||
};
|
||||
|
||||
// 单条气泡：挂载后 rAF 加 `.in` 触发淡入（复刻原版 _spawnBubble 的进场动画）
|
||||
function Bubble({ m }) {
|
||||
const [show, setShow] = useState(false);
|
||||
useEffect(() => {
|
||||
const raf = requestAnimationFrame(() => setShow(true));
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, []);
|
||||
return (
|
||||
<div className={`bubble ${m.role}${show ? ' in' : ''}${m.leaving ? ' out' : ''}`}>
|
||||
<div className="bubble-role">{m.role === 'user' ? 'You' : 'Assistant'}</div>
|
||||
<div className={`bubble-body${m.partial ? ' partial' : ''}`}>{m.text}</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function VoiceAssistant() {
|
||||
const [state, setState] = useState('idle');
|
||||
const [messages, setMessages] = useState([]);
|
||||
const [muted, setMuted] = useState(false);
|
||||
const [errorMsg, setErrorMsg] = useState('');
|
||||
// 园区数据（页脚概览条；后端 /api/dashboard/snapshot，离线本地模拟兜底）
|
||||
const d = useParkSim();
|
||||
|
||||
const clientRef = useRef(null);
|
||||
const audioCtxRef = useRef(null);
|
||||
const streamRef = useRef(null);
|
||||
// 噪声门弧线元素
|
||||
const micGateRef = useRef(null);
|
||||
const mgaTrackRef = useRef(null);
|
||||
const mgaFillRef = useRef(null);
|
||||
const mgaHitRef = useRef(null);
|
||||
const mgaHandleRef = useRef(null);
|
||||
// 气泡定时器：id -> { dismiss?, remove? }
|
||||
const bubbleTimersRef = useRef(new Map());
|
||||
const messagesRef = useRef([]);
|
||||
|
||||
const live = LIVE_STATES.has(state);
|
||||
|
||||
// ── 自动问候生命周期状态（须在 start 之前定义：start 依赖数组引用 sendGreeting）──
|
||||
const GREETING_TEXT = '你好！欢迎来到园区，我是智能语音助手，请问有什么可以帮你的吗？';
|
||||
const GREETING_WAIT_MS = 25000; // 问候播完后等待用户回答的超时
|
||||
const voiceStateRef = useRef(state);
|
||||
voiceStateRef.current = state;
|
||||
const pendingGreetingRef = useRef(false); // 连接成功后待发送的问候
|
||||
const greetingSentRef = useRef(false); // 问候已发出，等回复完成启动倒计时
|
||||
const greetingTimerRef = useRef(null); // 等待回答的倒计时句柄
|
||||
|
||||
// 主动发送问候（sendUserText → 后端 LLM 生成 → TTS 语音播放）
|
||||
const sendGreeting = useCallback(() => {
|
||||
const c = clientRef.current;
|
||||
if (!c) return;
|
||||
pendingGreetingRef.current = false;
|
||||
greetingSentRef.current = true;
|
||||
c.sendUserText(GREETING_TEXT);
|
||||
console.warn('[vision] 自动问候已发送:', GREETING_TEXT);
|
||||
}, []);
|
||||
|
||||
// ── 右下角气泡渲染（复刻原版 ui/chat.js）───────────────────────────────
|
||||
// 流程：新气泡插入 → rAF 加 `.in`（淡入）→ 4s 后加 `.out`（淡出）→ 400ms 移除。
|
||||
// 持续更新的气泡（partial 逐字流）续期 4s；栈上限 8 条，最旧优先淡出，
|
||||
// 但永不驱逐正在更新的用户气泡。全部动画定时器集中管理，卸载时清理。
|
||||
const clearBubbleTimers = useCallback((id) => {
|
||||
const t = bubbleTimersRef.current.get(id);
|
||||
if (!t) return;
|
||||
if (t.dismiss) clearTimeout(t.dismiss);
|
||||
if (t.remove) clearTimeout(t.remove);
|
||||
bubbleTimersRef.current.delete(id);
|
||||
}, []);
|
||||
|
||||
/** 立即把气泡标记为 `.out`，delay(默认400ms) 后从数组移除。 */
|
||||
const markLeaving = useCallback((id, delay = 400) => {
|
||||
setMessages((prev) => {
|
||||
const m = prev.find((x) => x.id === id);
|
||||
if (!m || m.leaving) return prev;
|
||||
return prev.map((x) => (x.id === id ? { ...x, leaving: true } : x));
|
||||
});
|
||||
const remove = setTimeout(() => {
|
||||
setMessages((prev) => prev.filter((x) => x.id !== id));
|
||||
bubbleTimersRef.current.delete(id);
|
||||
}, delay);
|
||||
bubbleTimersRef.current.set(id, { remove });
|
||||
}, []);
|
||||
|
||||
/** delay(默认4s) 后自动淡出；再次调用即续期（清掉旧计时重排）。 */
|
||||
const scheduleBubbleDismiss = useCallback((id, delay = 4000) => {
|
||||
clearBubbleTimers(id);
|
||||
const dismiss = setTimeout(() => {
|
||||
setMessages((prev) => {
|
||||
const m = prev.find((x) => x.id === id);
|
||||
if (!m || m.leaving) return prev;
|
||||
return prev.map((x) => (x.id === id ? { ...x, leaving: true } : x));
|
||||
});
|
||||
const remove = setTimeout(() => {
|
||||
setMessages((prev) => prev.filter((x) => x.id !== id));
|
||||
bubbleTimersRef.current.delete(id);
|
||||
}, 400);
|
||||
bubbleTimersRef.current.set(id, { remove });
|
||||
}, delay);
|
||||
bubbleTimersRef.current.set(id, { dismiss });
|
||||
}, [clearBubbleTimers]);
|
||||
|
||||
const pushMessage = useCallback((role, text, partial = false) => {
|
||||
const key = role === 'user' ? 'user' : 'assistant';
|
||||
let next = [...messagesRef.current];
|
||||
const last = next.length ? next[next.length - 1] : null;
|
||||
if (partial && last && last.role === key && !last.leaving) {
|
||||
// 同一条逐字流：更新文本 + 续期 4s（气泡保持显示直到说完）
|
||||
scheduleBubbleDismiss(last.id);
|
||||
next[next.length - 1] = { ...last, text, partial: true };
|
||||
} else {
|
||||
const id = `${Date.now()}-${Math.random()}`;
|
||||
next.push({ id, role: key, text, partial });
|
||||
// 上限 8 条：最旧优先淡出（保留正在更新的用户气泡）
|
||||
const visible = next.filter((m) => !m.leaving);
|
||||
if (visible.length > 8) {
|
||||
const activeUser = [...visible].reverse().find((m) => m.role === 'user' && m.partial);
|
||||
const victim = visible.find((m) => m !== activeUser) ?? visible[0];
|
||||
if (victim) {
|
||||
next = next.map((m) => (m.id === victim.id ? { ...m, leaving: true } : m));
|
||||
const remove = setTimeout(() => {
|
||||
setMessages((prev) => prev.filter((x) => x.id !== victim.id));
|
||||
bubbleTimersRef.current.delete(victim.id);
|
||||
}, 400);
|
||||
bubbleTimersRef.current.set(victim.id, { remove });
|
||||
}
|
||||
}
|
||||
scheduleBubbleDismiss(id);
|
||||
}
|
||||
messagesRef.current = next;
|
||||
setMessages(next);
|
||||
}, [scheduleBubbleDismiss, markLeaving]);
|
||||
|
||||
// 结束会话：淡出全部气泡（原版 teardown 行为）
|
||||
const dismissAllBubbles = useCallback(() => {
|
||||
const ids = messagesRef.current.map((m) => m.id);
|
||||
messagesRef.current = [];
|
||||
ids.forEach((id) => {
|
||||
if (id !== undefined) markLeaving(id);
|
||||
});
|
||||
}, [markLeaving]);
|
||||
|
||||
// 卸载时清理所有气泡定时器
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
for (const t of bubbleTimersRef.current.values()) {
|
||||
if (t.dismiss) clearTimeout(t.dismiss);
|
||||
if (t.remove) clearTimeout(t.remove);
|
||||
}
|
||||
bubbleTimersRef.current.clear();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// 噪声门弧线：一次性几何（track/fill/hit 同一条 200° 弧 + 阈值圆点）
|
||||
useEffect(() => {
|
||||
const d = fullArcD();
|
||||
mgaTrackRef.current?.setAttribute('d', d);
|
||||
mgaFillRef.current?.setAttribute('d', d);
|
||||
mgaHitRef.current?.setAttribute('d', d);
|
||||
const fill = mgaFillRef.current;
|
||||
if (fill) {
|
||||
fill.setAttribute('pathLength', '100');
|
||||
fill.style.strokeDasharray = '100 100';
|
||||
fill.style.strokeDashoffset = '100'; // 无电平时空弧
|
||||
}
|
||||
const p = arcPoint(dbToFraction(DEFAULT_GATE_DB));
|
||||
mgaHandleRef.current?.setAttribute('cx', String(p.x));
|
||||
mgaHandleRef.current?.setAttribute('cy', String(p.y));
|
||||
}, []);
|
||||
|
||||
// 实时麦克风电平 → 弧线填充 + 过门点亮（复刻原版 paintInputLevel）
|
||||
const paintInputLevel = useCallback((rms) => {
|
||||
if (!mgaFillRef.current || !micGateRef.current) return;
|
||||
const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB;
|
||||
const f = dbToFraction(db);
|
||||
mgaFillRef.current.style.strokeDashoffset = String(100 * (1 - f));
|
||||
const enabled = DEFAULT_GATE_DB > GATE_OFF_DB;
|
||||
micGateRef.current.classList.toggle('gate-open', enabled && f >= dbToFraction(DEFAULT_GATE_DB));
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (clientRef.current) return;
|
||||
setState('connecting');
|
||||
try {
|
||||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error('非安全上下文：请通过 http://localhost 或 HTTPS 访问，浏览器才允许麦克风');
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
streamRef.current = stream;
|
||||
if (!audioCtxRef.current) audioCtxRef.current = new AudioContext();
|
||||
if (audioCtxRef.current.state === 'suspended') await audioCtxRef.current.resume();
|
||||
|
||||
const client = new S2sWsRealtimeClient({
|
||||
directUrl: VOICE_URL,
|
||||
voice: DEFAULT_VOICE,
|
||||
instructions: DEFAULT_INSTRUCTIONS,
|
||||
tools: [],
|
||||
noiseGate: { enabled: true, thresholdDb: DEFAULT_GATE_DB },
|
||||
audioContext: audioCtxRef.current,
|
||||
micStream: stream,
|
||||
});
|
||||
const statusMap = {
|
||||
connected: 'listening', 'user-speaking': 'user-speaking',
|
||||
processing: 'processing', 'ai-speaking': 'ai-speaking', idle: 'idle',
|
||||
};
|
||||
client.addEventListener('status', (e) => {
|
||||
const s = e.detail?.status;
|
||||
setState(statusMap[s] ?? 'listening');
|
||||
});
|
||||
client.addEventListener('transcript', (e) => {
|
||||
const d = e.detail;
|
||||
pushMessage(d.role === 'user' ? 'user' : 'assistant', d.text, d.partial);
|
||||
// 用户开始说话（回答了问候）→ 取消挂断倒计时
|
||||
if (d.role === 'user' && d.text.trim() && greetingTimerRef.current) {
|
||||
clearTimeout(greetingTimerRef.current);
|
||||
greetingTimerRef.current = null;
|
||||
console.warn('[vision] 用户已回答，取消挂断倒计时');
|
||||
}
|
||||
});
|
||||
client.addEventListener('input-level', (e) => paintInputLevel(e.detail?.rms));
|
||||
client.addEventListener('response-finished', () => {
|
||||
setState('listening');
|
||||
// 自动问候的回复播放完毕 → 启动"等待用户回答"倒计时；无回答则自动挂断
|
||||
if (greetingSentRef.current) {
|
||||
greetingSentRef.current = false;
|
||||
if (greetingTimerRef.current) clearTimeout(greetingTimerRef.current);
|
||||
greetingTimerRef.current = setTimeout(() => {
|
||||
console.warn('[vision] 问候后用户未回答，自动挂断回初始界面');
|
||||
void stop();
|
||||
}, GREETING_WAIT_MS);
|
||||
}
|
||||
});
|
||||
client.addEventListener('error', () => setState('error'));
|
||||
|
||||
clientRef.current = client;
|
||||
await client.connect();
|
||||
setState('listening');
|
||||
// 自动问候：连接成功后发送问候（语音）
|
||||
if (pendingGreetingRef.current) sendGreeting();
|
||||
} catch (err) {
|
||||
setErrorMsg(err?.message || String(err));
|
||||
setState('error');
|
||||
}
|
||||
}, [pushMessage, paintInputLevel, sendGreeting]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
// 清理自动问候相关状态/倒计时
|
||||
if (greetingTimerRef.current) {
|
||||
clearTimeout(greetingTimerRef.current);
|
||||
greetingTimerRef.current = null;
|
||||
}
|
||||
pendingGreetingRef.current = false;
|
||||
greetingSentRef.current = false;
|
||||
const c = clientRef.current;
|
||||
clientRef.current = null;
|
||||
if (c) { try { await c.close(); } catch { /* noop */ } }
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
setMuted(false);
|
||||
setErrorMsg('');
|
||||
dismissAllBubbles();
|
||||
setState('idle');
|
||||
}, [dismissAllBubbles]);
|
||||
|
||||
const toggleMute = useCallback(() => {
|
||||
const c = clientRef.current;
|
||||
if (!c) return;
|
||||
setMuted((m) => { c.setMuted(!m); return !m; });
|
||||
}, []);
|
||||
|
||||
// ── 视觉识别触发：主动问候 + 后端记录/广播 ─────────────────────────────
|
||||
// 完整生命周期：
|
||||
// 触发（仅 idle 状态）→ 打开对话(start) → 连接成功发问候(sendUserText→LLM→语音)
|
||||
// → 回复播放完(response-finished) → 等待用户回答 25s → 无回答则自动挂断回 idle
|
||||
// → 静默 3 分钟（hook VISION_SILENT_MS）期间不再打扰
|
||||
// 循环依赖：onVisionTrigger 需读取 vision.faces，而它又要作为参数传给 useVisionDetection。
|
||||
// 解法：hook 的 onTrigger/onGesture 从 ref 取最新回调；回调在 vision 声明之后再定义。
|
||||
const visionTriggerRef = useRef(null);
|
||||
const gestureRef = useRef(null);
|
||||
|
||||
const vision = useVisionDetection({
|
||||
onTrigger: () => visionTriggerRef.current?.(),
|
||||
onGesture: (g) => gestureRef.current?.(g),
|
||||
});
|
||||
|
||||
// 手势 → 对话控制：举手 toggle —— 无对话则开始，正在对话则结束
|
||||
const onGesture = useCallback((g) => {
|
||||
if (g !== 'raise') return;
|
||||
if (clientRef.current) {
|
||||
void stop();
|
||||
} else {
|
||||
void start();
|
||||
}
|
||||
}, [start, stop]);
|
||||
|
||||
useEffect(() => {
|
||||
gestureRef.current = onGesture;
|
||||
}, [onGesture]);
|
||||
|
||||
const onVisionTrigger = useCallback(() => {
|
||||
// 仅 idle 允许自动问候：对话中/连接中/其他状态一律不打扰
|
||||
if (voiceStateRef.current !== 'idle') {
|
||||
console.warn('[vision] 自动问候跳过：当前状态', voiceStateRef.current);
|
||||
return;
|
||||
}
|
||||
pendingGreetingRef.current = true;
|
||||
if (clientRef.current) {
|
||||
sendGreeting();
|
||||
} else {
|
||||
void start(); // start 连接成功后检查 pendingGreetingRef 再发问候
|
||||
}
|
||||
fetch(`${API_BASE}/api/vision/event`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ event: 'triggered', faces: vision.faces, dwell_ms: VISION_DWELL_MS }),
|
||||
}).catch(() => { /* 上报失败不影响本地问候 */ });
|
||||
}, [vision.faces, start, sendGreeting]);
|
||||
|
||||
useEffect(() => {
|
||||
visionTriggerRef.current = onVisionTrigger;
|
||||
}, [onVisionTrigger]);
|
||||
|
||||
// 摄像头识别状态变化 → 上报后端（详细日志定位）
|
||||
useEffect(() => {
|
||||
if (!vision.status || vision.status === 'off' || vision.status === 'loading') return;
|
||||
fetch(`${API_BASE}/api/vision/event`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
event: vision.status === 'model-error' ? 'model_error' : vision.status,
|
||||
faces: vision.faces,
|
||||
dwell_ms: vision.dwellMs,
|
||||
detail: vision.errorInfo ? `${vision.errorInfo.name}: ${vision.errorInfo.message}` : '',
|
||||
}),
|
||||
}).catch(() => { /* 静默 */ });
|
||||
}, [vision.status]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
const visionStatusText = {
|
||||
off: '摄像头关闭',
|
||||
loading: '摄像头启动中…',
|
||||
detecting: `识别中 · ${vision.faces} 人${vision.faces > 0 ? `（后端 ${vision.latency ?? '-'}ms）` : ' · 未检到人脸'} · 链路${vision.modelPhase === 'ok' ? 'OK' : vision.modelPhase === 'failed' ? `失败:${(vision.modelErr || '').slice(0, 40)}` : '连接中'} · 亮度${vision.frameLum ?? '-'}${vision.detectErr ? ` · 异常:${String(vision.detectErr).slice(0, 50)}` : ''}`,
|
||||
facing: `面向大屏 ${(vision.dwellMs / 1000).toFixed(1)}s / ${VISION_DWELL_MS / 1000}s`,
|
||||
triggered: '已触发 · 主动问候',
|
||||
silent: '静默中',
|
||||
error: '摄像头不可用',
|
||||
}[vision.status] ?? '';
|
||||
|
||||
// 摄像头失败的具体原因
|
||||
const visionErrorText = (() => {
|
||||
if (vision.status !== 'error') return null;
|
||||
const name = vision.errorInfo?.name ?? '';
|
||||
const msg = vision.errorInfo?.message ?? '';
|
||||
if (!window.isSecureContext) return '非安全上下文：请用 localhost 或 HTTPS 访问';
|
||||
if (name === 'NotFoundError') return `未检测到摄像头设备：${msg || '请确认 USB 摄像头已接入本机'}`;
|
||||
if (name === 'NotAllowedError') return `摄像头权限被拒：请在浏览器地址栏允许 ${location.host} 使用摄像头`;
|
||||
if (name === 'NotReadableError') return '摄像头被其他应用占用或不可用，请关闭占用程序后刷新';
|
||||
return `摄像头启动失败（${name}）：${msg}`;
|
||||
})();
|
||||
|
||||
const caption = state === 'error' ? (errorMsg || 'Tap to retry') : (STATE_VIEWS[state] ?? '');
|
||||
|
||||
return (
|
||||
<div id="app">
|
||||
{/* 非安全上下文提示（局域网 IP 访问时浏览器禁用麦克风/摄像头） */}
|
||||
{!window.isSecureContext && (
|
||||
<div className="voice-secure-warn">
|
||||
⚠️ 当前页面不是安全上下文，浏览器禁止麦克风/摄像头。
|
||||
请改用 <b>http://localhost</b> 访问（或为后端配置 HTTPS）。
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 主舞台：中间圆球 */}
|
||||
<main className="stage">
|
||||
<div className={`orb-wrap${live ? ' live' : ''}`}>
|
||||
<div id="mic-gate" className="mic-gate" ref={micGateRef}>
|
||||
<svg id="mic-gate-arc" className="mic-gate-arc" viewBox="0 0 100 100" aria-hidden="true">
|
||||
<path id="mga-track" className="mga-track" fill="none" ref={mgaTrackRef} />
|
||||
<path id="mga-fill" className="mga-fill" fill="none" ref={mgaFillRef} />
|
||||
<path id="mga-hit" className="mga-hit" fill="none" ref={mgaHitRef} />
|
||||
<circle id="mga-handle" className="mga-handle" r="3" ref={mgaHandleRef} />
|
||||
</svg>
|
||||
<button
|
||||
id="mic-btn"
|
||||
className={`side-btn${muted ? ' muted' : ''}`}
|
||||
type="button"
|
||||
aria-label={muted ? 'Unmute' : 'Mute'}
|
||||
title={muted ? 'Unmute' : 'Mute'}
|
||||
aria-hidden={!live}
|
||||
tabIndex={live ? 0 : -1}
|
||||
onClick={toggleMute}
|
||||
>
|
||||
{icons.micOn}
|
||||
{icons.micOff}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<button
|
||||
id="main-circle"
|
||||
className={`circle ${STATE_CLASS[state] ?? 'state-idle'}`}
|
||||
type="button"
|
||||
aria-label="Start voice conversation"
|
||||
disabled={state === 'connecting'}
|
||||
onClick={live ? stop : start}
|
||||
>
|
||||
<span className="circle-glow" aria-hidden="true" />
|
||||
<span className="circle-ring" aria-hidden="true" />
|
||||
<span className="circle-ring-outer" aria-hidden="true" />
|
||||
<span className="circle-core">
|
||||
<span className="circle-indicator" aria-hidden="true">
|
||||
{icons.indMic}
|
||||
{icons.indError}
|
||||
<span className="ind ind-spinner" />
|
||||
<span className="ind ind-thinking"><span className="dot" /><span className="dot" /><span className="dot" /></span>
|
||||
<span className="ind ind-bars"><span className="bar" /><span className="bar" /><span className="bar" /><span className="bar" /><span className="bar" /></span>
|
||||
{icons.indVoice}
|
||||
</span>
|
||||
</span>
|
||||
</button>
|
||||
|
||||
<button
|
||||
id="stop-btn"
|
||||
className="side-btn"
|
||||
type="button"
|
||||
aria-label="End"
|
||||
title="End"
|
||||
aria-hidden={!live}
|
||||
tabIndex={live ? 0 : -1}
|
||||
onClick={stop}
|
||||
>
|
||||
{icons.stop}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<p id="circle-caption" className={`circle-caption${caption ? '' : ' empty'}${state === 'error' ? ' error' : ''}`} role="status">{caption}</p>
|
||||
<p id="circle-subcaption" className="circle-subcaption" hidden />
|
||||
</main>
|
||||
|
||||
{/* 页脚：园区概览条（与数据大屏 / 媒体播放页一致） */}
|
||||
<ParkOverviewStrip d={d} />
|
||||
|
||||
{/* 右下角气泡（复刻原版：进场 in / 4s 自动淡出 out / 阶梯字号） */}
|
||||
<div id="bubble-stack" className="bubble-stack" aria-live="polite" aria-atomic="false">
|
||||
{messages.map((m) => <Bubble key={m.id} m={m} />)}
|
||||
</div>
|
||||
|
||||
{/* 视觉识别：摄像头默认开启；video 始终渲染（ref 先于启动存在），未开启时隐藏 */}
|
||||
<div className={`voice-cam-pip${vision.cameraOn ? '' : ' hidden'}`}>
|
||||
<video ref={vision.videoRef} autoPlay playsInline muted />
|
||||
<div className="voice-cam-bar">
|
||||
<span className="voice-cam-title">
|
||||
<span className={`voice-cam-dot ${vision.status === 'triggered' ? 'trigger' : vision.status === 'facing' ? 'face' : 'on'}`} />
|
||||
智能识别中
|
||||
</span>
|
||||
<span>
|
||||
{vision.gesture === 'raise' && (
|
||||
<b style={{ color: '#2f6bff' }}>您好呀 · {live ? '开始对话' : '结束对话'}</b>
|
||||
)}
|
||||
{visionStatusText}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
{vision.status === 'error' && !vision.cameraOn && (
|
||||
<div className="voice-vision-status" style={{ bottom: 62 }}>⚠️ {visionErrorText}</div>
|
||||
)}
|
||||
{vision.status === 'facing' && (
|
||||
<div className="voice-vision-status">
|
||||
检测到 <b>{vision.faces} 人</b>，正对摄像头 <b>{(vision.dwellMs / 1000).toFixed(1)}s</b>，停留 {VISION_DWELL_MS / 1000}s 后自动问候
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
/* 实时语音对话页：iframe 占满内容区（页眉由 ScreenLayout 注入） */
|
||||
.voice-embed {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
.voice-embed iframe {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: none;
|
||||
display: block;
|
||||
background: #0d1117; /* 与原对话页深色背景一致 */
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
/* 实时语音对话页覆盖样式：
|
||||
1) 圆球在内容区垂直+水平居中（原版 #app 是 100vh，在 DPM 页眉下会偏高）
|
||||
2) 视觉识别：摄像头开关 / 预览 / 识别状态
|
||||
3) 底部园区概览条（复用 bd-strip，样式来自 datascreen.css） */
|
||||
#app {
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
/* stage 弹性撑满（页眉 + 页脚之外的剩余视窗），内部 flex 居中 orb-wrap */
|
||||
.stage {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* ---- 圆球下方状态文字 ----
|
||||
原版 --text-faint 是半透明白（为深色背景设计），在 DPM 浅蓝白背景下不可见。
|
||||
改为 DPM 深蓝墨色，与页眉/概览条同源（--bd-ink），对比明显且不接近背景色。 */
|
||||
.circle-caption {
|
||||
color: var(--bd-ink, #1d2c44);
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
opacity: 1;
|
||||
}
|
||||
.circle-caption.empty {
|
||||
opacity: 0;
|
||||
}
|
||||
.circle-caption.error {
|
||||
color: #b42318;
|
||||
opacity: 1;
|
||||
}
|
||||
.circle-caption.muted {
|
||||
color: #4a5b74;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* ---- 视觉识别 UI ---- */
|
||||
.voice-cam-toggle {
|
||||
position: fixed;
|
||||
right: 18px;
|
||||
top: 76px; /* DPM 页眉下方 */
|
||||
z-index: 50;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 7px 12px;
|
||||
border: 1px solid rgba(47, 107, 255, 0.35);
|
||||
border-radius: 20px;
|
||||
background: rgba(255, 255, 255, 0.92);
|
||||
color: #1d2c44;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
box-shadow: 0 4px 12px rgba(29, 79, 216, 0.12);
|
||||
}
|
||||
.voice-cam-toggle:hover { border-color: #2f6bff; }
|
||||
.voice-cam-toggle.on { background: #2f6bff; color: #fff; border-color: #2f6bff; }
|
||||
|
||||
/* 底部左侧摄像头实时窗口（放大，实时画面） */
|
||||
.voice-cam-pip {
|
||||
position: fixed;
|
||||
left: 18px;
|
||||
bottom: 62px; /* 上移避开底部园区概览条（bd-strip 约 44px 高 + 间隙） */
|
||||
z-index: 50;
|
||||
width: 380px;
|
||||
border-radius: 14px;
|
||||
overflow: hidden;
|
||||
border: 1px solid rgba(47, 107, 255, 0.45);
|
||||
box-shadow: 0 10px 28px rgba(29, 79, 216, 0.22);
|
||||
background: #0d1117;
|
||||
}
|
||||
.voice-cam-pip video {
|
||||
width: 100%;
|
||||
height: 216px;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
transform: scaleX(-1); /* 镜像自拍视角 */
|
||||
}
|
||||
.voice-cam-pip .voice-cam-bar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
padding: 7px 10px;
|
||||
font-size: 12px;
|
||||
color: #e6e8eb;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
.voice-cam-pip .voice-cam-title {
|
||||
display: inline-flex; align-items: center; gap: 6px; font-weight: 600;
|
||||
}
|
||||
.voice-cam-pip .voice-cam-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #7d8fa9;
|
||||
}
|
||||
.voice-cam-pip .voice-cam-dot.on { background: #17b26a; box-shadow: 0 0 0 3px rgba(23, 178, 106, 0.25); }
|
||||
.voice-cam-pip .voice-cam-dot.face { background: #4cc2ff; box-shadow: 0 0 0 3px rgba(76, 194, 255, 0.25); }
|
||||
.voice-cam-pip .voice-cam-dot.trigger { background: #f79009; box-shadow: 0 0 0 3px rgba(247, 144, 9, 0.3); }
|
||||
|
||||
/* 识别状态横幅（左下角预览上方） */
|
||||
.voice-vision-status {
|
||||
position: fixed;
|
||||
left: 18px;
|
||||
bottom: 200px; /* 跟随预览上移 44px，保持相对位置 */
|
||||
z-index: 50;
|
||||
font-size: 12px;
|
||||
color: #1d2c44;
|
||||
background: rgba(255, 255, 255, 0.92);
|
||||
border: 1px solid rgba(47, 107, 255, 0.25);
|
||||
border-radius: 8px;
|
||||
padding: 6px 10px;
|
||||
box-shadow: 0 4px 12px rgba(29, 79, 216, 0.12);
|
||||
line-height: 1.5;
|
||||
}
|
||||
.voice-vision-status b { color: #2f6bff; }
|
||||
|
||||
/* 非安全上下文警告条 */
|
||||
.voice-secure-warn {
|
||||
position: fixed;
|
||||
top: 76px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
z-index: 60;
|
||||
font-size: 12px;
|
||||
color: #b42318;
|
||||
background: #fef3f2;
|
||||
border: 1px solid #fda29b;
|
||||
border-radius: 8px;
|
||||
padding: 6px 12px;
|
||||
box-shadow: 0 4px 12px rgba(180, 35, 24, 0.12);
|
||||
}
|
||||
.voice-cam-pip.hidden { display: none; }
|
||||
@@ -12,6 +12,7 @@ export const PAGE_ORDER = [
|
||||
{ path: '/', label: '数据大屏' },
|
||||
{ path: '/twin', label: '数字孪生' },
|
||||
{ path: '/ai', label: 'AI 助手' },
|
||||
{ path: '/voice', label: '语音对话' },
|
||||
{ path: '/screen', label: '媒体轮播' },
|
||||
];
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM
|
||||
* conversion for the audio frames on the wire, transcript extraction from a
|
||||
* `response.done` payload, and a tiny URL helper. Kept separate from the client
|
||||
* so the protocol/state logic stays readable.
|
||||
*/
|
||||
|
||||
/** @param {string} url */
|
||||
export function trimTrailingSlash(url) {
|
||||
return url.endsWith("/") ? url.slice(0, -1) : url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the assistant transcript out of a `response.done` payload. The text
|
||||
* lives in `response.output[].content[].transcript` (audio) or `.text`. Used as
|
||||
* the source of truth for interrupted replies, where the dedicated
|
||||
* `*.transcript.done` event may never arrive.
|
||||
* @param {any} response
|
||||
* @returns {string}
|
||||
*/
|
||||
export function extractResponseTranscript(response) {
|
||||
const output = response?.output;
|
||||
if (!Array.isArray(output)) return "";
|
||||
/** @type {string[]} */
|
||||
const parts = [];
|
||||
for (const item of output) {
|
||||
for (const part of item?.content ?? []) {
|
||||
const text = part?.transcript ?? part?.text;
|
||||
if (typeof text === "string" && text.trim()) parts.push(text.trim());
|
||||
}
|
||||
}
|
||||
return parts.join(" ").trim();
|
||||
}
|
||||
|
||||
/** @param {ArrayBuffer} buf */
|
||||
export function base64FromArrayBuffer(buf) {
|
||||
const bytes = new Uint8Array(buf);
|
||||
// Chunked encoding so we don't blow up the call stack on long buffers.
|
||||
let binary = "";
|
||||
const chunk = 0x8000;
|
||||
for (let i = 0; i < bytes.length; i += chunk) {
|
||||
binary += String.fromCharCode.apply(null, /** @type {number[]} */ (
|
||||
/** @type {unknown} */ (bytes.subarray(i, i + chunk))
|
||||
));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/** @param {string} b64 */
|
||||
export function base64ToBytes(b64) {
|
||||
const binary = atob(b64);
|
||||
const len = binary.length;
|
||||
const out = new Uint8Array(len);
|
||||
for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the
|
||||
* mic input and the TTS output) and maps the low-frequency speech energy onto
|
||||
* the orb's CSS custom properties:
|
||||
* - `--bar0`..`--bar4` the 5-band level meter
|
||||
* - `--ai-audio-level` the global "Reachy talks" glow / scale pulse
|
||||
*
|
||||
* The bottom of the FFT is where speech energy lives, so the band edges stay
|
||||
* low — that keeps the bars dancing on voice rather than on noise. While the AI
|
||||
* is speaking we source the bars from the OUTPUT analyser so the orb pulses with
|
||||
* Reachy's voice instead of sitting dead while the user is silent.
|
||||
*/
|
||||
|
||||
// Exported so the client can size its AnalyserNodes to match our buffer.
|
||||
export const VIS_FFT_SIZE = 256;
|
||||
const VIS_BAND_COUNT = 5;
|
||||
const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52];
|
||||
const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy)
|
||||
const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade)
|
||||
|
||||
export class OrbVisualiser {
|
||||
/**
|
||||
* @param {AnalyserNode} micAnalyser
|
||||
* @param {AnalyserNode} outAnalyser
|
||||
* @param {() => boolean} isAiSpeaking Source the bars from the AI output when
|
||||
* true, otherwise from the mic.
|
||||
*/
|
||||
constructor(micAnalyser, outAnalyser, isAiSpeaking) {
|
||||
this._mic = micAnalyser;
|
||||
this._out = outAnalyser;
|
||||
this._isAiSpeaking = isAiSpeaking;
|
||||
this._buf = new Uint8Array(micAnalyser.frequencyBinCount);
|
||||
this._bands = new Float32Array(VIS_BAND_COUNT);
|
||||
this._aiLevel = 0;
|
||||
/** @type {number | null} */
|
||||
this._frame = null;
|
||||
}
|
||||
|
||||
/** Begin the rAF loop (idempotent). */
|
||||
start() {
|
||||
if (this._frame !== null) return;
|
||||
const root = document.documentElement;
|
||||
const tick = () => {
|
||||
this._frame = requestAnimationFrame(tick);
|
||||
this._update(root);
|
||||
};
|
||||
this._frame = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
/** Stop the loop and clear the CSS vars so the orb returns to rest. */
|
||||
stop() {
|
||||
if (this._frame !== null) {
|
||||
cancelAnimationFrame(this._frame);
|
||||
this._frame = null;
|
||||
}
|
||||
const root = document.documentElement;
|
||||
for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`);
|
||||
root.style.removeProperty("--ai-audio-level");
|
||||
}
|
||||
|
||||
/** @param {HTMLElement} root */
|
||||
_update(root) {
|
||||
// Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars.
|
||||
const source = this._isAiSpeaking() ? this._out : this._mic;
|
||||
source.getByteFrequencyData(this._buf);
|
||||
|
||||
for (let b = 0; b < VIS_BAND_COUNT; b++) {
|
||||
const lo = VIS_BAND_EDGES[b];
|
||||
const hi = VIS_BAND_EDGES[b + 1];
|
||||
let sum = 0;
|
||||
let n = 0;
|
||||
for (let i = lo; i < hi && i < this._buf.length; i++) {
|
||||
sum += this._buf[i];
|
||||
n += 1;
|
||||
}
|
||||
const target = n > 0 ? sum / (n * 255) : 0;
|
||||
const prev = this._bands[b];
|
||||
const k = target > prev ? VIS_ATTACK : VIS_RELEASE;
|
||||
const next = prev + (target - prev) * k;
|
||||
this._bands[b] = next;
|
||||
root.style.setProperty(`--bar${b}`, next.toFixed(3));
|
||||
}
|
||||
|
||||
// Global AI audio level: peak of the output analyser, used by the CSS to
|
||||
// make the orb's glow / scale react to Reachy's voice.
|
||||
this._out.getByteFrequencyData(this._buf);
|
||||
let peak = 0;
|
||||
const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]);
|
||||
for (let i = 0; i < limit; i++) {
|
||||
if (this._buf[i] > peak) peak = this._buf[i];
|
||||
}
|
||||
const aiTarget = peak / 255;
|
||||
const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE;
|
||||
this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k;
|
||||
root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,243 @@
|
||||
/* =========================================================
|
||||
摄像头实时识别 Hook —— YOLO 人脸检测（后端推理）
|
||||
链路：前端 getUserMedia 本地预览 → 每 ~700ms canvas 抽帧 → JPEG base64
|
||||
POST {API_BASE}/api/vision/frame → 后端 YOLO(yolov8n-face) 推理
|
||||
→ 返回人脸数/框 → 前端驱动状态机
|
||||
- 状态机：检测到人脸（面向大屏）持续 ≥10s → onTrigger；触发后静默 60s
|
||||
- 帧经局域网传到后端（大屏机同机部署则不出设备）；画面预览仍本地
|
||||
========================================================= */
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { API_BASE } from '../config';
|
||||
|
||||
export const VISION_DWELL_MS = 10000; // 面向停留阈值
|
||||
export const VISION_SILENT_MS = 180000; // 触发（自动问候）后静默 3 分钟，期间不重复触发
|
||||
const DETECT_INTERVAL_MS = 700; // ~1.4fps（后端推理 ~100-200ms + 传输）
|
||||
const FRAME_W = 480;
|
||||
const FRAME_H = 360;
|
||||
|
||||
export default function useVisionDetection({ onTrigger, onGesture, enabled = true }) {
|
||||
const [cameraOn, setCameraOn] = useState(false);
|
||||
const [status, setStatus] = useState('off'); // off|loading|detecting|facing|triggered|silent|error
|
||||
const [faces, setFaces] = useState(0);
|
||||
const [dwellMs, setDwellMs] = useState(0);
|
||||
const [gesture, setGesture] = useState(null); // raise(举手，toggle 对话)|null
|
||||
const [errorInfo, setErrorInfo] = useState(null);
|
||||
// 诊断：抽帧画面平均亮度（本地计算，<15 ≈ 黑屏）
|
||||
const [frameLum, setFrameLum] = useState(null);
|
||||
// 诊断：后端推理链路状态 loading|ok|failed
|
||||
const [modelPhase, setModelPhase] = useState('loading');
|
||||
const [modelErr, setModelErr] = useState(null);
|
||||
// 诊断：最近一次推理/请求异常
|
||||
const [detectErr, setDetectErr] = useState(null);
|
||||
// 诊断：后端推理耗时 ms
|
||||
const [latency, setLatency] = useState(null);
|
||||
|
||||
const videoRef = useRef(null);
|
||||
const streamRef = useRef(null);
|
||||
const timerRef = useRef(null);
|
||||
const canvasRef = useRef(null);
|
||||
const cancelledRef = useRef(false); // StrictMode/HMR 卸载标记
|
||||
const stateRef = useRef({
|
||||
facingSince: 0, silentUntil: 0, lastDetect: 0, lastDiag: 0, lastPlayRetry: 0, failCount: 0,
|
||||
// 手势状态机：举手持续 1.5s → raise（前端据此 toggle 对话：无对话开、有对话停）
|
||||
raiseActive: false, raiseStart: 0, raiseTriggered: false,
|
||||
});
|
||||
const triggerRef = useRef(onTrigger);
|
||||
triggerRef.current = onTrigger;
|
||||
const gestureRef = useRef(onGesture);
|
||||
gestureRef.current = onGesture;
|
||||
|
||||
const loop = useCallback(async () => {
|
||||
if (cancelledRef.current) return;
|
||||
const video = videoRef.current;
|
||||
const now = Date.now();
|
||||
const st = stateRef.current;
|
||||
if (!video || video.readyState < 2) {
|
||||
// 视频未就绪：兜底重试 play()（Safari 不自动播放 srcObject 流）
|
||||
if (video && video.srcObject && now - st.lastPlayRetry > 2000) {
|
||||
st.lastPlayRetry = now;
|
||||
video.play().catch(() => { /* 继续等待 */ });
|
||||
}
|
||||
timerRef.current = setTimeout(loop, 400);
|
||||
return;
|
||||
}
|
||||
if (now - st.lastDetect < DETECT_INTERVAL_MS) {
|
||||
timerRef.current = setTimeout(loop, 120);
|
||||
return;
|
||||
}
|
||||
st.lastDetect = now;
|
||||
|
||||
try {
|
||||
// 抽帧
|
||||
if (!canvasRef.current) {
|
||||
canvasRef.current = document.createElement('canvas');
|
||||
canvasRef.current.width = FRAME_W;
|
||||
canvasRef.current.height = FRAME_H;
|
||||
}
|
||||
const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true });
|
||||
ctx.drawImage(video, 0, 0, FRAME_W, FRAME_H);
|
||||
|
||||
// 亮度诊断（本地，每 2s）
|
||||
if (now - st.lastDiag > 2000) {
|
||||
st.lastDiag = now;
|
||||
try {
|
||||
const d = ctx.getImageData(0, 0, FRAME_W, FRAME_H).data;
|
||||
let s = 0;
|
||||
for (let i = 0; i < d.length; i += 4) s += d[i] + d[i + 1] + d[i + 2];
|
||||
setFrameLum(Math.round(s / (d.length / 4) / 3));
|
||||
} catch {
|
||||
setFrameLum(-1);
|
||||
}
|
||||
}
|
||||
|
||||
// JPEG base64 → 后端 YOLO
|
||||
const b64 = canvasRef.current.toDataURL('image/jpeg', 0.6).split(',')[1];
|
||||
setModelPhase('ok');
|
||||
const res = await fetch(`${API_BASE}/api/vision/frame`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ image: b64 }),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!data?.ok) throw new Error(data?.error || `HTTP ${res.status}`);
|
||||
setDetectErr(null);
|
||||
const n = data.faces ?? 0;
|
||||
setFaces(n);
|
||||
setLatency(data.latency_ms);
|
||||
st.failCount = 0;
|
||||
|
||||
// ── 手势状态机：举手持续 1.5s → raise；前端据此 toggle（无对话开 / 有对话停）──
|
||||
const raised = data.raised?.[0]; // 取第一个举手的人/手
|
||||
if (raised) {
|
||||
if (!st.raiseActive) {
|
||||
st.raiseActive = true;
|
||||
st.raiseStart = now;
|
||||
st.raiseTriggered = false;
|
||||
}
|
||||
if (!st.raiseTriggered && now - st.raiseStart >= 1500) {
|
||||
st.raiseTriggered = true;
|
||||
setGesture('raise');
|
||||
console.warn(`[vision] 手势：举手（持续 ${Math.round((now - st.raiseStart) / 1000)}s）→ toggle 对话`);
|
||||
gestureRef.current?.('raise');
|
||||
}
|
||||
} else if (st.raiseActive) {
|
||||
// 举手消失：重置，允许下次举手再次触发
|
||||
st.raiseActive = false;
|
||||
st.raiseStart = 0;
|
||||
st.raiseTriggered = false;
|
||||
setTimeout(() => setGesture(null), 2500);
|
||||
}
|
||||
|
||||
// 状态机：有人脸（面向大屏）持续 10s → 触发问候；触发后静默 60s
|
||||
if (now < st.silentUntil) {
|
||||
st.facingSince = 0;
|
||||
setDwellMs(0);
|
||||
setStatus('silent');
|
||||
} else if (n > 0) {
|
||||
st.facingSince = st.facingSince || now;
|
||||
const d = now - st.facingSince;
|
||||
setDwellMs(d);
|
||||
setStatus('facing');
|
||||
if (d >= VISION_DWELL_MS) {
|
||||
st.silentUntil = now + VISION_SILENT_MS;
|
||||
st.facingSince = 0;
|
||||
setDwellMs(0);
|
||||
setStatus('triggered');
|
||||
triggerRef.current?.();
|
||||
setTimeout(() => setStatus('detecting'), 3000);
|
||||
}
|
||||
} else {
|
||||
st.facingSince = 0;
|
||||
setDwellMs(0);
|
||||
setStatus('detecting');
|
||||
}
|
||||
} catch (e) {
|
||||
// 后端不可达/推理失败：错误显示到页面，连续 5 次标记链路失败
|
||||
setDetectErr(e?.message || String(e));
|
||||
st.failCount += 1;
|
||||
if (st.failCount >= 5) {
|
||||
setModelPhase('failed');
|
||||
setModelErr(e?.message || String(e));
|
||||
}
|
||||
}
|
||||
timerRef.current = setTimeout(loop, 120);
|
||||
}, []);
|
||||
|
||||
const startCamera = useCallback(async () => {
|
||||
if (cameraOn || !enabled) return;
|
||||
cancelledRef.current = false;
|
||||
setStatus('loading');
|
||||
setModelPhase('loading');
|
||||
setModelErr(null);
|
||||
setDetectErr(null);
|
||||
try {
|
||||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error('非安全上下文：请通过 http://localhost 或 HTTPS 访问（当前地址浏览器禁止摄像头）');
|
||||
}
|
||||
// 设备预检
|
||||
try {
|
||||
const devs = await navigator.mediaDevices.enumerateDevices();
|
||||
if (!devs.some((d) => d.kind === 'videoinput')) {
|
||||
const e = new Error('未检测到摄像头设备（请确认 USB 摄像头已接入本机）');
|
||||
e.name = 'NotFoundError';
|
||||
throw e;
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.name === 'NotFoundError') throw e;
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: 640, height: 480 },
|
||||
audio: false,
|
||||
});
|
||||
if (cancelledRef.current) {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
return;
|
||||
}
|
||||
streamRef.current = stream;
|
||||
const video = videoRef.current;
|
||||
if (!video) throw new Error('video element not mounted');
|
||||
video.srcObject = stream;
|
||||
try { await video.play(); } catch { /* muted autoplay 兜底 */ }
|
||||
if (cancelledRef.current) return;
|
||||
setCameraOn(true);
|
||||
setStatus('detecting');
|
||||
timerRef.current = setTimeout(loop, 300);
|
||||
} catch (e) {
|
||||
if (cancelledRef.current) return;
|
||||
setErrorInfo({ name: e?.name ?? 'Error', message: e?.message ?? String(e) });
|
||||
setStatus('error');
|
||||
console.warn('[vision] 摄像头启动失败:', e?.name, e?.message);
|
||||
}
|
||||
}, [cameraOn, enabled, loop]);
|
||||
|
||||
const stopCamera = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
if (timerRef.current) clearTimeout(timerRef.current);
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
stateRef.current = { facingSince: 0, silentUntil: 0, lastDetect: 0, lastDiag: 0, lastPlayRetry: 0, failCount: 0,
|
||||
raiseActive: false, raiseStart: 0, raiseTriggered: false };
|
||||
setCameraOn(false);
|
||||
setFaces(0);
|
||||
setDwellMs(0);
|
||||
setGesture(null);
|
||||
setFrameLum(null);
|
||||
setModelPhase('loading');
|
||||
setModelErr(null);
|
||||
setDetectErr(null);
|
||||
setLatency(null);
|
||||
setStatus('off');
|
||||
}, []);
|
||||
|
||||
// 默认开启：挂载即启动；卸载时释放
|
||||
useEffect(() => {
|
||||
if (enabled) void startCamera();
|
||||
return () => stopCamera();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [enabled]);
|
||||
|
||||
return {
|
||||
cameraOn, status, faces, dwellMs, gesture, latency, frameLum, modelPhase, modelErr, detectErr,
|
||||
errorInfo, startCamera, stopCamera, videoRef,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user