feat(voice): 实时语音对话页（React 完全复刻原版 UI）
- /voice 页面：中间圆球 + 左右按钮 + 状态图标 + 右下角气泡（进场/淡出/阶梯字号/上限8）+ 噪声门弧线电平，结构类名与原版一致 - s2s WebSocket 客户端 + worklet 音频（mic-capture/audio-playback）+ orb 可视化 - 摄像头实时预览（默认开启）+ 视觉识别状态条（后端 YOLO 推理） - 底部园区概览条（复用 ParkOverviewStrip），orb-wrap 视窗居中 - 页眉复用 ScreenLayout；左右键导航加入 /voice - 文档：docs/voice-integration-plan.md
This commit is contained in:
Pine
@@ -0,0 +1,57 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM
|
||||
* conversion for the audio frames on the wire, transcript extraction from a
|
||||
* `response.done` payload, and a tiny URL helper. Kept separate from the client
|
||||
* so the protocol/state logic stays readable.
|
||||
*/
|
||||
|
||||
/** @param {string} url */
|
||||
export function trimTrailingSlash(url) {
|
||||
return url.endsWith("/") ? url.slice(0, -1) : url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the assistant transcript out of a `response.done` payload. The text
|
||||
* lives in `response.output[].content[].transcript` (audio) or `.text`. Used as
|
||||
* the source of truth for interrupted replies, where the dedicated
|
||||
* `*.transcript.done` event may never arrive.
|
||||
* @param {any} response
|
||||
* @returns {string}
|
||||
*/
|
||||
export function extractResponseTranscript(response) {
|
||||
const output = response?.output;
|
||||
if (!Array.isArray(output)) return "";
|
||||
/** @type {string[]} */
|
||||
const parts = [];
|
||||
for (const item of output) {
|
||||
for (const part of item?.content ?? []) {
|
||||
const text = part?.transcript ?? part?.text;
|
||||
if (typeof text === "string" && text.trim()) parts.push(text.trim());
|
||||
}
|
||||
}
|
||||
return parts.join(" ").trim();
|
||||
}
|
||||
|
||||
/** @param {ArrayBuffer} buf */
|
||||
export function base64FromArrayBuffer(buf) {
|
||||
const bytes = new Uint8Array(buf);
|
||||
// Chunked encoding so we don't blow up the call stack on long buffers.
|
||||
let binary = "";
|
||||
const chunk = 0x8000;
|
||||
for (let i = 0; i < bytes.length; i += chunk) {
|
||||
binary += String.fromCharCode.apply(null, /** @type {number[]} */ (
|
||||
/** @type {unknown} */ (bytes.subarray(i, i + chunk))
|
||||
));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/** @param {string} b64 */
|
||||
export function base64ToBytes(b64) {
|
||||
const binary = atob(b64);
|
||||
const len = binary.length;
|
||||
const out = new Uint8Array(len);
|
||||
for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the
|
||||
* mic input and the TTS output) and maps the low-frequency speech energy onto
|
||||
* the orb's CSS custom properties:
|
||||
* - `--bar0`..`--bar4` the 5-band level meter
|
||||
* - `--ai-audio-level` the global "Reachy talks" glow / scale pulse
|
||||
*
|
||||
* The bottom of the FFT is where speech energy lives, so the band edges stay
|
||||
* low — that keeps the bars dancing on voice rather than on noise. While the AI
|
||||
* is speaking we source the bars from the OUTPUT analyser so the orb pulses with
|
||||
* Reachy's voice instead of sitting dead while the user is silent.
|
||||
*/
|
||||
|
||||
// Exported so the client can size its AnalyserNodes to match our buffer.
|
||||
export const VIS_FFT_SIZE = 256;
|
||||
const VIS_BAND_COUNT = 5;
|
||||
const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52];
|
||||
const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy)
|
||||
const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade)
|
||||
|
||||
export class OrbVisualiser {
|
||||
/**
|
||||
* @param {AnalyserNode} micAnalyser
|
||||
* @param {AnalyserNode} outAnalyser
|
||||
* @param {() => boolean} isAiSpeaking Source the bars from the AI output when
|
||||
* true, otherwise from the mic.
|
||||
*/
|
||||
constructor(micAnalyser, outAnalyser, isAiSpeaking) {
|
||||
this._mic = micAnalyser;
|
||||
this._out = outAnalyser;
|
||||
this._isAiSpeaking = isAiSpeaking;
|
||||
this._buf = new Uint8Array(micAnalyser.frequencyBinCount);
|
||||
this._bands = new Float32Array(VIS_BAND_COUNT);
|
||||
this._aiLevel = 0;
|
||||
/** @type {number | null} */
|
||||
this._frame = null;
|
||||
}
|
||||
|
||||
/** Begin the rAF loop (idempotent). */
|
||||
start() {
|
||||
if (this._frame !== null) return;
|
||||
const root = document.documentElement;
|
||||
const tick = () => {
|
||||
this._frame = requestAnimationFrame(tick);
|
||||
this._update(root);
|
||||
};
|
||||
this._frame = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
/** Stop the loop and clear the CSS vars so the orb returns to rest. */
|
||||
stop() {
|
||||
if (this._frame !== null) {
|
||||
cancelAnimationFrame(this._frame);
|
||||
this._frame = null;
|
||||
}
|
||||
const root = document.documentElement;
|
||||
for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`);
|
||||
root.style.removeProperty("--ai-audio-level");
|
||||
}
|
||||
|
||||
/** @param {HTMLElement} root */
|
||||
_update(root) {
|
||||
// Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars.
|
||||
const source = this._isAiSpeaking() ? this._out : this._mic;
|
||||
source.getByteFrequencyData(this._buf);
|
||||
|
||||
for (let b = 0; b < VIS_BAND_COUNT; b++) {
|
||||
const lo = VIS_BAND_EDGES[b];
|
||||
const hi = VIS_BAND_EDGES[b + 1];
|
||||
let sum = 0;
|
||||
let n = 0;
|
||||
for (let i = lo; i < hi && i < this._buf.length; i++) {
|
||||
sum += this._buf[i];
|
||||
n += 1;
|
||||
}
|
||||
const target = n > 0 ? sum / (n * 255) : 0;
|
||||
const prev = this._bands[b];
|
||||
const k = target > prev ? VIS_ATTACK : VIS_RELEASE;
|
||||
const next = prev + (target - prev) * k;
|
||||
this._bands[b] = next;
|
||||
root.style.setProperty(`--bar${b}`, next.toFixed(3));
|
||||
}
|
||||
|
||||
// Global AI audio level: peak of the output analyser, used by the CSS to
|
||||
// make the orb's glow / scale react to Reachy's voice.
|
||||
this._out.getByteFrequencyData(this._buf);
|
||||
let peak = 0;
|
||||
const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]);
|
||||
for (let i = 0; i < limit; i++) {
|
||||
if (this._buf[i] > peak) peak = this._buf[i];
|
||||
}
|
||||
const aiTarget = peak / 255;
|
||||
const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE;
|
||||
this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k;
|
||||
root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,243 @@
|
||||
/* =========================================================
|
||||
摄像头实时识别 Hook —— YOLO 人脸检测（后端推理）
|
||||
链路：前端 getUserMedia 本地预览 → 每 ~700ms canvas 抽帧 → JPEG base64
|
||||
POST {API_BASE}/api/vision/frame → 后端 YOLO(yolov8n-face) 推理
|
||||
→ 返回人脸数/框 → 前端驱动状态机
|
||||
- 状态机：检测到人脸（面向大屏）持续 ≥10s → onTrigger；触发后静默 60s
|
||||
- 帧经局域网传到后端（大屏机同机部署则不出设备）；画面预览仍本地
|
||||
========================================================= */
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { API_BASE } from '../config';
|
||||
|
||||
export const VISION_DWELL_MS = 10000; // 面向停留阈值
|
||||
export const VISION_SILENT_MS = 180000; // 触发（自动问候）后静默 3 分钟，期间不重复触发
|
||||
const DETECT_INTERVAL_MS = 700; // ~1.4fps（后端推理 ~100-200ms + 传输）
|
||||
const FRAME_W = 480;
|
||||
const FRAME_H = 360;
|
||||
|
||||
export default function useVisionDetection({ onTrigger, onGesture, enabled = true }) {
|
||||
const [cameraOn, setCameraOn] = useState(false);
|
||||
const [status, setStatus] = useState('off'); // off|loading|detecting|facing|triggered|silent|error
|
||||
const [faces, setFaces] = useState(0);
|
||||
const [dwellMs, setDwellMs] = useState(0);
|
||||
const [gesture, setGesture] = useState(null); // raise(举手，toggle 对话)|null
|
||||
const [errorInfo, setErrorInfo] = useState(null);
|
||||
// 诊断：抽帧画面平均亮度（本地计算，<15 ≈ 黑屏）
|
||||
const [frameLum, setFrameLum] = useState(null);
|
||||
// 诊断：后端推理链路状态 loading|ok|failed
|
||||
const [modelPhase, setModelPhase] = useState('loading');
|
||||
const [modelErr, setModelErr] = useState(null);
|
||||
// 诊断：最近一次推理/请求异常
|
||||
const [detectErr, setDetectErr] = useState(null);
|
||||
// 诊断：后端推理耗时 ms
|
||||
const [latency, setLatency] = useState(null);
|
||||
|
||||
const videoRef = useRef(null);
|
||||
const streamRef = useRef(null);
|
||||
const timerRef = useRef(null);
|
||||
const canvasRef = useRef(null);
|
||||
const cancelledRef = useRef(false); // StrictMode/HMR 卸载标记
|
||||
const stateRef = useRef({
|
||||
facingSince: 0, silentUntil: 0, lastDetect: 0, lastDiag: 0, lastPlayRetry: 0, failCount: 0,
|
||||
// 手势状态机：举手持续 1.5s → raise（前端据此 toggle 对话：无对话开、有对话停）
|
||||
raiseActive: false, raiseStart: 0, raiseTriggered: false,
|
||||
});
|
||||
const triggerRef = useRef(onTrigger);
|
||||
triggerRef.current = onTrigger;
|
||||
const gestureRef = useRef(onGesture);
|
||||
gestureRef.current = onGesture;
|
||||
|
||||
const loop = useCallback(async () => {
|
||||
if (cancelledRef.current) return;
|
||||
const video = videoRef.current;
|
||||
const now = Date.now();
|
||||
const st = stateRef.current;
|
||||
if (!video || video.readyState < 2) {
|
||||
// 视频未就绪：兜底重试 play()（Safari 不自动播放 srcObject 流）
|
||||
if (video && video.srcObject && now - st.lastPlayRetry > 2000) {
|
||||
st.lastPlayRetry = now;
|
||||
video.play().catch(() => { /* 继续等待 */ });
|
||||
}
|
||||
timerRef.current = setTimeout(loop, 400);
|
||||
return;
|
||||
}
|
||||
if (now - st.lastDetect < DETECT_INTERVAL_MS) {
|
||||
timerRef.current = setTimeout(loop, 120);
|
||||
return;
|
||||
}
|
||||
st.lastDetect = now;
|
||||
|
||||
try {
|
||||
// 抽帧
|
||||
if (!canvasRef.current) {
|
||||
canvasRef.current = document.createElement('canvas');
|
||||
canvasRef.current.width = FRAME_W;
|
||||
canvasRef.current.height = FRAME_H;
|
||||
}
|
||||
const ctx = canvasRef.current.getContext('2d', { willReadFrequently: true });
|
||||
ctx.drawImage(video, 0, 0, FRAME_W, FRAME_H);
|
||||
|
||||
// 亮度诊断（本地，每 2s）
|
||||
if (now - st.lastDiag > 2000) {
|
||||
st.lastDiag = now;
|
||||
try {
|
||||
const d = ctx.getImageData(0, 0, FRAME_W, FRAME_H).data;
|
||||
let s = 0;
|
||||
for (let i = 0; i < d.length; i += 4) s += d[i] + d[i + 1] + d[i + 2];
|
||||
setFrameLum(Math.round(s / (d.length / 4) / 3));
|
||||
} catch {
|
||||
setFrameLum(-1);
|
||||
}
|
||||
}
|
||||
|
||||
// JPEG base64 → 后端 YOLO
|
||||
const b64 = canvasRef.current.toDataURL('image/jpeg', 0.6).split(',')[1];
|
||||
setModelPhase('ok');
|
||||
const res = await fetch(`${API_BASE}/api/vision/frame`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ image: b64 }),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!data?.ok) throw new Error(data?.error || `HTTP ${res.status}`);
|
||||
setDetectErr(null);
|
||||
const n = data.faces ?? 0;
|
||||
setFaces(n);
|
||||
setLatency(data.latency_ms);
|
||||
st.failCount = 0;
|
||||
|
||||
// ── 手势状态机：举手持续 1.5s → raise；前端据此 toggle（无对话开 / 有对话停）──
|
||||
const raised = data.raised?.[0]; // 取第一个举手的人/手
|
||||
if (raised) {
|
||||
if (!st.raiseActive) {
|
||||
st.raiseActive = true;
|
||||
st.raiseStart = now;
|
||||
st.raiseTriggered = false;
|
||||
}
|
||||
if (!st.raiseTriggered && now - st.raiseStart >= 1500) {
|
||||
st.raiseTriggered = true;
|
||||
setGesture('raise');
|
||||
console.warn(`[vision] 手势：举手（持续 ${Math.round((now - st.raiseStart) / 1000)}s）→ toggle 对话`);
|
||||
gestureRef.current?.('raise');
|
||||
}
|
||||
} else if (st.raiseActive) {
|
||||
// 举手消失：重置，允许下次举手再次触发
|
||||
st.raiseActive = false;
|
||||
st.raiseStart = 0;
|
||||
st.raiseTriggered = false;
|
||||
setTimeout(() => setGesture(null), 2500);
|
||||
}
|
||||
|
||||
// 状态机：有人脸（面向大屏）持续 10s → 触发问候；触发后静默 60s
|
||||
if (now < st.silentUntil) {
|
||||
st.facingSince = 0;
|
||||
setDwellMs(0);
|
||||
setStatus('silent');
|
||||
} else if (n > 0) {
|
||||
st.facingSince = st.facingSince || now;
|
||||
const d = now - st.facingSince;
|
||||
setDwellMs(d);
|
||||
setStatus('facing');
|
||||
if (d >= VISION_DWELL_MS) {
|
||||
st.silentUntil = now + VISION_SILENT_MS;
|
||||
st.facingSince = 0;
|
||||
setDwellMs(0);
|
||||
setStatus('triggered');
|
||||
triggerRef.current?.();
|
||||
setTimeout(() => setStatus('detecting'), 3000);
|
||||
}
|
||||
} else {
|
||||
st.facingSince = 0;
|
||||
setDwellMs(0);
|
||||
setStatus('detecting');
|
||||
}
|
||||
} catch (e) {
|
||||
// 后端不可达/推理失败：错误显示到页面，连续 5 次标记链路失败
|
||||
setDetectErr(e?.message || String(e));
|
||||
st.failCount += 1;
|
||||
if (st.failCount >= 5) {
|
||||
setModelPhase('failed');
|
||||
setModelErr(e?.message || String(e));
|
||||
}
|
||||
}
|
||||
timerRef.current = setTimeout(loop, 120);
|
||||
}, []);
|
||||
|
||||
const startCamera = useCallback(async () => {
|
||||
if (cameraOn || !enabled) return;
|
||||
cancelledRef.current = false;
|
||||
setStatus('loading');
|
||||
setModelPhase('loading');
|
||||
setModelErr(null);
|
||||
setDetectErr(null);
|
||||
try {
|
||||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error('非安全上下文：请通过 http://localhost 或 HTTPS 访问（当前地址浏览器禁止摄像头）');
|
||||
}
|
||||
// 设备预检
|
||||
try {
|
||||
const devs = await navigator.mediaDevices.enumerateDevices();
|
||||
if (!devs.some((d) => d.kind === 'videoinput')) {
|
||||
const e = new Error('未检测到摄像头设备（请确认 USB 摄像头已接入本机）');
|
||||
e.name = 'NotFoundError';
|
||||
throw e;
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.name === 'NotFoundError') throw e;
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: 640, height: 480 },
|
||||
audio: false,
|
||||
});
|
||||
if (cancelledRef.current) {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
return;
|
||||
}
|
||||
streamRef.current = stream;
|
||||
const video = videoRef.current;
|
||||
if (!video) throw new Error('video element not mounted');
|
||||
video.srcObject = stream;
|
||||
try { await video.play(); } catch { /* muted autoplay 兜底 */ }
|
||||
if (cancelledRef.current) return;
|
||||
setCameraOn(true);
|
||||
setStatus('detecting');
|
||||
timerRef.current = setTimeout(loop, 300);
|
||||
} catch (e) {
|
||||
if (cancelledRef.current) return;
|
||||
setErrorInfo({ name: e?.name ?? 'Error', message: e?.message ?? String(e) });
|
||||
setStatus('error');
|
||||
console.warn('[vision] 摄像头启动失败:', e?.name, e?.message);
|
||||
}
|
||||
}, [cameraOn, enabled, loop]);
|
||||
|
||||
const stopCamera = useCallback(() => {
|
||||
cancelledRef.current = true;
|
||||
if (timerRef.current) clearTimeout(timerRef.current);
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
stateRef.current = { facingSince: 0, silentUntil: 0, lastDetect: 0, lastDiag: 0, lastPlayRetry: 0, failCount: 0,
|
||||
raiseActive: false, raiseStart: 0, raiseTriggered: false };
|
||||
setCameraOn(false);
|
||||
setFaces(0);
|
||||
setDwellMs(0);
|
||||
setGesture(null);
|
||||
setFrameLum(null);
|
||||
setModelPhase('loading');
|
||||
setModelErr(null);
|
||||
setDetectErr(null);
|
||||
setLatency(null);
|
||||
setStatus('off');
|
||||
}, []);
|
||||
|
||||
// 默认开启：挂载即启动；卸载时释放
|
||||
useEffect(() => {
|
||||
if (enabled) void startCamera();
|
||||
return () => stopCamera();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [enabled]);
|
||||
|
||||
return {
|
||||
cameraOn, status, faces, dwellMs, gesture, latency, frameLum, modelPhase, modelErr, detectErr,
|
||||
errorInfo, startCamera, stopCamera, videoRef,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user