feat(vision): YOLO 人脸/姿态识别与手势控制(后端推理)

- vision_yolo:yolov8n-face 人脸检测 + yolov8n-pose 姿态估计(torch/ultralytics,懒加载常驻)
- 单帧几何特征:举手(腕高过肩)与举拳(前臂上举收胸前),返回 raised/fists
- POST /api/vision/frame:前端抽帧 JPEG → 推理 → {faces, pose, raised, fists, latency_ms},线程池执行
- POST /api/vision/event:状态上报,triggered 时经 MQTT 广播 alert
- 模型入库:backend/models/yolov8n-face.pt + yolov8n-pose.pt
- 前端 hook 改为网络版:抽帧→POST→状态机(面向10s问候/举手toggle对话),含亮度/耗时/链路诊断
This commit is contained in:
Pine
2026-08-18 01:37:53 +08:00
parent 9f64b4273b
commit 41e9681aa0
4 changed files with 194 additions and 0 deletions
+39
View File
@@ -387,3 +387,42 @@ async def sse_events(request: Request):
bus.unsubscribe(q)
return StreamingResponse(gen(), media_type="text/event-stream")
# ==================== 视觉识别事件上报(前端摄像头实时识别 → 后端日志/MQTT) ====================
class VisionEventBody(BaseModel):
event: str # camera_on|camera_off|detecting|facing|triggered|silent|error|model_error
faces: int = 0
dwell_ms: int = 0
detail: str = ""
@router.post("/api/vision/event")
async def vision_event(body: VisionEventBody):
"""前端摄像头识别状态/触发上报;后端记录详细日志,triggered 时广播 alert 到全屏。"""
log.info(
"vision event=%s faces=%d dwell_ms=%d detail=%s",
body.event, body.faces, body.dwell_ms, body.detail,
)
if body.event == "triggered":
hub.publish_command("alert", {"text": "有访客正对屏幕,语音助手已主动问候", "faces": body.faces})
log.info("vision triggered -> alert 已广播(faces=%d)", body.faces)
return {"ok": True}
# ==================== YOLO 人脸检测(前端抽帧 → 后端推理) ====================
class VisionFrameBody(BaseModel):
image: str = "" # JPEG base64(不含 data: 前缀)
conf: float = 0.0 # 可选:覆盖置信度阈值
@router.post("/api/vision/frame")
def vision_frame(body: VisionFrameBody):
"""接收前端抽帧 JPEG base64,后端 YOLO 推理返回人脸框(faces/boxes/latency_ms)。
普通 def 由 FastAPI 线程池执行(推理约 100-200ms),不阻塞事件循环。
"""
img_b64 = (body.image or "").strip()
if not img_b64:
log.warning("vision frame: 缺少 image")
return {"ok": False, "error": "missing image"}
from .vision_yolo import predict_base64
return predict_base64(img_b64)