feat(vision): YOLO 人脸/姿态识别与手势控制(后端推理)
- vision_yolo:yolov8n-face 人脸检测 + yolov8n-pose 姿态估计(torch/ultralytics,懒加载常驻)
- 单帧几何特征:举手(腕高过肩)与举拳(前臂上举收胸前),返回 raised/fists
- POST /api/vision/frame:前端抽帧 JPEG → 推理 → {faces, pose, raised, fists, latency_ms},线程池执行
- POST /api/vision/event:状态上报,triggered 时经 MQTT 广播 alert
- 模型入库:backend/models/yolov8n-face.pt + yolov8n-pose.pt
- 前端 hook 改为网络版:抽帧→POST→状态机(面向10s问候/举手toggle对话),含亮度/耗时/链路诊断
This commit is contained in:
@@ -387,3 +387,42 @@ async def sse_events(request: Request):
|
||||
bus.unsubscribe(q)
|
||||
|
||||
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||
|
||||
|
||||
# ==================== 视觉识别事件上报(前端摄像头实时识别 → 后端日志/MQTT) ====================
|
||||
class VisionEventBody(BaseModel):
|
||||
event: str # camera_on|camera_off|detecting|facing|triggered|silent|error|model_error
|
||||
faces: int = 0
|
||||
dwell_ms: int = 0
|
||||
detail: str = ""
|
||||
|
||||
@router.post("/api/vision/event")
|
||||
async def vision_event(body: VisionEventBody):
|
||||
"""前端摄像头识别状态/触发上报;后端记录详细日志,triggered 时广播 alert 到全屏。"""
|
||||
log.info(
|
||||
"vision event=%s faces=%d dwell_ms=%d detail=%s",
|
||||
body.event, body.faces, body.dwell_ms, body.detail,
|
||||
)
|
||||
if body.event == "triggered":
|
||||
hub.publish_command("alert", {"text": "有访客正对屏幕,语音助手已主动问候", "faces": body.faces})
|
||||
log.info("vision triggered -> alert 已广播(faces=%d)", body.faces)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ==================== YOLO 人脸检测(前端抽帧 → 后端推理) ====================
|
||||
class VisionFrameBody(BaseModel):
|
||||
image: str = "" # JPEG base64(不含 data: 前缀)
|
||||
conf: float = 0.0 # 可选:覆盖置信度阈值
|
||||
|
||||
@router.post("/api/vision/frame")
|
||||
def vision_frame(body: VisionFrameBody):
|
||||
"""接收前端抽帧 JPEG base64,后端 YOLO 推理返回人脸框(faces/boxes/latency_ms)。
|
||||
|
||||
普通 def 由 FastAPI 线程池执行(推理约 100-200ms),不阻塞事件循环。
|
||||
"""
|
||||
img_b64 = (body.image or "").strip()
|
||||
if not img_b64:
|
||||
log.warning("vision frame: 缺少 image")
|
||||
return {"ok": False, "error": "missing image"}
|
||||
from .vision_yolo import predict_base64
|
||||
return predict_base64(img_b64)
|
||||
|
||||
Reference in New Issue
Block a user