Pine
41e9681aa0
- vision_yolo：yolov8n-face 人脸检测 + yolov8n-pose 姿态估计（torch/ultralytics，懒加载常驻）
- 单帧几何特征：举手（腕高过肩）与举拳（前臂上举收胸前），返回 raised/fists
- POST /api/vision/frame：前端抽帧 JPEG → 推理 → {faces, pose, raised, fists, latency_ms}，线程池执行
- POST /api/vision/event：状态上报，triggered 时经 MQTT 广播 alert
- 模型入库：backend/models/yolov8n-face.pt + yolov8n-pose.pt
- 前端 hook 改为网络版：抽帧→POST→状态机（面向10s问候/举手toggle对话），含亮度/耗时/链路诊断
429 lines
15 KiB
Python
429 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""全部 REST 路由 —— 数据 / 媒体 / 播放列表 / 设置 / AI / 展示控制"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import time
|
||
from pathlib import Path
|
||
|
||
from fastapi import APIRouter, Request, UploadFile, File
|
||
from fastapi.responses import JSONResponse, StreamingResponse
|
||
from pydantic import BaseModel, Field
|
||
|
||
from .ai_tools import run_chat, run_tools
|
||
from .asr import transcribe as asr_transcribe
|
||
from .config import settings
|
||
from .event_bus import bus
|
||
from .llm import run_chat as llm_run_chat
|
||
from .mqtt import hub
|
||
from .sim_engine import sim_engine
|
||
from .storage import storage
|
||
|
||
log = logging.getLogger("dpm.api")
|
||
|
||
router = APIRouter()
|
||
|
||
ALLOWED_EXT = {".mp4", ".mkv", ".avi", ".jpg", ".jpeg", ".png"}
|
||
|
||
|
||
def _media_type(path):
|
||
lower = path.lower()
|
||
if lower.endswith((".mp4", ".mkv", ".avi")):
|
||
return "video"
|
||
return "image"
|
||
|
||
|
||
# ==================== 数据模型 ====================
|
||
|
||
class SettingsBody(BaseModel):
|
||
volume: int | None = None
|
||
play_mode: str | None = None
|
||
image_duration: int | None = None
|
||
fullscreen: bool | None = None
|
||
autostart: bool | None = None
|
||
username: str | None = None
|
||
password: str | None = None
|
||
|
||
|
||
class ActionBody(BaseModel):
|
||
action: str
|
||
|
||
|
||
class PathBody(BaseModel):
|
||
path: str
|
||
|
||
|
||
class UrlBody(BaseModel):
|
||
url: str
|
||
|
||
|
||
class StatePayload(BaseModel):
|
||
status: str = ""
|
||
index: int = 0
|
||
name: str = ""
|
||
media_type: str = ""
|
||
|
||
|
||
class StateBody(BaseModel):
|
||
state: StatePayload
|
||
|
||
|
||
class DisplayCommandBody(BaseModel):
|
||
action: str
|
||
params: dict = Field(default_factory=dict)
|
||
|
||
|
||
class ChatMessage(BaseModel):
|
||
role: str
|
||
content: str
|
||
|
||
|
||
class ChatBody(BaseModel):
|
||
messages: list[ChatMessage]
|
||
|
||
|
||
# ==================== 认证 / 设置 ====================
|
||
|
||
@router.post("/api/login")
|
||
async def login(request: Request):
|
||
form = await request.form()
|
||
s = storage.get_settings()
|
||
return JSONResponse({"success": form.get("username") == s["username"] and form.get("password") == s["password"]})
|
||
|
||
|
||
@router.get("/api/settings")
|
||
async def get_settings():
|
||
s = storage.get_settings()
|
||
return {k: s[k] for k in ("volume", "play_mode", "image_duration", "fullscreen", "autostart")}
|
||
|
||
|
||
@router.post("/api/settings")
|
||
async def update_settings(body: SettingsBody):
|
||
storage.update_settings(**body.model_dump(exclude_none=True))
|
||
s = storage.get_settings()
|
||
hub.publish_command("settings_changed", {"volume": s["volume"], "play_mode": s["play_mode"]})
|
||
return {"ok": True}
|
||
|
||
|
||
# ==================== 媒体 ====================
|
||
|
||
def _list_media():
|
||
files = []
|
||
media_dir = Path(settings.MEDIA_DIR)
|
||
if media_dir.exists():
|
||
for p in sorted(media_dir.iterdir()):
|
||
if not p.is_file() or p.suffix.lower() not in ALLOWED_EXT:
|
||
continue
|
||
files.append({
|
||
"name": p.name, "relative_path": p.name,
|
||
"type": _media_type(p.name), "url": f"/file/{p.name}", "source": "local",
|
||
})
|
||
for item in storage.get_url_media():
|
||
files.append({
|
||
"name": item.get("name", item.get("url", "")),
|
||
"relative_path": item.get("url", ""),
|
||
"type": item.get("type", "image"),
|
||
"url": item.get("url", ""),
|
||
"source": "url",
|
||
})
|
||
return {"files": files}
|
||
|
||
|
||
@router.get("/media")
|
||
async def list_media():
|
||
return _list_media()
|
||
|
||
|
||
@router.post("/upload")
|
||
async def upload(file: UploadFile = File(...)):
|
||
name = file.filename or "file"
|
||
ext = Path(name).suffix.lower()
|
||
if ext not in ALLOWED_EXT:
|
||
return JSONResponse({"ok": False, "error": "不支持的文件类型"}, status_code=400)
|
||
settings.MEDIA_DIR.mkdir(parents=True, exist_ok=True)
|
||
stem = Path(name).stem
|
||
new_name = f"{stem}_{time.strftime('%Y%m%d%H%M%S')}{ext}"
|
||
save_path = settings.MEDIA_DIR / new_name
|
||
with save_path.open("wb") as f:
|
||
while True:
|
||
chunk = await file.read(1024 * 1024)
|
||
if not chunk:
|
||
break
|
||
f.write(chunk)
|
||
hub.publish_command("playlist_changed")
|
||
return {"ok": True, "name": new_name}
|
||
|
||
|
||
@router.post("/api/delete")
|
||
async def delete_media(body: PathBody):
|
||
path = body.path
|
||
if not path.startswith(("http://", "https://")):
|
||
target = settings.MEDIA_DIR / path
|
||
try:
|
||
if target.exists() and target.is_file():
|
||
target.unlink()
|
||
except OSError:
|
||
pass
|
||
storage.delete_media(path)
|
||
hub.publish_command("playlist_changed")
|
||
return {"ok": True}
|
||
|
||
|
||
@router.post("/api/media/add-url")
|
||
async def add_url(body: UrlBody):
|
||
url = body.url.strip()
|
||
if not url:
|
||
return {"ok": False}
|
||
name = url.rsplit("/", 1)[-1].split("?", 1)[0] or url
|
||
t = _media_type(url)
|
||
is_new = storage.add_url_media(url, name, t)
|
||
return {"ok": True, "duplicate": None if is_new else True}
|
||
|
||
|
||
# ==================== 播放列表 ====================
|
||
|
||
def _playlist_files():
|
||
files = []
|
||
media_dir = Path(settings.MEDIA_DIR)
|
||
for item in storage.get_playlist():
|
||
path = item.get("path", "")
|
||
if item.get("source") == "url":
|
||
files.append({
|
||
"name": item.get("name") or path,
|
||
"relative_path": path, "type": _media_type(path), "url": path, "source": "url",
|
||
})
|
||
else:
|
||
fp = media_dir / path
|
||
if fp.exists():
|
||
files.append({
|
||
"name": fp.name, "relative_path": path,
|
||
"type": _media_type(path), "url": f"/file/{path}", "source": "local",
|
||
})
|
||
s = storage.get_settings()
|
||
return {"files": files, "volume": s["volume"], "play_mode": s["play_mode"], "image_duration": s["image_duration"]}
|
||
|
||
|
||
@router.get("/api/playlist")
|
||
async def get_playlist():
|
||
return _playlist_files()
|
||
|
||
|
||
@router.post("/api/playlist/add")
|
||
async def add_playlist(body: PathBody):
|
||
storage.add_to_playlist(body.path)
|
||
hub.publish_command("playlist_changed")
|
||
return {"ok": True}
|
||
|
||
|
||
@router.post("/api/playlist/remove")
|
||
async def remove_playlist(body: PathBody):
|
||
storage.remove_from_playlist(body.path)
|
||
hub.publish_command("playlist_changed")
|
||
return {"ok": True}
|
||
|
||
|
||
@router.post("/api/playlist/play")
|
||
async def play_playlist_item(body: PathBody):
|
||
"""指定媒体立即播放：加入播放列表（如未在）→ MQTT 广播 play_target → 大屏跳转播放"""
|
||
path = body.path
|
||
storage.add_to_playlist(path)
|
||
hub.publish_command("playlist_changed")
|
||
cmd = hub.publish_command("play_target", {"path": path})
|
||
return {"ok": True, "cmd_id": cmd.get("cmd_id")}
|
||
|
||
|
||
# ==================== 播放控制 / 状态 ====================
|
||
|
||
_CONTROL_ACTIONS = {"play", "pause", "next", "prev"}
|
||
|
||
|
||
@router.post("/api/control")
|
||
async def control(body: ActionBody):
|
||
if body.action not in _CONTROL_ACTIONS:
|
||
return JSONResponse({"ok": False, "error": "action 必须是 play/pause/next/prev"}, status_code=400)
|
||
hub.publish_command(body.action)
|
||
return {"ok": True, "action": body.action}
|
||
|
||
|
||
@router.get("/api/state")
|
||
async def get_state():
|
||
return {"state": {"status": "unknown", "index": 0, "name": "", "media_type": ""}}
|
||
|
||
|
||
@router.post("/api/state")
|
||
async def update_state(body: StateBody):
|
||
hub.publish_command("state_update", body.state.model_dump())
|
||
return {"ok": True}
|
||
|
||
|
||
@router.post("/api/display-command")
|
||
async def display_command(body: ActionBody):
|
||
if body.action == "minimize":
|
||
hub.publish_command("minimize")
|
||
return {"ok": True, "action": "minimize"}
|
||
return JSONResponse({"ok": False, "error": "不支持的显示命令"}, status_code=400)
|
||
|
||
|
||
# ==================== 数据（大屏全部数据来自后端） ====================
|
||
|
||
@router.get("/api/dashboard/snapshot")
|
||
async def dashboard_snapshot():
|
||
return sim_engine.snapshot()
|
||
|
||
|
||
@router.get("/api/dashboard/overview")
|
||
async def dashboard_overview():
|
||
return sim_engine.snapshot()
|
||
|
||
|
||
# ==================== 园区企业 ====================
|
||
|
||
_COMPANIES = [
|
||
{"name": "云南派音人工智能科技", "zone": "加速区", "room": "A1", "color": "#4c8dff"},
|
||
{"name": "米勒克尔蓝宝石珠宝", "zone": "加速区", "room": "A2", "color": "#4c8dff"},
|
||
{"name": "中泰研学合作", "zone": "加速区", "room": "A3", "color": "#4c8dff"},
|
||
{"name": "云南宸中低空经济", "zone": "加速区", "room": "A4", "color": "#4c8dff"},
|
||
{"name": "昆明智海银高文化科技", "zone": "加速区", "room": "A5", "color": "#4c8dff"},
|
||
{"name": "AI机器人大模型训练", "zone": "加速区", "room": "A6", "color": "#4c8dff"},
|
||
{"name": "云南廷秀文旅康养", "zone": "加速区", "room": "A7", "color": "#4c8dff"},
|
||
{"name": "瀚颖AI+教育信息咨询", "zone": "加速区", "room": "A8", "color": "#4c8dff"},
|
||
{"name": "云南大学AI+创业平台", "zone": "加速区", "room": "A9", "color": "#4c8dff"},
|
||
{"name": "仰光客厅", "zone": "国际区", "room": "I1", "color": "#22d3ee"},
|
||
{"name": "云南上古绝学文化", "zone": "国际区", "room": "I2", "color": "#22d3ee"},
|
||
{"name": "中越生物医疗", "zone": "国际区", "room": "I3", "color": "#22d3ee"},
|
||
{"name": "酷享野农AI农业", "zone": "国际区", "room": "I4", "color": "#22d3ee"},
|
||
{"name": "滇缅国际设计", "zone": "国际区", "room": "I5", "color": "#22d3ee"},
|
||
{"name": "昆明舒诺生物科技", "zone": "国际区", "room": "I6", "color": "#22d3ee"},
|
||
{"name": "达岸教育管理", "zone": "国际区", "room": "I7", "color": "#22d3ee"},
|
||
{"name": "Facebook越南跨境电商", "zone": "成长区", "room": "G1", "color": "#34d399"},
|
||
{"name": "研X同行者网络", "zone": "成长区", "room": "G2", "color": "#34d399"},
|
||
{"name": "朵哈·玫瑰特色产业链", "zone": "成长区", "room": "G3", "color": "#34d399"},
|
||
{"name": "昆明云韵体育", "zone": "成长区", "room": "G4", "color": "#34d399"},
|
||
{"name": "南菌优培食用菌", "zone": "成长区", "room": "G5", "color": "#34d399"},
|
||
{"name": "五华区丽裳文化", "zone": "成长区", "room": "G6", "color": "#34d399"},
|
||
{"name": "云南星瑞航空", "zone": "成长区", "room": "G7", "color": "#34d399"},
|
||
{"name": "综合直播私域平台", "zone": "成长区", "room": "G8", "color": "#34d399"},
|
||
{"name": "园区管理办公室", "zone": "园区管理", "room": "M", "color": "#fbbf24"},
|
||
]
|
||
|
||
|
||
@router.get("/api/park/companies")
|
||
async def park_companies():
|
||
return {"companies": _COMPANIES}
|
||
|
||
|
||
@router.get("/api/park/zones")
|
||
async def park_zones():
|
||
zones = ["加速区", "国际区", "成长区"]
|
||
counts = {z: sum(1 for c in _COMPANIES if c["zone"] == z) for z in zones}
|
||
colors = {"加速区": "#4c8dff", "国际区": "#22d3ee", "成长区": "#34d399"}
|
||
return {"zones": [{"name": z, "count": counts[z], "color": colors[z]} for z in zones]}
|
||
|
||
|
||
# ==================== AI 对话（通义千问 + 工具调用） ====================
|
||
|
||
@router.post("/api/ai/chat")
|
||
async def ai_chat(body: ChatBody):
|
||
"""大模型对话：工具调用（切页/控制/卡片/通知）经 MQTT 广播到所有大屏
|
||
返回 {reply, tools, model}；tools 供请求端本地同步执行"""
|
||
messages = [m.model_dump() for m in body.messages]
|
||
result = llm_run_chat(messages)
|
||
return result
|
||
|
||
|
||
@router.post("/api/ai/asr")
|
||
async def ai_asr(file: UploadFile = File(...), format: str = "m4a"):
|
||
"""语音识别：上传录音 → 阿里云 paraformer 转写为文本"""
|
||
data = await file.read()
|
||
if not data:
|
||
return JSONResponse({"ok": False, "error": "空音频"}, status_code=400)
|
||
try:
|
||
text = asr_transcribe(data, fmt=format)
|
||
return {"ok": True, "text": text}
|
||
except Exception as e: # noqa: BLE001
|
||
log.warning("ASR 转写失败: %s", e)
|
||
return JSONResponse({"ok": False, "error": str(e)}, status_code=502)
|
||
|
||
|
||
# ==================== 展示控制（管理端 → MQTT） ====================
|
||
|
||
_VALID_DISPLAY_ACTIONS = {"navigate", "navigate_rel", "play", "pause", "next", "prev",
|
||
"alert", "show_card", "set_mode", "minimize"}
|
||
|
||
|
||
@router.post("/api/display/command")
|
||
async def display_command_publish(body: DisplayCommandBody):
|
||
"""管理端/任意客户端通过 REST 发控制指令 → 后端转 MQTT 广播给所有大屏"""
|
||
if body.action not in _VALID_DISPLAY_ACTIONS:
|
||
return JSONResponse({"ok": False, "error": f"不支持的指令: {body.action}"}, status_code=400)
|
||
cmd = hub.publish_command(body.action, body.params)
|
||
return {"ok": cmd.get("published", False), "cmd_id": cmd["cmd_id"], "action": body.action,
|
||
"mqtt_connected": hub.connected}
|
||
|
||
|
||
@router.get("/api/display/state")
|
||
async def display_state():
|
||
return hub.status()
|
||
|
||
|
||
# ==================== SSE 兼容通道（MQTT 不可用时前端回退） ====================
|
||
|
||
@router.get("/api/events")
|
||
async def sse_events(request: Request):
|
||
q = bus.subscribe()
|
||
|
||
async def gen():
|
||
try:
|
||
while True:
|
||
if await request.is_disconnected():
|
||
break
|
||
try:
|
||
data = await asyncio.wait_for(q.get(), timeout=15)
|
||
yield f"data: {data}\n\n"
|
||
except asyncio.TimeoutError:
|
||
yield ": keepalive\n\n"
|
||
finally:
|
||
bus.unsubscribe(q)
|
||
|
||
return StreamingResponse(gen(), media_type="text/event-stream")
|
||
|
||
|
||
# ==================== 视觉识别事件上报（前端摄像头实时识别 → 后端日志/MQTT） ====================
|
||
class VisionEventBody(BaseModel):
|
||
event: str # camera_on|camera_off|detecting|facing|triggered|silent|error|model_error
|
||
faces: int = 0
|
||
dwell_ms: int = 0
|
||
detail: str = ""
|
||
|
||
@router.post("/api/vision/event")
|
||
async def vision_event(body: VisionEventBody):
|
||
"""前端摄像头识别状态/触发上报；后端记录详细日志，triggered 时广播 alert 到全屏。"""
|
||
log.info(
|
||
"vision event=%s faces=%d dwell_ms=%d detail=%s",
|
||
body.event, body.faces, body.dwell_ms, body.detail,
|
||
)
|
||
if body.event == "triggered":
|
||
hub.publish_command("alert", {"text": "有访客正对屏幕，语音助手已主动问候", "faces": body.faces})
|
||
log.info("vision triggered -> alert 已广播（faces=%d）", body.faces)
|
||
return {"ok": True}
|
||
|
||
|
||
# ==================== YOLO 人脸检测（前端抽帧 → 后端推理） ====================
|
||
class VisionFrameBody(BaseModel):
|
||
image: str = "" # JPEG base64（不含 data: 前缀）
|
||
conf: float = 0.0 # 可选：覆盖置信度阈值
|
||
|
||
@router.post("/api/vision/frame")
|
||
def vision_frame(body: VisionFrameBody):
|
||
"""接收前端抽帧 JPEG base64，后端 YOLO 推理返回人脸框（faces/boxes/latency_ms）。
|
||
|
||
普通 def 由 FastAPI 线程池执行（推理约 100-200ms），不阻塞事件循环。
|
||
"""
|
||
img_b64 = (body.image or "").strip()
|
||
if not img_b64:
|
||
log.warning("vision frame: 缺少 image")
|
||
return {"ok": False, "error": "missing image"}
|
||
from .vision_yolo import predict_base64
|
||
return predict_base64(img_b64)
|