feat: 大屏后端（FastAPI+MQTT+sim_engine）
- REST :10085 / MQTT 控制 / SSE 回退 / AI+语音 - 数据源规划：接核心服务端 + 算力引擎真实数据
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""管理后台页面（FastAPI + Jinja2 服务端渲染）
|
||||
GET /admin → 登录页 或 管理台（按 Cookie 鉴权）
|
||||
POST /admin/login → 表单登录，成功写入签名 Cookie
|
||||
GET /admin/logout → 清除 Cookie
|
||||
媒体库 / 播放列表 / 设置均由 Jinja 服务端渲染，动态操作用少量 JS 调 REST
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from pathlib import Path
|
||||
|
||||
from .config import settings
|
||||
from .storage import storage
|
||||
from .routers import _list_media, _playlist_files
|
||||
|
||||
log = logging.getLogger("dpm.admin")
|
||||
|
||||
TEMPLATES_DIR = Path(__file__).resolve().parent.parent / "templates"
|
||||
templates = Jinja2Templates(directory=str(TEMPLATES_DIR))
|
||||
|
||||
|
||||
# ---------------- 签名 Cookie ----------------
|
||||
|
||||
def _admin_token():
|
||||
return hmac.new(settings.ADMIN_SECRET.encode(), b"dpm-admin", hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _verify_admin_cookie(cookie):
|
||||
return bool(cookie) and hmac.compare_digest(cookie, _admin_token())
|
||||
|
||||
|
||||
def _set_admin_cookie(resp):
|
||||
resp.set_cookie("dpm_admin", _admin_token(), max_age=7 * 86400, httponly=True, samesite="lax")
|
||||
return resp
|
||||
|
||||
|
||||
def _clear_admin_cookie(resp):
|
||||
resp.delete_cookie("dpm_admin")
|
||||
return resp
|
||||
|
||||
|
||||
# ---------------- 路由 ----------------
|
||||
|
||||
async def admin_page(request: Request):
|
||||
if not _verify_admin_cookie(request.cookies.get("dpm_admin")):
|
||||
return templates.TemplateResponse(
|
||||
request=request,
|
||||
name="admin.html",
|
||||
context={
|
||||
"logged_in": False,
|
||||
"error": request.query_params.get("error") == "1",
|
||||
},
|
||||
)
|
||||
|
||||
files = _list_media()["files"]
|
||||
playlist = _playlist_files()
|
||||
s = storage.get_settings()
|
||||
return templates.TemplateResponse(
|
||||
request=request,
|
||||
name="admin.html",
|
||||
context={
|
||||
"logged_in": True,
|
||||
"error": False,
|
||||
"files": files,
|
||||
"playlist": playlist["files"],
|
||||
"play_mode": s.get("play_mode", "sequential"),
|
||||
"image_duration": s.get("image_duration", 5),
|
||||
"volume": s.get("volume", 80),
|
||||
"sfx_volume": s.get("sfx_volume", 60),
|
||||
"fullscreen": s.get("fullscreen", True),
|
||||
"autostart": s.get("autostart", False),
|
||||
"from_screen": request.query_params.get("from") == "screen",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def admin_login(request: Request):
|
||||
form = await request.form()
|
||||
s = storage.get_settings()
|
||||
if form.get("username") == s.get("username") and form.get("password") == s.get("password"):
|
||||
return _set_admin_cookie(RedirectResponse("/admin", status_code=303))
|
||||
return RedirectResponse("/admin?error=1", status_code=303)
|
||||
|
||||
|
||||
async def admin_logout():
|
||||
return _clear_admin_cookie(RedirectResponse("/admin", status_code=303))
|
||||
@@ -0,0 +1,181 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""AI 助手工具调用引擎
|
||||
后端收到对话 → 规则引擎判断意图 → 产出文字回复 + 工具调用列表
|
||||
工具执行时通过 MQTT 广播 opc/display/command，前端（所有大屏）同步响应
|
||||
工具：navigate（切页） / control（媒体控制） / alert（通知） / show_card（展示卡片）
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from .mqtt import hub
|
||||
|
||||
log = logging.getLogger("dpm.ai_tools")
|
||||
|
||||
# ---------------- 工具执行 ----------------
|
||||
|
||||
def execute_tool(tool, screen_id=""):
|
||||
"""执行一个工具调用：发布 MQTT 控制指令（带发起屏幕 screen_id，只发到该屏），返回是否发布成功"""
|
||||
t = tool.get("type")
|
||||
params = tool.get("params") or {}
|
||||
if t == "navigate":
|
||||
return hub.publish_command("navigate", {"page": params.get("page", "home")}, screen_id=screen_id).get("published", False)
|
||||
if t == "control":
|
||||
action = params.get("action")
|
||||
if action in ("play", "pause", "next", "prev"):
|
||||
return hub.publish_command(action, screen_id=screen_id).get("published", False)
|
||||
if action == "set_mode":
|
||||
return hub.publish_command("set_mode", {"mode": params.get("mode", "auto")}, screen_id=screen_id).get("published", False)
|
||||
return False
|
||||
if t == "alert":
|
||||
return hub.publish_command("alert", {"title": params.get("title", "提示"), "content": params.get("content", "")}, screen_id=screen_id).get("published", False)
|
||||
if t == "show_card":
|
||||
return hub.publish_command("show_card", params, screen_id=screen_id).get("published", False)
|
||||
return False
|
||||
|
||||
|
||||
def run_tools(tools, screen_id=""):
|
||||
"""执行一组工具（带发起屏幕 screen_id），返回 [{type, ok}] 并汇总是否全部发布成功"""
|
||||
results = []
|
||||
all_ok = True
|
||||
for tool in tools or []:
|
||||
ok = bool(execute_tool(tool, screen_id))
|
||||
log.info("ai_tools: 执行工具 %s params=%s screen=%s -> %s", tool.get("type"),
|
||||
json.dumps(tool.get("params") or {}, ensure_ascii=False), screen_id or "all", ok)
|
||||
results.append({"type": tool.get("type"), "ok": ok})
|
||||
all_ok = all_ok and ok
|
||||
return results, all_ok
|
||||
|
||||
|
||||
# ---------------- 意图 → 回复 + 工具 ----------------
|
||||
|
||||
PAGES = {
|
||||
"home": "/", "数据大屏": "/", "首页": "/",
|
||||
"twin": "/twin", "数字孪生": "/twin", "3d": "/twin", "孪生": "/twin",
|
||||
"ai": "/ai", "ai助手": "/ai", "智能助手": "/ai",
|
||||
"screen": "/screen", "媒体": "/screen", "轮播": "/screen", "媒体轮播": "/screen",
|
||||
}
|
||||
|
||||
RULES = [
|
||||
{
|
||||
"match": ["入驻", "申请", "入孵", "入园", "如何加入", "怎样加入"],
|
||||
"reply": "入驻流程（按《入驻管理规程》，每年原则上两次）：\n\n1. 提出申请：提交《入驻申请表》《创业计划书》、身份证（外籍提供护照签证）、学历证明、营业执照、资质专利证书等材料\n2. 项目初审：服务机构初审，材料不齐一次性告知补全\n3. 入驻评审：从《昆明市就业创业导师库》抽取 3 名评审专家，路演答辩，量化打分 100 分制，80 分及以上通过\n4. 评审公示：报主管部门备案后在昆明市人社局官网公示\n5. 签署协议：签订《入驻孵化协议》，分配场地/工位，正式入驻\n\n按官方统计（2026年7月）园区实际入驻企业 38 家。咨询电话：18087174891。",
|
||||
"tools": [{"type": "show_card", "params": {"card": "overview"}}],
|
||||
},
|
||||
{
|
||||
"match": ["政策", "补贴", "优惠", "扶持", "减免", "奖励", "资助"],
|
||||
"reply": "园区政策服务覆盖「七位一体」中的政策维度：\n\n· 场地租金减免：符合条件的大学生创业项目可享受场地优惠\n· 创业补贴：协助对接省、市两级创业补贴与一次性创业资助\n· 税费优惠：对接小微企业所得税减免、增值税优惠等政策\n· 项目申报：协助高新技术企业认定、科技型中小企业备案申报辅导\n\n更多政策细节可咨询园区政策服务窗口（电话：18087174891）。",
|
||||
"tools": [],
|
||||
},
|
||||
{
|
||||
"match": ["场地", "办公室", "工位", "租金", "收费", "面积", "多少钱", "价格"],
|
||||
"reply": "园区建筑面积约 3000 ㎡（官方统计口径），可容纳约 50 个创业团队，按发展阶段设置四大孵化区域（《宣传册》）：\n\n· OPC创业空间：AI技术支撑、数字经济/AI应用/软件开发/文创设计/电商直播等轻资产领域，拎包入驻，最长 2 年\n· 创业苗圃区：创意阶段未注册团队，开放式共享工位，最长 2 年\n· 孵化加速区：已注册初创企业，独立办公空间，基础 2 年可延 1 年（累计≤3年）\n· 国际创客区：归国/外籍留学生，个性化办公空间，参照加速区\n\n办公场地免费提供（含桌椅、文件柜、高速网络等基础配套），水电网络全配套、拎包入驻。",
|
||||
"tools": [{"type": "show_card", "params": {"card": "zones"}}],
|
||||
},
|
||||
{
|
||||
"match": ["企业", "公司", "名录", "名单", "入驻企业", "有哪些企业", "查企业"],
|
||||
"reply": "按官方统计（2026年7月），园区实际入驻企业 38 家；《入驻企业信息表》收录本期入驻企业（项目）名录 39 家，包括云南派音人工智能科技（PineSound）、米勒克尔蓝宝石珠宝、云南宸中低空经济、仰光客厅、中越生物医疗、酷享野农AI农业、研X同行者网络等。\n\n行业分布：人工智能 7 家、跨境电商 5 家、教育培训 4 家、文旅康养 4 家、文化创意 4 家、生物医药 4 家、低空经济 3 家、现代农业 3 家、绿色环保 2 家、企业服务 2 家、新材料 1 家。\n\n我已在屏幕上为您展示企业分布卡片。",
|
||||
"tools": [{"type": "show_card", "params": {"card": "companies"}}],
|
||||
},
|
||||
{
|
||||
"match": ["融资", "投资", "资金", "贷款", "估值", "资本"],
|
||||
"reply": "园区融资服务包含：\n\n· 融资对接：定期组织路演与投资机构对接活动\n· 经营成效：入驻团队累计经营收入约 2.19 亿元（省级绩效累计口径，截至2026年5月）\n· 金融支持：协助对接银行创业贷款、政府创业担保贷款\n· 财务辅导：提供融资方案设计与商业计划书辅导",
|
||||
"tools": [],
|
||||
},
|
||||
{
|
||||
"match": ["就业", "招聘", "岗位", "人才", "用工"],
|
||||
"reply": "园区就业数据（口径说明）：\n\n· 官方统计（2026年7月）：当年累计带动就业 213 人\n· 省级绩效累计口径（截至2026年5月）：累计带动就业 1,529 人（本年度 106 人）\n\n· 企业招聘：在驻企业持续发布岗位\n· 人才对接：与在昆高校共建实习实训基地\n· 创业带动：园区鼓励以创业带动就业",
|
||||
"tools": [],
|
||||
},
|
||||
{
|
||||
"match": ["导师", "培训", "课程", "辅导", "孵化服务", "孵化器"],
|
||||
"reply": "园区组建创业指导专家团队 50 人，提供全周期孵化服务：\n\n· 一对一辅导：创业导师定期坐诊，覆盖商业模式、法律财税、市场营销\n· 培训课程：创业训练营、行业专题沙龙、政策解读会\n· 资源对接：技术、供应链、渠道等产业资源链接\n· 公共服务：工商注册、知识产权、法律咨询一站式代办",
|
||||
"tools": [],
|
||||
},
|
||||
{
|
||||
"match": ["AI", "模型", "算力", "tokens", "token", "工具", "智能", "赋能", "大模型"],
|
||||
"reply": "园区 AI 赋能服务是「七位一体」的核心特色：\n\n· 大模型推理：接入 DeepSeek、通义千问、智谱 GLM-4、豆包、讯飞星火等多模型\n· 工具调用：文档生成、数据查询、图像创作、代码执行、语音合成等智能工具\n· 算力与成本：实时提供 token 消耗统计与速率监控\n\n今日园区 AI 推理任务持续运行，服务节点 12/39 在线。",
|
||||
"tools": [],
|
||||
},
|
||||
{
|
||||
"match": ["加速区", "国际区", "成长区", "分区", "区域", "区别", "布局"],
|
||||
"reply": "园区按创业企业（项目）发展阶段设置四大孵化区域（《宣传册》《入驻管理规程》）：\n\n· OPC创业空间：AI技术支撑、轻资产领域，拎包入驻，孵化周期最长 2 年\n· 创业苗圃区：创意阶段未注册团队，共享工位，最长 2 年\n· 孵化加速区：已注册初创企业，独立办公，基础 2 年可延 1 年（累计≤3年）\n· 国际创客区：归国/外籍留学生，个性化空间，参照加速区\n\n我已在屏幕上为您展示分区介绍卡片。",
|
||||
"tools": [{"type": "show_card", "params": {"card": "zones"}}],
|
||||
},
|
||||
{
|
||||
"match": ["切", "去", "打开", "显示", "进入"],
|
||||
"reply": None, # 由下方页面/控制规则补充
|
||||
},
|
||||
{
|
||||
"match": ["你好", "您好", "hi", "hello", "嗨", "在吗"],
|
||||
"reply": "您好！我是 OPC 园区 AI 智能助手。\n\n可以问我：入驻流程、政策扶持、场地收费、AI 赋能、企业名录、融资服务等问题；也可以让我「切换到数字孪生页」「打开媒体轮播」或「展示企业卡片」。",
|
||||
"tools": [],
|
||||
},
|
||||
{
|
||||
"match": ["谢谢", "感谢", "好的", "明白"],
|
||||
"reply": "不客气，随时为您服务。您也可以让我展示园区数据卡片，或切换到任意展示页面。",
|
||||
"tools": [],
|
||||
},
|
||||
]
|
||||
|
||||
DEFAULT_REPLY = "园区提供「空间 + 孵化 + 融资 + 政策 + 资源 + AI 赋能 + 综合服务」七位一体服务。\n\n按官方统计（2026年7月）：实际入驻企业 38 家，当年累计带动就业 213 人，当年生产经营总额 252.15 万元；省级绩效累计口径（截至2026年5月）：累计孵化企业 181 家、累计带动就业 1,529 人、累计经营收入约 2.19 亿元。\n\n您可以这样问我：\n· 如何申请入驻园区？\n· 展示入驻企业卡片\n· 切换到数字孪生页面"
|
||||
|
||||
|
||||
def _detect_page(text):
|
||||
for key, path in PAGES.items():
|
||||
if key in text:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def _detect_control(text):
|
||||
if any(k in text for k in ["播放", "继续播放", "开始播放"]):
|
||||
return "play"
|
||||
if any(k in text for k in ["暂停", "停止播放"]):
|
||||
return "pause"
|
||||
if any(k in text for k in ["下一个", "下一张", "下一屏", "切换下一"]):
|
||||
return "next"
|
||||
if any(k in text for k in ["上一个", "上一张", "上一屏", "切换上一"]):
|
||||
return "prev"
|
||||
return None
|
||||
|
||||
|
||||
def run_chat(messages):
|
||||
"""输入对话消息列表（[{"role","content"}]），返回 {reply, tools}"""
|
||||
text = ""
|
||||
for m in reversed(messages or []):
|
||||
if m.get("role") in ("user", "me"):
|
||||
text = (m.get("content") or "").strip()
|
||||
break
|
||||
|
||||
reply = None
|
||||
tools = []
|
||||
|
||||
# 1) 页面切换意图（优先级最高）
|
||||
page = _detect_page(text)
|
||||
if page:
|
||||
label = {"/": "数据大屏", "/twin": "数字孪生", "/ai": "AI 助手", "/screen": "媒体轮播"}.get(page, page)
|
||||
reply = f"好的，已为您切换到「{label}」页面。"
|
||||
tools = [{"type": "navigate", "params": {"page": page}}]
|
||||
return {"reply": reply, "tools": tools}
|
||||
|
||||
# 2) 媒体控制意图
|
||||
ctrl = _detect_control(text)
|
||||
if ctrl:
|
||||
names = {"play": "播放", "pause": "暂停", "next": "切换下一项", "prev": "切换上一项"}
|
||||
reply = f"好的，已执行：{names[ctrl]}。"
|
||||
tools = [{"type": "control", "params": {"action": ctrl}}]
|
||||
return {"reply": reply, "tools": tools}
|
||||
|
||||
# 3) 知识库规则
|
||||
for rule in RULES:
|
||||
if any(k in text for k in rule["match"]):
|
||||
reply = rule["reply"]
|
||||
tools = rule["tools"]
|
||||
break
|
||||
|
||||
if reply is None:
|
||||
reply = DEFAULT_REPLY
|
||||
|
||||
# 工具已在后端通过 MQTT 广播；返回值供请求端本地同步执行
|
||||
return {"reply": reply, "tools": tools}
|
||||
@@ -0,0 +1,93 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""语音识别 —— 阿里云 DashScope paraformer-realtime-v2（流式）
|
||||
前端录音上传（m4a/wav/ogg/opus/pcm）→ 这里转写为文本
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import certifi
|
||||
|
||||
# macOS 系统 Python 无系统 CA：必须在导入 dashscope 之前设置，
|
||||
# aiohttp 才会使用 certifi 根证书（否则 WS 连接报 SSL 证书错误）
|
||||
os.environ.setdefault("SSL_CERT_FILE", certifi.where())
|
||||
|
||||
import dashscope # noqa: E402
|
||||
from dashscope.audio.asr import Recognition, RecognitionCallback # noqa: E402
|
||||
|
||||
from .config import settings # noqa: E402
|
||||
|
||||
log = logging.getLogger("dpm.asr")
|
||||
|
||||
|
||||
def _err_text(result):
|
||||
"""安全提取 RecognitionResult 的错误描述（其 __str__ 有缺陷）"""
|
||||
for attr in ("message", "code"):
|
||||
v = getattr(result, attr, None)
|
||||
if v:
|
||||
return f"{attr}={v}"
|
||||
return repr(result)
|
||||
|
||||
# 前端 MediaRecorder 可能产生的容器/编码 → paraformer 格式名
|
||||
FORMAT_ALIASES = {
|
||||
"mp4": "m4a", "m4a": "m4a", "aac": "m4a",
|
||||
"webm": "ogg", "opus": "opus", "ogg": "ogg",
|
||||
"wav": "wav", "pcm": "pcm", "mp3": "mp3",
|
||||
}
|
||||
|
||||
# m4a/mp3 等压缩格式采样率由服务端自动识别
|
||||
_AUTO_SAMPLE_FORMATS = {"m4a", "mp3", "ogg", "opus", "wav"}
|
||||
|
||||
|
||||
class _Callback(RecognitionCallback):
|
||||
def __init__(self):
|
||||
self.sentences = []
|
||||
self.error = None
|
||||
|
||||
def on_message(self, message):
|
||||
try:
|
||||
header = message.get("header", {})
|
||||
if header.get("action") != "result":
|
||||
return
|
||||
payload = message.get("payload", {})
|
||||
sentences = payload.get("sentence", {}).get("sentences") or []
|
||||
for s in sentences:
|
||||
if s.get("sentence_end"):
|
||||
self.sentences.append(s.get("text", ""))
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("ASR 消息解析异常: %s", e)
|
||||
|
||||
def on_error(self, result):
|
||||
self.error = result
|
||||
log.warning("ASR 错误: %s", _err_text(result))
|
||||
|
||||
|
||||
def transcribe(audio_bytes: bytes, fmt: str = "m4a") -> str:
|
||||
"""上传音频字节 → 返回转写文本（空串表示未识别到内容）"""
|
||||
if not settings.DASHSCOPE_API_KEY:
|
||||
raise RuntimeError("未配置 DASHSCOPE_API_KEY，无法使用阿里云语音识别")
|
||||
|
||||
dashscope.api_key = settings.DASHSCOPE_API_KEY
|
||||
fmt = FORMAT_ALIASES.get((fmt or "m4a").lower().lstrip("."), "m4a")
|
||||
|
||||
cb = _Callback()
|
||||
rec = Recognition(
|
||||
model=settings.ASR_MODEL,
|
||||
format=fmt,
|
||||
sample_rate=16000 if fmt in ("pcm", "wav") else 0, # 压缩格式自动识别
|
||||
callback=cb,
|
||||
)
|
||||
|
||||
# 分块发送，模拟流式（适当间隔，避免触发服务端 batching 报错）
|
||||
rec.start()
|
||||
chunk = 16 * 1024
|
||||
for i in range(0, len(audio_bytes), chunk):
|
||||
rec.send_audio_frame(audio_bytes[i:i + chunk])
|
||||
time.sleep(0.08)
|
||||
time.sleep(0.5) # 等待尾部识别
|
||||
rec.stop()
|
||||
|
||||
if cb.error:
|
||||
raise RuntimeError(f"阿里云语音识别失败: {_err_text(cb.error)}")
|
||||
return "".join(cb.sentences).strip()
|
||||
@@ -0,0 +1,102 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""OPC 智能园区后端 —— 全局配置（backend/.env 可覆盖，未配置时用默认值）"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent # backend/
|
||||
|
||||
|
||||
def _load_dotenv(path):
|
||||
"""极简 .env 解析：KEY=VALUE，支持 # 注释与引号"""
|
||||
if not Path(path).exists():
|
||||
return
|
||||
for raw in Path(path).read_text("utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
_load_dotenv(BASE_DIR / ".env")
|
||||
|
||||
|
||||
def _env(key, default):
|
||||
return os.getenv(key, default)
|
||||
|
||||
|
||||
class Settings:
|
||||
# ---- HTTP ----
|
||||
HOST = _env("DPM_HOST", _env("FASTAPI_HOST", "0.0.0.0"))
|
||||
PORT = int(_env("DPM_PORT", _env("FASTAPI_PORT", "8000")))
|
||||
|
||||
# ---- 数据目录 ----
|
||||
MEDIA_DIR = Path(_env("DPM_MEDIA_DIR", str(BASE_DIR / "media")))
|
||||
DATA_FILE = Path(_env("DPM_DATA_FILE", str(BASE_DIR / "data.json")))
|
||||
DIST_DIR = Path(_env("DPM_DIST_DIR", str(BASE_DIR.parent / "dist"))) # 前端构建产物（可选托管）
|
||||
|
||||
# ---- MQTT（优先使用 .env 中的局域网 Broker 配置） ----
|
||||
MQTT_ENABLED = _env("DPM_MQTT_ENABLED", "1") == "1"
|
||||
MQTT_HOST = _env("DPM_MQTT_HOST", _env("MQTT_BROKER_HOST", "192.168.1.9"))
|
||||
MQTT_PORT = int(_env("DPM_MQTT_PORT", _env("MQTT_BROKER_PORT", "1883")))
|
||||
MQTT_USERNAME = _env("DPM_MQTT_USERNAME", _env("MQTT_USERNAME", "")) or None
|
||||
MQTT_PASSWORD = _env("DPM_MQTT_PASSWORD", _env("MQTT_PASSWORD", "")) or None
|
||||
MQTT_CLIENT_ID = _env("DPM_MQTT_CLIENT_ID", "dpm-backend")
|
||||
# 前端 WebSocket 接入地址（默认按 .env 的 broker 主机 + EMQX 常规 ws 端口）
|
||||
MQTT_WS_URL = _env(
|
||||
"DPM_MQTT_WS",
|
||||
f"ws://{_env('MQTT_PUBLIC_HOST', _env('MQTT_BROKER_HOST', 'localhost'))}:8083/mqtt",
|
||||
)
|
||||
MQTT_TICK_INTERVAL = float(_env("DPM_MQTT_TICK_INTERVAL", "2.2"))
|
||||
|
||||
# ---- 主题 ----
|
||||
TOPIC_COMMAND = _env("DPM_TOPIC_COMMAND", "opc/display/command")
|
||||
TOPIC_TICK = _env("DPM_TOPIC_TICK", "opc/dashboard/tick")
|
||||
TOPIC_ACK = _env("DPM_TOPIC_ACK", "opc/display/ack")
|
||||
TOPIC_HEARTBEAT = _env("DPM_TOPIC_HEARTBEAT", "opc/display/heartbeat")
|
||||
SCREEN_TTL = float(_env("DPM_SCREEN_TTL", "30")) # 大屏心跳过期秒数
|
||||
|
||||
# ---- 阿里云 DashScope（LLM + 语音识别） ----
|
||||
DASHSCOPE_API_KEY = _env("DASHSCOPE_API_KEY", "")
|
||||
LLM_MODEL = _env("DPM_LLM_MODEL", "qwen-plus") # 通义千问
|
||||
ASR_MODEL = _env("DPM_ASR_MODEL", "paraformer-realtime-v2") # 流式语音识别
|
||||
LLM_BASE_URL = _env("DPM_LLM_BASE_URL", "https://dashscope.aliyuncs.com/compatible-mode/v1")
|
||||
ASR_TIMEOUT = float(_env("DPM_ASR_TIMEOUT", "30"))
|
||||
|
||||
# ---- 管理后台 ----
|
||||
ADMIN_SECRET = _env("DPM_ADMIN_SECRET", "dpm-admin-secret-change-me")
|
||||
|
||||
# ---- s2s 实时语音栈（VAD本地 + 云ASR/LLM/TTS） ----
|
||||
S2S_ENABLED = _env("DPM_S2S_ENABLED", "0") == "1"
|
||||
S2S_HOST = _env("DPM_S2S_HOST", "0.0.0.0")
|
||||
S2S_PORT = int(_env("DPM_S2S_PORT", "8765"))
|
||||
S2S_NUM_PIPELINES = int(_env("DPM_S2S_NUM_PIPELINES", "1"))
|
||||
VISION_LLM_MODEL = _env("DPM_VISION_LLM_MODEL", "qwen3-vl-flash") # 对话开场画面识别（多模态）
|
||||
S2S_STT_MODEL = _env("DPM_S2S_STT_MODEL", "qwen3-asr-flash-realtime")
|
||||
S2S_LLM_MODEL = _env("DPM_S2S_LLM_MODEL", "qwen-plus")
|
||||
S2S_TTS_MODEL = _env("DPM_S2S_TTS_MODEL", "qwen3-tts-flash-realtime")
|
||||
S2S_TTS_VOICE = _env("DPM_S2S_TTS_VOICE", "Cherry")
|
||||
S2S_BUILD_RETRIES = int(_env("DPM_S2S_BUILD_RETRIES", "4"))
|
||||
S2S_BUILD_RETRY_DELAY = float(_env("DPM_S2S_BUILD_RETRY_DELAY", "10"))
|
||||
S2S_LOG_LEVEL = _env("DPM_S2S_LOG_LEVEL", "INFO").upper()
|
||||
# 对外可访问的 s2s 实时语音 WS 地址（供 /api/config 下发；DPM_VOICE_WS 可显式覆盖）
|
||||
VOICE_WS = _env("DPM_VOICE_WS", "") # 显式指定时优先使用（s2s 独立进程场景）
|
||||
S2S_WS_URL = (
|
||||
VOICE_WS
|
||||
or f"ws://{_env('MQTT_PUBLIC_HOST', _env('MQTT_BROKER_HOST', 'localhost'))}:{S2S_PORT}/v1/realtime"
|
||||
)
|
||||
# 语音助手基础提示词（.env 设 DPM_S2S_INSTRUCTIONS 可覆盖；知识库内容由 rag 自动拼接）
|
||||
S2S_INSTRUCTIONS = _env(
|
||||
"DPM_S2S_INSTRUCTIONS",
|
||||
"你是昆小创，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、"
|
||||
"query_companies（企业名录）、control_display（大屏控制）、get_time（时间）。"
|
||||
"涉及园区数据/企业/大屏控制时务必调用工具获取准确信息。请用简洁专业的中文回答，不超过三句话。",
|
||||
)
|
||||
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,44 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""SSE 事件总线 —— 兼容旧通道（前端 MQTT 不可用时回退）
|
||||
所有 MQTT 控制指令同时推送到 SSE /api/events
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
|
||||
class EventBus:
|
||||
def __init__(self):
|
||||
self._queues = set()
|
||||
self._loop = None
|
||||
|
||||
def set_loop(self, loop):
|
||||
self._loop = loop
|
||||
|
||||
def subscribe(self):
|
||||
q = asyncio.Queue(maxsize=256)
|
||||
self._queues.add(q)
|
||||
return q
|
||||
|
||||
def unsubscribe(self, q):
|
||||
self._queues.discard(q)
|
||||
|
||||
def emit(self, payload: dict):
|
||||
"""线程安全：从任意线程广播事件"""
|
||||
if not self._loop:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._emit(payload), self._loop)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
async def _emit(self, payload):
|
||||
data = json.dumps(payload, ensure_ascii=False)
|
||||
for q in list(self._queues):
|
||||
try:
|
||||
q.put_nowait(data)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
|
||||
bus = EventBus()
|
||||
@@ -0,0 +1,261 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""AI 对话引擎 —— 阿里云 DashScope（通义千问）+ 函数调用
|
||||
工具调用执行后通过 MQTT 广播，所有大屏同步响应
|
||||
DashScope 不可用时自动回退本地规则引擎（离线兜底）
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import certifi
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("dpm.llm")
|
||||
|
||||
|
||||
def _trunc(s, n=400):
|
||||
"""日志用截断：超长内容只留前 n 字并标注总长。"""
|
||||
s = str(s)
|
||||
return s if len(s) <= n else s[:n] + f"...（共{len(s)}字）"
|
||||
|
||||
|
||||
# macOS 系统 Python 无系统 CA，使用 certifi 提供的根证书
|
||||
_SSL_CTX = ssl.create_default_context(cafile=certifi.where())
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"你是「昆明市大学生创业园 · OPC 智能园区」的 AI 智能助手，运行在大屏展播系统上。"
|
||||
"园区提供空间、孵化、融资、政策、资源、AI 赋能、综合服务七位一体服务。"
|
||||
"回答园区相关问题时，必须严格依据【园区知识库】中的标准资料口径回答，"
|
||||
"不得使用资料外的数字；涉及运营数据时请区分口径（官方统计/省级绩效累计/AI平台指标），"
|
||||
"并在需要时调用 get_park_overview 工具获取实时准确数据。"
|
||||
"当可能需要「切换页面 / 控制播放 / 展示卡片 / 弹出通知」时，必须调用对应工具；"
|
||||
"其余园区相关问题用简洁、专业的中文回答，可适当使用列表。"
|
||||
"你必须基于知识库或者工具调用结果回答问题，不能编造数据或内容。"
|
||||
"如果有不确定的内容，请明确告知用户「我不确定，请咨询园区工作人员」。"
|
||||
"如果用户询问园区的企业，没有指定的情况下，必须介绍派音人工智能"
|
||||
"企业信息（企业名称、负责人、入驻分区、简介）一律以【园区知识库·检索命中】中企业名录原文为准；"
|
||||
"严禁自行编造或补充企业备案号、成立时间、注册资本、投资方、获奖、政府项目等知识库未记载的细节；"
|
||||
"若用户问到的企业细节在知识库中没有记载，请如实说明「该信息在园区资料中未记载」，不要编造。"
|
||||
)
|
||||
|
||||
# 暴露给大模型的工具定义（执行时经 MQTT 广播到前端）
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "navigate_page",
|
||||
"description": "切换大屏展示页面（数据大屏 / 数字孪生 / AI 助手 / 媒体轮播）",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"page": {"type": "string", "enum": ["/", "/twin", "/ai", "/screen"],
|
||||
"description": "目标页面路径"},
|
||||
},
|
||||
"required": ["page"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "media_control",
|
||||
"description": "控制媒体播放（播放/暂停/下一项/上一项）",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "enum": ["play", "pause", "next", "prev"]},
|
||||
},
|
||||
"required": ["action"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "show_card",
|
||||
"description": "在大屏上展示信息卡片（企业分布 / 分区介绍 / 园区总览 / 自定义内容）",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"card": {"type": "string", "enum": ["companies", "zones", "overview", "custom"]},
|
||||
"title": {"type": "string"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
"required": ["card"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "send_alert",
|
||||
"description": "在大屏上弹出通知提示",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
"required": ["title", "content"],
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
_TOOL_MAP = {
|
||||
"navigate_page": "navigate",
|
||||
"media_control": "control",
|
||||
"show_card": "show_card",
|
||||
"send_alert": "alert",
|
||||
}
|
||||
|
||||
|
||||
def _chat_once(messages, with_tools=True):
|
||||
log.info("llm: 调用模型 %s（%d 条消息, 带工具=%s）", settings.LLM_MODEL, len(messages), with_tools)
|
||||
payload = {
|
||||
"model": settings.LLM_MODEL,
|
||||
"messages": messages,
|
||||
"temperature": 0.6,
|
||||
}
|
||||
if with_tools:
|
||||
payload["tools"] = TOOLS
|
||||
req = urllib.request.Request(
|
||||
f"{settings.LLM_BASE_URL}/chat/completions",
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {settings.DASHSCOPE_API_KEY}",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=60, context=_SSL_CTX) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def _fallback(messages, screen_id=""):
|
||||
"""DashScope 不可用时回退本地规则引擎"""
|
||||
from .ai_tools import run_chat as rule_run
|
||||
|
||||
result = rule_run(messages)
|
||||
# 规则引擎产出的工具同样执行（MQTT 广播，带发起屏幕），并汇总发布结果
|
||||
_, all_ok = _exec_all(result.get("tools", []), screen_id)
|
||||
result["model"] = "rule-engine"
|
||||
result["mqtt_published"] = all_ok
|
||||
return result
|
||||
|
||||
|
||||
def _exec_all(tools, screen_id=""):
|
||||
"""执行工具列表（带发起屏幕 screen_id，工具结果只发到该屏），返回 (results, all_ok)"""
|
||||
from .ai_tools import run_tools
|
||||
|
||||
results, all_ok = run_tools(tools, screen_id)
|
||||
return results, all_ok
|
||||
|
||||
|
||||
def run_chat(messages, screen_id=""):
|
||||
"""入口：{reply, tools, model}
|
||||
tools 已在后端执行（MQTT 广播，带发起屏幕 screen_id），返回值供请求端本地同步执行"""
|
||||
if not settings.DASHSCOPE_API_KEY:
|
||||
log.warning("未配置 DASHSCOPE_API_KEY，使用本地规则引擎")
|
||||
return _fallback(messages, screen_id)
|
||||
|
||||
# 注入园区知识库（标准资料口径）：按用户 query 检索命中段落动态注入，
|
||||
# 替代原 1200 字截断全量注入（避免尾部知识丢失）。无命中时回退 brief()。
|
||||
system = SYSTEM_PROMPT
|
||||
kb = ""
|
||||
retrieved_count = 0
|
||||
try:
|
||||
from .rag import brief, retrieve
|
||||
query = ""
|
||||
for m in reversed(messages or []):
|
||||
if m.get("role") in ("user", "me"):
|
||||
query = (m.get("content") or "").strip()
|
||||
break
|
||||
hits = retrieve(query, top_k=4) if query else []
|
||||
if hits:
|
||||
retrieved_count = len(hits)
|
||||
kb = "\n\n".join(f"[{i + 1}] {c}" for i, c in enumerate(hits))
|
||||
log.info("llm: 知识库检索命中 %d 段（query=%s）", len(hits), _trunc(query, 120))
|
||||
else:
|
||||
kb = brief()
|
||||
if kb:
|
||||
system += f"\n\n【园区知识库 · 检索命中】\n{kb}"
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("llm: 知识库注入失败（仅用基础提示词）: %s", e)
|
||||
|
||||
log.info("llm: ── 对话开始（%d 条输入消息）──", len(messages or []))
|
||||
log.info("llm: 系统提示词=%s", _trunc(system, 700))
|
||||
if kb:
|
||||
log.info("llm: 注入知识库内容=%s", _trunc(kb, 500))
|
||||
for m in (messages or [])[-5:]:
|
||||
log.info("llm: 输入[%s] %s", m.get("role"), _trunc(m.get("content", ""), 300))
|
||||
|
||||
msgs = [{"role": "system", "content": system}] + [
|
||||
{"role": m.get("role") == "me" and "user" or m.get("role", "user"), "content": m.get("content", "")}
|
||||
for m in (messages or [])
|
||||
]
|
||||
|
||||
try:
|
||||
data = _chat_once(msgs)
|
||||
choice = data["choices"][0]["message"]
|
||||
reply = choice.get("content") or ""
|
||||
tool_calls = choice.get("tool_calls") or []
|
||||
executed = []
|
||||
|
||||
if tool_calls:
|
||||
log.info("llm: 模型请求调用 %d 个工具：", len(tool_calls))
|
||||
for tc in tool_calls:
|
||||
fn = tc.get("function", {})
|
||||
log.info("llm: 工具 %s args=%s", fn.get("name", ""), _trunc(fn.get("arguments", ""), 300))
|
||||
for tc in tool_calls:
|
||||
fn = tc.get("function", {})
|
||||
name = fn.get("name", "")
|
||||
try:
|
||||
args = json.loads(fn.get("arguments") or "{}")
|
||||
except json.JSONDecodeError:
|
||||
args = {}
|
||||
tool = {"type": _TOOL_MAP.get(name, name), "params": args}
|
||||
executed.append(tool)
|
||||
msgs.append({"role": "assistant", "content": None, "tool_calls": tool_calls})
|
||||
msgs.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.get("id", ""),
|
||||
"content": json.dumps({"ok": True}, ensure_ascii=False),
|
||||
})
|
||||
# 工具执行（MQTT 广播），并汇总发布结果
|
||||
results, all_ok = _exec_all(executed, screen_id)
|
||||
log.info("llm: 工具执行结果 %s", results)
|
||||
for t in executed:
|
||||
log.info("llm: 执行 %s %s", t["type"], json.dumps(t["params"], ensure_ascii=False))
|
||||
# 二次调用：携带工具结果生成最终回复
|
||||
try:
|
||||
data2 = _chat_once(msgs, with_tools=False)
|
||||
reply = data2["choices"][0]["message"].get("content") or reply
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("LLM 二次调用失败（保留工具回复）: %s", e)
|
||||
else:
|
||||
all_ok = True
|
||||
|
||||
if not reply:
|
||||
reply = "已完成操作。您还可以让我切换页面、控制播放或展示园区卡片。"
|
||||
|
||||
log.info("llm: 最终回复=%s", _trunc(reply, 600))
|
||||
log.info("llm: ── 对话结束 ──")
|
||||
return {
|
||||
"reply": reply,
|
||||
"tools": executed,
|
||||
"model": settings.LLM_MODEL,
|
||||
"mqtt_published": all_ok,
|
||||
"retrieved": retrieved_count > 0,
|
||||
"retrieved_count": retrieved_count,
|
||||
"tool_names": [t.get("type") for t in executed if t.get("type")],
|
||||
}
|
||||
except urllib.error.HTTPError as e:
|
||||
log.warning("DashScope HTTP %s: %s", e.code, e.read()[:300])
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("DashScope 调用失败，回退规则引擎: %s", e)
|
||||
|
||||
return _fallback(messages)
|
||||
@@ -0,0 +1,187 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""OPC 智能园区后端 —— FastAPI 入口
|
||||
启动：uvicorn app.main:app --host 0.0.0.0 --port 8000
|
||||
或：python main.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .admin import admin_login, admin_logout, admin_page
|
||||
from .config import settings
|
||||
from .event_bus import bus
|
||||
from .mqtt import hub
|
||||
from .routers import router
|
||||
from .sim_engine import sim_engine
|
||||
from .s2s_bridge import start_s2s_backend
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
log = logging.getLogger("dpm.main")
|
||||
|
||||
DIST = Path(settings.DIST_DIR)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# 启动准备
|
||||
settings.MEDIA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
bus.set_loop(asyncio.get_running_loop())
|
||||
hub.start()
|
||||
|
||||
stop = threading.Event()
|
||||
|
||||
# s2s 实时语音栈（可选）：后台线程构建管线池并起 RealtimeServer(:8765)
|
||||
if settings.S2S_ENABLED:
|
||||
start_s2s_backend(stop)
|
||||
|
||||
def tick_loop():
|
||||
"""数据引擎每 2.2s 推进一次；MQTT 连接时同步推送快照"""
|
||||
while not stop.is_set():
|
||||
try:
|
||||
snap = sim_engine.tick()
|
||||
hub.publish_tick(snap)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("数据 tick 异常: %s", e)
|
||||
stop.wait(settings.MQTT_TICK_INTERVAL)
|
||||
|
||||
threading.Thread(target=tick_loop, daemon=True).start()
|
||||
log.info("OPC 智能园区后端启动于 http://%s:%s", settings.HOST, settings.PORT)
|
||||
if settings.MQTT_ENABLED:
|
||||
log.info("MQTT Broker: %s:%s（前端 WS: %s）", settings.MQTT_HOST, settings.MQTT_PORT, settings.MQTT_WS_URL)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
stop.set()
|
||||
hub.stop()
|
||||
log.info("OPC 智能园区后端已停止")
|
||||
|
||||
|
||||
app = FastAPI(title="OPC 智能园区后端", version="1.0.0", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def no_cache_admin_static(request: Request, call_next):
|
||||
"""管理后台页面与静态资源禁用缓存，避免旧 JS/CSS 残留导致按钮无响应"""
|
||||
response = await call_next(request)
|
||||
if request.url.path.startswith(("/static/", "/admin")):
|
||||
response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
response.headers["Expires"] = "0"
|
||||
return response
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
# 媒体资源静态服务（上传/播放的文件）
|
||||
settings.MEDIA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
app.mount("/file", StaticFiles(directory=str(settings.MEDIA_DIR)), name="media")
|
||||
|
||||
# 管理后台静态资源（admin.css / admin.js）
|
||||
_STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
|
||||
app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static")
|
||||
|
||||
# 管理后台（FastAPI + Jinja2 服务端渲染）—— 必须在 SPA 回退之前注册
|
||||
app.add_api_route("/admin", admin_page, methods=["GET"], include_in_schema=False)
|
||||
app.add_api_route("/admin/login", admin_login, methods=["POST"], include_in_schema=False)
|
||||
app.add_api_route("/admin/logout", admin_logout, methods=["GET"], include_in_schema=False)
|
||||
|
||||
|
||||
# ---------- 客户端安装包下载（必须在 SPA 回退之前注册） ----------
|
||||
|
||||
_DOWNLOAD_ROOT = Path(__file__).resolve().parent.parent # backend/
|
||||
|
||||
|
||||
@app.get("/download", include_in_schema=False)
|
||||
@app.get("/11", include_in_schema=False)
|
||||
async def download_installer():
|
||||
"""下载 Windows 客户端安装包（backend/ 下 *.setup.exe，如 云超服..._x64-setup.exe）。"""
|
||||
for f in sorted(_DOWNLOAD_ROOT.glob("*setup.exe")):
|
||||
if f.is_file():
|
||||
return FileResponse(
|
||||
path=str(f),
|
||||
filename=f.name,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
return JSONResponse({"detail": "安装包不存在"}, status_code=404)
|
||||
|
||||
|
||||
# ---------- 客户端更新检测（仅 Windows：比对版本 → 下发安装包下载地址） ----------
|
||||
|
||||
def _latest_installer():
|
||||
"""返回 backend/ 下最新（按文件名排序取最大）的 *.setup.exe。"""
|
||||
for f in sorted(_DOWNLOAD_ROOT.glob("*setup.exe"), reverse=True):
|
||||
if f.is_file():
|
||||
return f
|
||||
return None
|
||||
|
||||
|
||||
def _version_from_name(name: str) -> str:
|
||||
"""从安装包文件名解析版本，如 `..._0.1.0_x64-setup.exe` → `0.1.0`。"""
|
||||
m = re.search(r"_(\d+\.\d+(?:\.\d+)?)", name or "")
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
def _version_key(v: str):
|
||||
return tuple(int(x) for x in v.split(".") if x.isdigit())
|
||||
|
||||
|
||||
class UpdateCheckBody(BaseModel):
|
||||
current: str = "" # 前端上报的当前版本，如 0.1.0
|
||||
|
||||
|
||||
@app.post("/api/update/check", include_in_schema=False)
|
||||
async def update_check(body: UpdateCheckBody):
|
||||
"""检测更新：返回最新版本号、安装包下载地址、是否需要更新（仅 Windows）。"""
|
||||
inst = _latest_installer()
|
||||
if not inst:
|
||||
return {"ok": False, "error": "installer not found"}
|
||||
latest = _version_from_name(inst.name)
|
||||
current = (body.current or "").strip()
|
||||
update_available = False
|
||||
if latest and current and current != latest:
|
||||
update_available = _version_key(latest) > _version_key(current)
|
||||
elif latest and not current:
|
||||
update_available = True
|
||||
return {
|
||||
"ok": True,
|
||||
"latest_version": latest,
|
||||
"installer_name": inst.name,
|
||||
"download_url": "/download",
|
||||
"update_available": update_available,
|
||||
}
|
||||
|
||||
|
||||
# ---------- 前端静态托管 + SPA 回退（浏览器直接访问 :8000 即可） ----------
|
||||
|
||||
@app.get("/{path:path}", include_in_schema=False)
|
||||
async def spa_fallback(path: str):
|
||||
if not DIST.exists():
|
||||
return JSONResponse({"detail": "前端未构建（dist 目录不存在）"}, status_code=404)
|
||||
dist = DIST.resolve()
|
||||
if not path:
|
||||
target = dist / "index.html"
|
||||
else:
|
||||
target = (dist / path).resolve()
|
||||
if target.is_file() and target.is_relative_to(dist):
|
||||
return FileResponse(target)
|
||||
index = dist / "index.html"
|
||||
if index.exists():
|
||||
return FileResponse(index)
|
||||
return JSONResponse({"detail": "Not Found"}, status_code=404)
|
||||
@@ -0,0 +1,242 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""MQTT 发布/订阅中心 —— 后端 ↔ 前端控制通道
|
||||
发布：opc/display/command（页面/媒体/卡片控制）、opc/dashboard/tick（数据快照）
|
||||
订阅：opc/display/heartbeat（大屏在线心跳，统计在线大屏数）
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import paho.mqtt.client as mqtt
|
||||
|
||||
from .config import settings
|
||||
from .event_bus import bus
|
||||
|
||||
log = logging.getLogger("dpm.mqtt")
|
||||
|
||||
|
||||
def new_cmd_id():
|
||||
return uuid.uuid4().hex[:12]
|
||||
|
||||
|
||||
def _unique_client_id(base):
|
||||
"""每个客户端进程使用唯一 client_id，避免与其它实例竞争被 broker 踢下线"""
|
||||
return f"{base}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
|
||||
|
||||
|
||||
class MqttHub:
|
||||
def __init__(self):
|
||||
self.client = None
|
||||
self.connected = False
|
||||
self._lock = threading.RLock()
|
||||
self.devices = {} # device_id(client_id) -> {role, parent_id, last_seen}
|
||||
self.last_command = None # {action, params, ts, published}
|
||||
|
||||
# ---------- 生命周期 ----------
|
||||
def start(self):
|
||||
if not settings.MQTT_ENABLED:
|
||||
log.info("MQTT 已禁用（DPM_MQTT_ENABLED=0）")
|
||||
return
|
||||
try:
|
||||
self.client = mqtt.Client(
|
||||
client_id=_unique_client_id(settings.MQTT_CLIENT_ID),
|
||||
clean_session=True,
|
||||
protocol=mqtt.MQTTv311,
|
||||
)
|
||||
if settings.MQTT_USERNAME:
|
||||
self.client.username_pw_set(settings.MQTT_USERNAME, settings.MQTT_PASSWORD)
|
||||
self.client.on_connect = self._on_connect
|
||||
self.client.on_disconnect = self._on_disconnect
|
||||
self.client.on_message = self._on_message
|
||||
self.client.connect_async(settings.MQTT_HOST, settings.MQTT_PORT, keepalive=30)
|
||||
self.client.loop_start()
|
||||
log.info("MQTT 连接中 %s:%s ...", settings.MQTT_HOST, settings.MQTT_PORT)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("MQTT 初始化失败（数据 API 不受影响）: %s", e)
|
||||
|
||||
def stop(self):
|
||||
if self.client:
|
||||
try:
|
||||
self.client.loop_stop()
|
||||
self.client.disconnect()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self.client = None
|
||||
|
||||
def _on_connect(self, client, userdata, flags, rc):
|
||||
if rc == 0:
|
||||
self.connected = True
|
||||
log.info("MQTT 已连接 %s:%s", settings.MQTT_HOST, settings.MQTT_PORT)
|
||||
client.subscribe(settings.TOPIC_HEARTBEAT, qos=0)
|
||||
else:
|
||||
log.warning("MQTT 连接失败 rc=%s", rc)
|
||||
|
||||
def _on_disconnect(self, client, userdata, rc):
|
||||
self.connected = False
|
||||
if rc != 0:
|
||||
log.warning("MQTT 断开（rc=%s），自动重连中...", rc)
|
||||
|
||||
def _on_message(self, client, userdata, msg):
|
||||
"""接收大屏心跳：更新设备表在线时间（设备由注册表登记 role/parent）"""
|
||||
if msg.topic == settings.TOPIC_HEARTBEAT:
|
||||
try:
|
||||
payload = json.loads(msg.payload.decode("utf-8"))
|
||||
cid = payload.get("client_id") or msg.topic
|
||||
with self._lock:
|
||||
cur = self.devices.get(cid, {"role": "", "parent_id": ""})
|
||||
cur["last_seen"] = time.time()
|
||||
self.devices[cid] = cur
|
||||
self._prune()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def _prune(self):
|
||||
"""剔除超时未上报的设备"""
|
||||
cutoff = time.time() - settings.SCREEN_TTL
|
||||
self.devices = {k: v for k, v in self.devices.items() if v.get("last_seen", 0) > cutoff}
|
||||
|
||||
# ---------- 状态查询 ----------
|
||||
def screens_online(self):
|
||||
with self._lock:
|
||||
return sum(1 for v in self.devices.values() if v.get("last_seen", 0) > time.time() - settings.SCREEN_TTL)
|
||||
|
||||
def status(self):
|
||||
with self._lock:
|
||||
self._prune()
|
||||
devices = [
|
||||
{
|
||||
"device_id": cid,
|
||||
"role": v.get("role", ""),
|
||||
"parent_id": v.get("parent_id", ""),
|
||||
}
|
||||
for cid, v in self.devices.items()
|
||||
]
|
||||
return {
|
||||
"mqtt_connected": self.connected,
|
||||
"mqtt_host": f"{settings.MQTT_HOST}:{settings.MQTT_PORT}",
|
||||
"screens_online": len(devices),
|
||||
"devices": devices,
|
||||
"last_command": self.last_command,
|
||||
}
|
||||
|
||||
# ---------- 发布 ----------
|
||||
def publish(self, topic, payload, qos=1, retain=False):
|
||||
if not self.client or not self.connected:
|
||||
log.debug("MQTT 未连接，丢弃发布 %s", topic)
|
||||
return False
|
||||
try:
|
||||
info = self.client.publish(topic, json.dumps(payload, ensure_ascii=False), qos=qos, retain=retain)
|
||||
return info.rc == mqtt.MQTT_ERR_SUCCESS
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("MQTT 发布失败: %s", e)
|
||||
return False
|
||||
|
||||
def register_device(self, device_id, role="", parent_id=""):
|
||||
"""前端上报设备：唯一 device_id + 角色(main/secondary) + 从属主屏 parent_id。"""
|
||||
device_id = (device_id or "").strip()
|
||||
if not device_id:
|
||||
return
|
||||
with self._lock:
|
||||
cur = self.devices.get(device_id, {})
|
||||
if role:
|
||||
cur["role"] = role
|
||||
if parent_id:
|
||||
cur["parent_id"] = parent_id
|
||||
cur["last_seen"] = time.time()
|
||||
self.devices[device_id] = cur
|
||||
self._prune()
|
||||
log.info("mqtt: 设备注册 id=%s role=%s parent=%s", device_id, role or cur.get("role"), parent_id or cur.get("parent_id"))
|
||||
|
||||
def publish_command(self, action, params=None, screen_id=None, screen_role=None):
|
||||
"""按客户端(设备)路由下发指令：
|
||||
· screen_id 精确到某台设备 → 只发到该设备独有 topic
|
||||
· screen_role(main/secondary) → 解析到该角色的设备，发到其独有 topic
|
||||
· 两者皆空（全部）→ 遍历在线设备列表，依次下发到各自独有 topic
|
||||
客户端只订阅自己的独有频道，因此定向指令绝不会被其它屏幕收到。"""
|
||||
payload = {
|
||||
"cmd_id": new_cmd_id(),
|
||||
"ts": int(time.time() * 1000),
|
||||
"action": action,
|
||||
"params": params or {},
|
||||
}
|
||||
target = None
|
||||
requested = bool(screen_id or screen_role in ("main", "secondary"))
|
||||
if screen_id:
|
||||
target = (screen_id or "").strip()
|
||||
elif screen_role in ("main", "secondary"):
|
||||
with self._lock:
|
||||
for cid, v in self.devices.items():
|
||||
if v.get("role") == screen_role:
|
||||
target = cid
|
||||
break
|
||||
if target:
|
||||
# 定向：只发到目标设备独有频道
|
||||
return self._publish_to(payload, target, screen_role)
|
||||
if requested:
|
||||
# 指定了目标但未找到对应设备：不广播，避免误发到全部
|
||||
log.warning("mqtt: 目标设备未找到（id=%s role=%s），未下发", screen_id, screen_role)
|
||||
payload["published"] = False
|
||||
with self._lock:
|
||||
self.last_command = {"action": action, "params": params or {}, "ts": payload["ts"], "published": False, "topic": "none"}
|
||||
return payload
|
||||
|
||||
# 全部：遍历在线设备列表，依次下发到各自独有频道
|
||||
with self._lock:
|
||||
cutoff = time.time() - settings.SCREEN_TTL
|
||||
online = [cid for cid, v in self.devices.items() if v.get("last_seen", 0) > cutoff]
|
||||
ok = True
|
||||
published_topics = []
|
||||
for cid in online:
|
||||
p = dict(payload)
|
||||
p["screen_id"] = cid
|
||||
with self._lock:
|
||||
p["screen_role"] = self.devices.get(cid, {}).get("role", "")
|
||||
if self.publish(f"{settings.TOPIC_COMMAND}/{cid}", p):
|
||||
bus.emit(p)
|
||||
published_topics.append(cid)
|
||||
else:
|
||||
ok = False
|
||||
payload["published"] = ok
|
||||
payload["screen_id"] = ""
|
||||
payload["screen_role"] = "both"
|
||||
with self._lock:
|
||||
self.last_command = {
|
||||
"action": action,
|
||||
"params": params or {},
|
||||
"ts": payload["ts"],
|
||||
"published": ok,
|
||||
"topic": "all",
|
||||
}
|
||||
log.info("mqtt: 全部下发 %s 到 %d 台设备", action, len(published_topics))
|
||||
return payload
|
||||
|
||||
def _publish_to(self, payload, target, screen_role=""):
|
||||
"""发到指定设备独有频道，并记录 last_command。"""
|
||||
topic = f"{settings.TOPIC_COMMAND}/{target}"
|
||||
payload["screen_id"] = target
|
||||
with self._lock:
|
||||
payload["screen_role"] = self.devices.get(target, {}).get("role", screen_role or "")
|
||||
ok = self.publish(topic, payload)
|
||||
payload["published"] = ok
|
||||
with self._lock:
|
||||
self.last_command = {
|
||||
"action": payload["action"],
|
||||
"params": payload.get("params") or {},
|
||||
"ts": payload["ts"],
|
||||
"published": ok,
|
||||
"topic": topic,
|
||||
}
|
||||
if ok:
|
||||
bus.emit(payload)
|
||||
return payload
|
||||
|
||||
def publish_tick(self, snapshot):
|
||||
payload = {"ts": int(time.time() * 1000), "snapshot": snapshot}
|
||||
self.publish(settings.TOPIC_TICK, payload, qos=0)
|
||||
|
||||
|
||||
hub = MqttHub()
|
||||
@@ -0,0 +1,460 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""园区知识库 —— 轻量本地 RAG（embedding + numpy 余弦检索，双路索引）
|
||||
|
||||
升级为「细粒度分块 + LLM 生成问答对 + 双路向量索引（原句 + 问题）+ 余弦检索返回原文」：
|
||||
- 知识源：backend/knowledge/park.md（按 H2 节 → 原子事实两级分块）
|
||||
- 向量化：dashscope text-embedding-v4（云端 API，索引构建一次后缓存，构建时 L2 归一化）
|
||||
- 问答：复用 llm._chat_once 对每块生成 3-5 个口语化问句，缓存到 kb_qa.json（无 key 自动单路降级）
|
||||
- 检索：query 同时比对原句向量与问题向量，按块去重，返回 top_k 个原文块
|
||||
- 提供：
|
||||
- retrieve(query, top_k) → 检索命中的原文块（对话动态注入）
|
||||
- full_kb() → 整库原文（小库全量注入，供语音 instructions）
|
||||
- brief() → 前 N 字摘要（兼容 /api/kb/brief 与检索无命中回退）
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
log = logging.getLogger("dpm.rag")
|
||||
|
||||
|
||||
def _trunc(s, n=300):
|
||||
"""日志用截断：超长内容只留前 n 字并标注总长。"""
|
||||
s = str(s)
|
||||
return s if len(s) <= n else s[:n] + f"...（共{len(s)}字）"
|
||||
|
||||
|
||||
KB_DIR = Path(__file__).resolve().parent.parent / "knowledge"
|
||||
KB_GLOB = "*.md" # 知识库 = knowledge/ 下全部 markdown（park.md + opc.md 等）
|
||||
META_FILE = KB_DIR / "kb_meta.json" # 元数据（JSON，可读）：model/hash/chunks/questions/embed_texts/embed_owner
|
||||
VECTOR_FILE = KB_DIR / "kb_vectors.npy" # 向量矩阵（二进制 numpy float32，快载小体积）
|
||||
LEGACY_INDEX_FILE = KB_DIR / "kb_index.json" # 旧版单文件索引（迁移后删除）
|
||||
QA_FILE = KB_DIR / "kb_qa.json"
|
||||
EMBED_MODEL = "text-embedding-v4"
|
||||
EMBED_BATCH = 10 # DashScope embedding 单批上限
|
||||
QA_BATCH = 5 # 每批几块生成问答
|
||||
CHUNK_MAX = 240 # 单块超过此长度按句拆
|
||||
|
||||
_index = None # 缓存 {chunks, questions, embed_texts, embed_owner, embeddings(归一化 np.ndarray)}
|
||||
|
||||
|
||||
def _kb_text() -> str:
|
||||
"""拼接 knowledge/ 下全部 markdown 内容（按文件名排序，稳定顺序）。"""
|
||||
parts = []
|
||||
for f in sorted(KB_DIR.glob(KB_GLOB)):
|
||||
try:
|
||||
parts.append(f.read_text("utf-8"))
|
||||
except OSError as e: # noqa: BLE001
|
||||
log.warning("rag: 跳过知识文件 %s: %s", f.name, e)
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _data_hash() -> str:
|
||||
"""全部知识文件内容签名，用于索引/问答缓存失效判断。"""
|
||||
return hashlib.sha1(_kb_text().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _split_long(text: str, limit: int = CHUNK_MAX) -> list[str]:
|
||||
"""超长块按句号/分号/顿号等边界拆成多块，保留语义完整。"""
|
||||
if len(text) <= limit:
|
||||
return [text]
|
||||
parts: list[str] = []
|
||||
cur = ""
|
||||
for seg in re.split(r"(?<=[。！？；、])", text):
|
||||
if len(cur) + len(seg) > limit and cur:
|
||||
parts.append(cur.strip())
|
||||
cur = ""
|
||||
cur += seg
|
||||
if cur.strip():
|
||||
parts.append(cur.strip())
|
||||
return [p for p in parts if len(p) > 8]
|
||||
|
||||
|
||||
def _chunk_md(text: str) -> list[str]:
|
||||
"""两级切分：H2 节标题作上下文前缀 + 节内原子事实。
|
||||
|
||||
企业条目（加粗公司名行 `**...｜公司名**` + 负责人/简介列表）合并为单个块，
|
||||
保证「企业名称 + 企业介绍」作为一个整体可被按名召回。
|
||||
"""
|
||||
chunks: list[str] = []
|
||||
section = ""
|
||||
para: list[str] = []
|
||||
company: list[str] | None = None # 正在累积的单个企业块
|
||||
|
||||
def flush_para():
|
||||
nonlocal para
|
||||
if para:
|
||||
t = "\n".join(para).strip()
|
||||
if len(t) > 8:
|
||||
chunks.extend(_split_long(t))
|
||||
para = []
|
||||
|
||||
def flush_company():
|
||||
nonlocal company
|
||||
if company:
|
||||
t = "\n".join(company).strip()
|
||||
if len(t) > 8:
|
||||
chunks.append(t) # 企业块不拆，保证名称+介绍同块
|
||||
company = None
|
||||
|
||||
def add_fact(content: str):
|
||||
full = f"{section}\n{content}" if section else content
|
||||
if len(full) > 8:
|
||||
chunks.extend(_split_long(full))
|
||||
|
||||
for line in text.splitlines():
|
||||
s = line.rstrip()
|
||||
if s.startswith("# "): # H1：不产块
|
||||
flush_para(); flush_company()
|
||||
continue
|
||||
if s.startswith("## "): # H2 小节：切换上下文前缀
|
||||
flush_para(); flush_company()
|
||||
section = s[3:].strip()
|
||||
continue
|
||||
if s.startswith("###"): # H3 子节：上下文（如分区），flush 企业
|
||||
flush_para(); flush_company()
|
||||
para.append(s.strip())
|
||||
continue
|
||||
stripped = s.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
# 企业条目：加粗公司名行作为单个企业块的起点
|
||||
if stripped.startswith("**") and "｜" in stripped:
|
||||
flush_para(); flush_company()
|
||||
company = [stripped]
|
||||
continue
|
||||
if company is not None:
|
||||
# 企业块的跟随行（负责人/简介等列表项）
|
||||
if stripped.startswith("- ") or stripped.startswith("问：") or re.match(r"^\d+[\.、]\s", stripped):
|
||||
company.append(stripped)
|
||||
else:
|
||||
flush_company()
|
||||
para.append(stripped)
|
||||
continue
|
||||
if stripped.startswith("- "): # 列表项 = 原子事实
|
||||
flush_para(); add_fact(stripped)
|
||||
elif re.match(r"^\d+[\.、]\s", stripped): # 编号步
|
||||
flush_para(); add_fact(stripped)
|
||||
elif stripped.startswith("问："): # 单条问答
|
||||
flush_para(); add_fact(stripped)
|
||||
else: # 普通段落：累积后整段成块（超长再拆）
|
||||
para.append(stripped)
|
||||
flush_para()
|
||||
flush_company()
|
||||
return chunks
|
||||
|
||||
|
||||
def _company_name(chunk: str) -> str:
|
||||
"""从企业块提取企业名称：`**加速1｜云南派音人工智能科技有限公司**` → `云南派音人工智能科技有限公司`。"""
|
||||
m = re.match(r"^\*\*[^｜\n]*｜([^*]+)\*\*", chunk) or re.match(r"^\*\*([^*]+)\*\*", chunk)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def _embed(texts: list[str]) -> list[list[float]]:
|
||||
"""dashscope text-embedding-v4 批量向量化（每批 ≤10 条，按 text_index 排序拼接）。"""
|
||||
import dashscope
|
||||
|
||||
api_key = os.environ.get("DASHSCOPE_API_KEY") or ""
|
||||
out: list[list[float]] = []
|
||||
for i in range(0, len(texts), EMBED_BATCH):
|
||||
batch = texts[i:i + EMBED_BATCH]
|
||||
resp = dashscope.TextEmbedding.call(
|
||||
model=EMBED_MODEL,
|
||||
input=batch,
|
||||
api_key=api_key,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"embedding API {resp.status_code}: {resp.code} {resp.message}"
|
||||
)
|
||||
embs = sorted(
|
||||
resp.output["embeddings"],
|
||||
key=lambda e: e.get("text_index", e.get("index", 0)),
|
||||
)
|
||||
out.extend(e["embedding"] for e in embs)
|
||||
log.info("rag: embedding 完成 %d 条（%s，每批 %d）", len(texts), EMBED_MODEL, EMBED_BATCH)
|
||||
return out
|
||||
|
||||
|
||||
def _normalize(vectors: list[list[float]]) -> np.ndarray:
|
||||
"""转为 float32 np.ndarray 并做 L2 行归一化（点积即余弦相似度）。"""
|
||||
arr = np.asarray(vectors, dtype=np.float32)
|
||||
norms = np.linalg.norm(arr, axis=1, keepdims=True)
|
||||
norms[norms == 0] = 1.0
|
||||
return arr / norms
|
||||
|
||||
|
||||
# ── LLM 问答生成 ────────────────────────────────────────────────
|
||||
|
||||
def _parse_json_array(text: str) -> list:
|
||||
"""解析模型返回的 JSON 数组：剥 markdown 围栏 → json.loads → 截取首尾方括号兜底。"""
|
||||
text = (text or "").strip()
|
||||
if text.startswith("```"):
|
||||
lines = [l for l in text.splitlines() if not l.strip().startswith("```")]
|
||||
text = "\n".join(lines).strip()
|
||||
for candidate in (text,):
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except Exception:
|
||||
pass
|
||||
i = text.find("[")
|
||||
j = text.rfind("]")
|
||||
if i != -1 and j > i:
|
||||
try:
|
||||
return json.loads(text[i:j + 1])
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def _qa_for_batch(chunks: list[str]) -> list[list[str]]:
|
||||
"""对一批分块，用 LLM 生成每块的 3-5 个口语化问句（输出 JSON 数组，与块一一对应）。"""
|
||||
from .llm import _chat_once
|
||||
|
||||
numbered = "\n\n".join(f"[{i}]\n{c}" for i, c in enumerate(chunks))
|
||||
|
||||
def build_prompt():
|
||||
return (
|
||||
"下面是园区知识库的若干分块，每块标有 [编号]。"
|
||||
"请为每一块生成 3-5 个用户可能会问的自然口语化中文问句，"
|
||||
"覆盖该块全部信息点与常见同义问法。"
|
||||
f"严格只输出一个 JSON 数组，数组长度必须为 {len(chunks)}，"
|
||||
"第 i 个元素是第 i 块的问题字符串数组。不要输出任何其他文字或解释。\n\n"
|
||||
+ numbered
|
||||
)
|
||||
|
||||
for attempt in range(2):
|
||||
try:
|
||||
data = _chat_once([{"role": "user", "content": build_prompt()}], with_tools=False)
|
||||
content = data["choices"][0]["message"].get("content") or ""
|
||||
arr = _parse_json_array(content)
|
||||
if isinstance(arr, list) and arr:
|
||||
out: list[list[str]] = []
|
||||
for i in range(len(chunks)):
|
||||
qs = arr[i] if i < len(arr) and isinstance(arr[i], list) else []
|
||||
out.append([str(q).strip() for q in qs if str(q).strip()][:5])
|
||||
return out
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("rag: QA 批生成失败（尝试 %d/%d）: %s", attempt + 1, 2, e)
|
||||
return [[] for _ in chunks]
|
||||
|
||||
|
||||
def _first_line(chunk: str) -> str:
|
||||
"""取块首行（去标题），用作无问句时的兜底检索问题。"""
|
||||
for ln in chunk.splitlines():
|
||||
ln = ln.strip()
|
||||
if ln and not ln.startswith("#"):
|
||||
return ln
|
||||
return chunk[:60]
|
||||
|
||||
|
||||
def _ensure_questions(chunks: list[str], questions) -> list[list[str]]:
|
||||
"""确保每块都有至少一个检索问题：未生成问句的块，用块首句兜底（保证全量覆盖）。"""
|
||||
qs: list[list[str]] = [list(q or []) for q in (questions or [])]
|
||||
if len(qs) < len(chunks):
|
||||
qs.extend([] for _ in range(len(chunks) - len(qs)))
|
||||
for i, c in enumerate(chunks):
|
||||
if not qs[i]:
|
||||
qs[i] = [_first_line(c)]
|
||||
return qs
|
||||
|
||||
|
||||
def _load_questions(chunks: list[str]) -> list[list[str]]:
|
||||
"""加载/生成问答缓存；返回与 chunks 对齐的问题数组（保证每块 ≥1 个问题）。无 key 时单路回退。"""
|
||||
from .config import settings
|
||||
|
||||
if not settings.DASHSCOPE_API_KEY:
|
||||
log.warning("rag: 无 DASHSCOPE_API_KEY，跳过问答生成（单路检索）")
|
||||
return _ensure_questions(chunks, [[] for _ in chunks])
|
||||
h = _data_hash()
|
||||
if QA_FILE.exists():
|
||||
try:
|
||||
data = json.loads(QA_FILE.read_text("utf-8"))
|
||||
if data.get("data_hash") == h and data.get("chunks") == chunks:
|
||||
log.info("rag: 问答缓存已加载（%d 块）", len(chunks))
|
||||
return _ensure_questions(chunks, data["questions"])
|
||||
except Exception: # noqa: BLE001
|
||||
log.warning("rag: 问答缓存损坏，重建")
|
||||
questions: list[list[str]] = []
|
||||
total = len(chunks)
|
||||
for i in range(0, total, QA_BATCH):
|
||||
questions.extend(_qa_for_batch(chunks[i:i + QA_BATCH]))
|
||||
log.info("rag: 问答生成进度 %d/%d", min(i + QA_BATCH, total), total)
|
||||
questions = _ensure_questions(chunks, questions)
|
||||
QA_FILE.write_text(
|
||||
json.dumps({"data_hash": h, "chunks": chunks, "questions": questions}, ensure_ascii=False),
|
||||
"utf-8",
|
||||
)
|
||||
log.info("rag: 问答缓存已生成（%d 块）-> %s", len(chunks), QA_FILE)
|
||||
return questions
|
||||
|
||||
|
||||
# ── 双路向量索引 ────────────────────────────────────────────────
|
||||
|
||||
def _migrate_legacy():
|
||||
"""把旧版单文件 kb_index.json（向量内嵌在 JSON）就地转为 kb_vectors.npy + kb_meta.json，再删除旧文件。"""
|
||||
if not LEGACY_INDEX_FILE.exists() or META_FILE.exists():
|
||||
return
|
||||
try:
|
||||
d = json.loads(LEGACY_INDEX_FILE.read_text("utf-8"))
|
||||
np.save(VECTOR_FILE, np.asarray(d["embeddings"], dtype=np.float32))
|
||||
META_FILE.write_text(
|
||||
json.dumps({
|
||||
"model": d.get("model"),
|
||||
"data_hash": d.get("data_hash"),
|
||||
"chunks": d["chunks"],
|
||||
"questions": d.get("questions", []),
|
||||
"embed_texts": d["embed_texts"],
|
||||
"embed_owner": d["embed_owner"],
|
||||
}, ensure_ascii=False),
|
||||
"utf-8",
|
||||
)
|
||||
LEGACY_INDEX_FILE.unlink()
|
||||
log.info("rag: 已迁移旧 kb_index.json -> kb_vectors.npy + kb_meta.json（无重建，未调 API）")
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("rag: 旧索引迁移失败（将重建）: %s", e)
|
||||
|
||||
|
||||
def _load_index():
|
||||
"""构建/加载双路向量索引（原句 + 生成问题）。向量存二进制 .npy，元数据存 JSON。"""
|
||||
global _index
|
||||
if _index is not None:
|
||||
return _index
|
||||
_migrate_legacy()
|
||||
h = _data_hash()
|
||||
if META_FILE.exists() and VECTOR_FILE.exists():
|
||||
try:
|
||||
data = json.loads(META_FILE.read_text("utf-8"))
|
||||
valid = (
|
||||
data.get("model") == EMBED_MODEL
|
||||
and data.get("data_hash") == h
|
||||
and len(data.get("embed_texts", [])) == len(data.get("embed_owner", []))
|
||||
)
|
||||
if valid:
|
||||
vectors = np.load(VECTOR_FILE) # 已归一化 float32
|
||||
if len(vectors) == len(data["embed_texts"]):
|
||||
_index = {
|
||||
"chunks": data["chunks"],
|
||||
"questions": data.get("questions", []),
|
||||
"embed_texts": data["embed_texts"],
|
||||
"embed_owner": data["embed_owner"],
|
||||
"embeddings": vectors,
|
||||
}
|
||||
log.info("rag: 双路知识索引已加载（%d 块 → %d 向量, %s）",
|
||||
len(_index["chunks"]), len(_index["embed_texts"]), EMBED_MODEL)
|
||||
return _index
|
||||
log.warning("rag: 索引失效（model/hash/长度不符），重建")
|
||||
except Exception: # noqa: BLE001
|
||||
log.warning("rag: 索引损坏，重建")
|
||||
# 构建
|
||||
chunks = _chunk_md(_kb_text())
|
||||
questions = _load_questions(chunks)
|
||||
embed_texts: list[str] = []
|
||||
embed_owner: list[int] = []
|
||||
for i, c in enumerate(chunks):
|
||||
embed_texts.append(c)
|
||||
embed_owner.append(i)
|
||||
# 企业块：把「企业名称」作为确定性问题加入索引，保证按名称可召回（名称为问、介绍为答）
|
||||
cname = _company_name(c)
|
||||
if cname:
|
||||
embed_texts.append(cname)
|
||||
embed_owner.append(i)
|
||||
for q in questions[i]:
|
||||
embed_texts.append(q)
|
||||
embed_owner.append(i)
|
||||
log.info("rag: 构建双路索引（%d 块 → %d 向量，%s embedding 中…）",
|
||||
len(chunks), len(embed_texts), EMBED_MODEL)
|
||||
arr = _normalize(_embed(embed_texts))
|
||||
_index = {
|
||||
"chunks": chunks,
|
||||
"questions": questions,
|
||||
"embed_texts": embed_texts,
|
||||
"embed_owner": embed_owner,
|
||||
"embeddings": arr,
|
||||
}
|
||||
np.save(VECTOR_FILE, arr)
|
||||
META_FILE.write_text(
|
||||
json.dumps({
|
||||
"model": EMBED_MODEL,
|
||||
"data_hash": h,
|
||||
"chunks": chunks,
|
||||
"questions": questions,
|
||||
"embed_texts": embed_texts,
|
||||
"embed_owner": embed_owner,
|
||||
}, ensure_ascii=False),
|
||||
"utf-8",
|
||||
)
|
||||
log.info("rag: 双路索引构建完成（%d 块, dim=%d）-> %s + %s",
|
||||
len(chunks), arr.shape[1], VECTOR_FILE, META_FILE)
|
||||
return _index
|
||||
|
||||
|
||||
# ── 对外接口 ────────────────────────────────────────────────────
|
||||
|
||||
def retrieve(query: str, top_k: int = 3, threshold: float = 0.3) -> list[str]:
|
||||
"""双路检索：query 同时比对原句与问题向量，按块去重，返回 top_k 个原文块。"""
|
||||
if not query or not query.strip():
|
||||
return []
|
||||
idx = _load_index()
|
||||
q = _normalize([_embed([query])[0]])[0]
|
||||
sims = np.dot(idx["embeddings"], q)
|
||||
order = np.argsort(-sims)
|
||||
best: dict[int, float] = {} # owner -> 该块最高分
|
||||
for e in order:
|
||||
owner = idx["embed_owner"][int(e)]
|
||||
sc = float(sims[int(e)])
|
||||
if sc < threshold:
|
||||
break
|
||||
if owner not in best:
|
||||
best[owner] = sc
|
||||
if len(best) >= top_k:
|
||||
break
|
||||
ordered = sorted(best.items(), key=lambda kv: kv[1], reverse=True)
|
||||
hits = [idx["chunks"][o] for o, _ in ordered]
|
||||
log.info("rag: retrieve() query=%s top_k=%d threshold=%.2f -> 命中 %d 段",
|
||||
_trunc(query, 120), top_k, threshold, len(hits))
|
||||
for j, (o, sc) in enumerate(ordered):
|
||||
log.info("rag: [%d] %.4f %s", j, sc, _trunc(idx["chunks"][o], 300))
|
||||
return hits
|
||||
|
||||
|
||||
def brief(max_chars: int = 1200) -> str:
|
||||
"""固定知识摘要（前 N 字；兼容 /api/kb/brief 与检索无命中回退）。"""
|
||||
idx = _load_index()
|
||||
out = []
|
||||
total = 0
|
||||
for c in idx["chunks"]:
|
||||
if total + len(c) > max_chars:
|
||||
break
|
||||
out.append(c)
|
||||
total += len(c)
|
||||
text = "\n\n".join(out)
|
||||
log.info("rag: brief() 知识库摘要 %d 字 -> %s", total, _trunc(text, 300))
|
||||
return text
|
||||
|
||||
|
||||
def full_kb() -> str:
|
||||
"""整库原文（小库全量注入，用于语音 instructions，修复尾部知识丢失）。"""
|
||||
idx = _load_index()
|
||||
text = "\n\n".join(idx["chunks"])
|
||||
log.info("rag: full_kb() 全库 %d 字 / %d 块", len(text), len(idx["chunks"]))
|
||||
return text
|
||||
|
||||
|
||||
def build_instructions() -> str:
|
||||
"""语音助手的完整基础提示词 = 配置提示词 + 整库知识（服务端统一组装）。"""
|
||||
from .config import settings
|
||||
base = settings.S2S_INSTRUCTIONS.strip()
|
||||
try:
|
||||
kb = full_kb()
|
||||
if kb:
|
||||
base += f"\n\n【园区知识库】\n{kb}"
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("rag: 知识库拼接失败（仅用基础提示词）: %s", e)
|
||||
log.info("rag: build_instructions() 语音系统提示词=%s", _trunc(base, 600))
|
||||
return base
|
||||
@@ -0,0 +1,660 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""全部 REST 路由 —— 数据 / 媒体 / 播放列表 / 设置 / AI / 展示控制"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Request, UploadFile, File
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .ai_tools import run_chat, run_tools
|
||||
from .asr import transcribe as asr_transcribe
|
||||
from .config import settings
|
||||
from .event_bus import bus
|
||||
from .llm import run_chat as llm_run_chat
|
||||
from .mqtt import hub
|
||||
from .sim_engine import sim_engine
|
||||
from .storage import storage
|
||||
|
||||
log = logging.getLogger("dpm.api")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
ALLOWED_EXT = {".mp4", ".mkv", ".avi", ".jpg", ".jpeg", ".png"}
|
||||
|
||||
|
||||
def _media_type(path):
|
||||
lower = path.lower()
|
||||
if lower.endswith((".mp4", ".mkv", ".avi")):
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
|
||||
# ==================== 数据模型 ====================
|
||||
|
||||
class SettingsBody(BaseModel):
|
||||
volume: int | None = None
|
||||
sfx_volume: int | None = None
|
||||
play_mode: str | None = None
|
||||
image_duration: int | None = None
|
||||
fullscreen: bool | None = None
|
||||
autostart: bool | None = None
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
|
||||
|
||||
class ActionBody(BaseModel):
|
||||
action: str
|
||||
|
||||
|
||||
class PathBody(BaseModel):
|
||||
path: str
|
||||
|
||||
|
||||
class UrlBody(BaseModel):
|
||||
url: str
|
||||
|
||||
|
||||
class StatePayload(BaseModel):
|
||||
status: str = ""
|
||||
index: int = 0
|
||||
name: str = ""
|
||||
media_type: str = ""
|
||||
|
||||
|
||||
class StateBody(BaseModel):
|
||||
state: StatePayload
|
||||
|
||||
|
||||
class DisplayCommandBody(BaseModel):
|
||||
action: str
|
||||
params: dict = Field(default_factory=dict)
|
||||
screen_id: str = "" # 目标屏幕唯一 client_id（精确到某台）
|
||||
screen_role: str = "" # main | secondary（按角色定位）
|
||||
|
||||
|
||||
class ChatMessage(BaseModel):
|
||||
role: str
|
||||
content: str
|
||||
|
||||
|
||||
class ChatBody(BaseModel):
|
||||
messages: list[ChatMessage]
|
||||
screen_id: str = "" # 发起对话的屏幕设备 id（工具结果只发到该屏）
|
||||
|
||||
|
||||
# ==================== 认证 / 设置 ====================
|
||||
|
||||
@router.get("/api/health")
|
||||
async def health():
|
||||
"""健康检查（Docker healthcheck / 运维探活）"""
|
||||
return {
|
||||
"ok": True,
|
||||
"service": "dpm-backend",
|
||||
"mqtt": hub.connected if getattr(hub, "connected", None) is not None else False,
|
||||
"ts": __import__("time").time(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/login")
|
||||
async def login(request: Request):
|
||||
form = await request.form()
|
||||
s = storage.get_settings()
|
||||
return JSONResponse({"success": form.get("username") == s["username"] and form.get("password") == s["password"]})
|
||||
|
||||
|
||||
@router.get("/api/settings")
|
||||
async def get_settings():
|
||||
s = storage.get_settings()
|
||||
return {k: s[k] for k in ("volume", "sfx_volume", "play_mode", "image_duration", "fullscreen", "autostart")}
|
||||
|
||||
|
||||
@router.get("/api/config")
|
||||
async def runtime_config(request: Request):
|
||||
"""运行配置（供前端启动引导覆盖）：MQTT 地址/账号、API 基址、语音地址
|
||||
—— 打包部署时展播端从后端拉取，避免构建期写死的局域网 IP 失效
|
||||
"""
|
||||
ws = settings.MQTT_WS_URL
|
||||
# 后端可达主机：优先取客户端实际访问本服务的 Host（与 api_base 一致），
|
||||
# 回退到从 MQTT broker 地址提取。
|
||||
host = "192.168.1.9"
|
||||
try:
|
||||
host = ws.split("://", 1)[1].split(":", 1)[0]
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
rh = (request.headers.get("host") or "").split(":")[0]
|
||||
if rh:
|
||||
host = rh
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
# s2s 与后端同机启动 → voice_url 默认跟随后端主机；显式 DPM_VOICE_WS 优先
|
||||
voice_url = settings.S2S_WS_URL
|
||||
if not settings.VOICE_WS and host:
|
||||
voice_url = f"ws://{host}:{settings.S2S_PORT}/v1/realtime"
|
||||
return {
|
||||
"ok": True,
|
||||
"mqtt_url": ws,
|
||||
"mqtt_username": settings.MQTT_USERNAME or "",
|
||||
"mqtt_password": settings.MQTT_PASSWORD or "",
|
||||
"voice_url": voice_url,
|
||||
"api_base": f"http://{host}:{settings.PORT}",
|
||||
"broker_host": host,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/settings")
|
||||
async def update_settings(body: SettingsBody):
|
||||
storage.update_settings(**body.model_dump(exclude_none=True))
|
||||
s = storage.get_settings()
|
||||
hub.publish_command("settings_changed", {
|
||||
"volume": s["volume"], "sfx_volume": s["sfx_volume"], "play_mode": s["play_mode"],
|
||||
})
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ==================== 媒体 ====================
|
||||
|
||||
def _list_media():
|
||||
files = []
|
||||
media_dir = Path(settings.MEDIA_DIR)
|
||||
if media_dir.exists():
|
||||
for p in sorted(media_dir.iterdir()):
|
||||
if not p.is_file() or p.suffix.lower() not in ALLOWED_EXT:
|
||||
continue
|
||||
files.append({
|
||||
"name": p.name, "relative_path": p.name,
|
||||
"type": _media_type(p.name), "url": f"/file/{p.name}", "source": "local",
|
||||
})
|
||||
for item in storage.get_url_media():
|
||||
files.append({
|
||||
"name": item.get("name", item.get("url", "")),
|
||||
"relative_path": item.get("url", ""),
|
||||
"type": item.get("type", "image"),
|
||||
"url": item.get("url", ""),
|
||||
"source": "url",
|
||||
})
|
||||
return {"files": files}
|
||||
|
||||
|
||||
@router.get("/media")
|
||||
async def list_media():
|
||||
return _list_media()
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
async def upload(file: UploadFile = File(...)):
|
||||
name = file.filename or "file"
|
||||
ext = Path(name).suffix.lower()
|
||||
if ext not in ALLOWED_EXT:
|
||||
return JSONResponse({"ok": False, "error": "不支持的文件类型"}, status_code=400)
|
||||
settings.MEDIA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
stem = Path(name).stem
|
||||
new_name = f"{stem}_{time.strftime('%Y%m%d%H%M%S')}{ext}"
|
||||
save_path = settings.MEDIA_DIR / new_name
|
||||
with save_path.open("wb") as f:
|
||||
while True:
|
||||
chunk = await file.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
hub.publish_command("playlist_changed")
|
||||
return {"ok": True, "name": new_name}
|
||||
|
||||
|
||||
@router.post("/api/delete")
|
||||
async def delete_media(body: PathBody):
|
||||
path = body.path
|
||||
if not path.startswith(("http://", "https://")):
|
||||
target = settings.MEDIA_DIR / path
|
||||
try:
|
||||
if target.exists() and target.is_file():
|
||||
target.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
storage.delete_media(path)
|
||||
hub.publish_command("playlist_changed")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/api/media/add-url")
|
||||
async def add_url(body: UrlBody):
|
||||
url = body.url.strip()
|
||||
if not url:
|
||||
return {"ok": False}
|
||||
name = url.rsplit("/", 1)[-1].split("?", 1)[0] or url
|
||||
t = _media_type(url)
|
||||
is_new = storage.add_url_media(url, name, t)
|
||||
return {"ok": True, "duplicate": None if is_new else True}
|
||||
|
||||
|
||||
# ==================== 播放列表 ====================
|
||||
|
||||
def _playlist_files():
|
||||
files = []
|
||||
media_dir = Path(settings.MEDIA_DIR)
|
||||
for item in storage.get_playlist():
|
||||
path = item.get("path", "")
|
||||
if item.get("source") == "url":
|
||||
files.append({
|
||||
"name": item.get("name") or path,
|
||||
"relative_path": path, "type": _media_type(path), "url": path, "source": "url",
|
||||
})
|
||||
else:
|
||||
fp = media_dir / path
|
||||
if fp.exists():
|
||||
files.append({
|
||||
"name": fp.name, "relative_path": path,
|
||||
"type": _media_type(path), "url": f"/file/{path}", "source": "local",
|
||||
})
|
||||
s = storage.get_settings()
|
||||
return {"files": files, "volume": s["volume"], "play_mode": s["play_mode"], "image_duration": s["image_duration"]}
|
||||
|
||||
|
||||
@router.get("/api/playlist")
|
||||
async def get_playlist():
|
||||
return _playlist_files()
|
||||
|
||||
|
||||
@router.post("/api/playlist/add")
|
||||
async def add_playlist(body: PathBody):
|
||||
storage.add_to_playlist(body.path)
|
||||
hub.publish_command("playlist_changed")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/api/playlist/remove")
|
||||
async def remove_playlist(body: PathBody):
|
||||
storage.remove_from_playlist(body.path)
|
||||
hub.publish_command("playlist_changed")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/api/playlist/play")
|
||||
async def play_playlist_item(body: PathBody):
|
||||
"""指定媒体立即播放：加入播放列表（如未在）→ MQTT 广播 play_target → 大屏跳转播放"""
|
||||
path = body.path
|
||||
storage.add_to_playlist(path)
|
||||
hub.publish_command("playlist_changed")
|
||||
cmd = hub.publish_command("play_target", {"path": path})
|
||||
return {"ok": True, "cmd_id": cmd.get("cmd_id")}
|
||||
|
||||
|
||||
# ==================== 播放控制 / 状态 ====================
|
||||
|
||||
_CONTROL_ACTIONS = {"play", "pause", "next", "prev"}
|
||||
|
||||
|
||||
@router.post("/api/control")
|
||||
async def control(body: ActionBody):
|
||||
if body.action not in _CONTROL_ACTIONS:
|
||||
return JSONResponse({"ok": False, "error": "action 必须是 play/pause/next/prev"}, status_code=400)
|
||||
hub.publish_command(body.action)
|
||||
return {"ok": True, "action": body.action}
|
||||
|
||||
|
||||
@router.get("/api/state")
|
||||
async def get_state():
|
||||
return {"state": {"status": "unknown", "index": 0, "name": "", "media_type": ""}}
|
||||
|
||||
|
||||
@router.post("/api/state")
|
||||
async def update_state(body: StateBody):
|
||||
hub.publish_command("state_update", body.state.model_dump())
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@router.post("/api/display-command")
|
||||
async def display_command(body: ActionBody):
|
||||
if body.action == "minimize":
|
||||
hub.publish_command("minimize")
|
||||
return {"ok": True, "action": "minimize"}
|
||||
return JSONResponse({"ok": False, "error": "不支持的显示命令"}, status_code=400)
|
||||
|
||||
|
||||
# ==================== 数据（大屏全部数据来自后端） ====================
|
||||
|
||||
@router.get("/api/dashboard/snapshot")
|
||||
async def dashboard_snapshot():
|
||||
return sim_engine.snapshot()
|
||||
|
||||
|
||||
@router.get("/api/dashboard/overview")
|
||||
async def dashboard_overview():
|
||||
return sim_engine.snapshot()
|
||||
|
||||
|
||||
# ==================== 园区企业 ====================
|
||||
|
||||
_COMPANIES = [
|
||||
{"name": "云南派音人工智能科技", "zone": "加速区", "room": "A1", "color": "#4c8dff"},
|
||||
{"name": "米勒克尔蓝宝石珠宝", "zone": "加速区", "room": "A2", "color": "#4c8dff"},
|
||||
{"name": "中泰研学合作", "zone": "加速区", "room": "A3", "color": "#4c8dff"},
|
||||
{"name": "云南宸中低空经济", "zone": "加速区", "room": "A4", "color": "#4c8dff"},
|
||||
{"name": "昆明智海银高文化科技", "zone": "加速区", "room": "A5", "color": "#4c8dff"},
|
||||
{"name": "AI机器人大模型训练", "zone": "加速区", "room": "A6", "color": "#4c8dff"},
|
||||
{"name": "云南廷秀文旅康养", "zone": "加速区", "room": "A7", "color": "#4c8dff"},
|
||||
{"name": "瀚颖AI+教育信息咨询", "zone": "加速区", "room": "A8", "color": "#4c8dff"},
|
||||
{"name": "云南大学AI+创业平台", "zone": "加速区", "room": "A9", "color": "#4c8dff"},
|
||||
{"name": "仰光客厅", "zone": "国际区", "room": "I1", "color": "#22d3ee"},
|
||||
{"name": "云南上古绝学文化", "zone": "国际区", "room": "I2", "color": "#22d3ee"},
|
||||
{"name": "中越生物医疗", "zone": "国际区", "room": "I3", "color": "#22d3ee"},
|
||||
{"name": "酷享野农AI农业", "zone": "国际区", "room": "I4", "color": "#22d3ee"},
|
||||
{"name": "滇缅国际设计", "zone": "国际区", "room": "I5", "color": "#22d3ee"},
|
||||
{"name": "昆明舒诺生物科技", "zone": "国际区", "room": "I6", "color": "#22d3ee"},
|
||||
{"name": "达岸教育管理", "zone": "国际区", "room": "I7", "color": "#22d3ee"},
|
||||
{"name": "Facebook越南跨境电商", "zone": "成长区", "room": "G1", "color": "#34d399"},
|
||||
{"name": "研X同行者网络", "zone": "成长区", "room": "G2", "color": "#34d399"},
|
||||
{"name": "朵哈·玫瑰特色产业链", "zone": "成长区", "room": "G3", "color": "#34d399"},
|
||||
{"name": "昆明云韵体育", "zone": "成长区", "room": "G4", "color": "#34d399"},
|
||||
{"name": "南菌优培食用菌", "zone": "成长区", "room": "G5", "color": "#34d399"},
|
||||
{"name": "五华区丽裳文化", "zone": "成长区", "room": "G6", "color": "#34d399"},
|
||||
{"name": "云南星瑞航空", "zone": "成长区", "room": "G7", "color": "#34d399"},
|
||||
{"name": "综合直播私域平台", "zone": "成长区", "room": "G8", "color": "#34d399"},
|
||||
{"name": "园区管理办公室", "zone": "园区管理", "room": "M", "color": "#fbbf24"},
|
||||
]
|
||||
|
||||
|
||||
@router.get("/api/park/companies")
|
||||
async def park_companies():
|
||||
return {"companies": _COMPANIES}
|
||||
|
||||
|
||||
@router.get("/api/park/zones")
|
||||
async def park_zones():
|
||||
zones = ["加速区", "国际区", "成长区"]
|
||||
counts = {z: sum(1 for c in _COMPANIES if c["zone"] == z) for z in zones}
|
||||
colors = {"加速区": "#4c8dff", "国际区": "#22d3ee", "成长区": "#34d399"}
|
||||
return {"zones": [{"name": z, "count": counts[z], "color": colors[z]} for z in zones]}
|
||||
|
||||
|
||||
# ==================== AI 对话（通义千问 + 工具调用） ====================
|
||||
|
||||
# 预设问题（围绕昆明市大学生创业园 / OPC 园区性质设计，前端从后端读取）
|
||||
AI_GROUPED_QUESTIONS = [
|
||||
{
|
||||
"title": "入驻与流程",
|
||||
"questions": [
|
||||
"如何申请入驻园区？",
|
||||
"园区入驻的条件有哪些？",
|
||||
"入驻需要准备哪些材料？",
|
||||
"入驻流程是怎样的？",
|
||||
"入驻评审如何打分？",
|
||||
"入驻需要多长时间？",
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "政策与扶持",
|
||||
"questions": [
|
||||
"园区有哪些创业政策扶持？",
|
||||
"如何申请创业补贴？",
|
||||
"如何申请创业担保贷款？",
|
||||
"对高校毕业生有什么优惠？",
|
||||
"科技成果转化有哪些支持？",
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "OPC 概念",
|
||||
"questions": [
|
||||
"什么是 OPC？",
|
||||
"OPC 创业有哪些模式？",
|
||||
"OPC 适合哪些人？",
|
||||
"OPC 创业者从哪里开始？",
|
||||
"OPC 常用的人工智能工具有哪些？",
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "场地与服务",
|
||||
"questions": [
|
||||
"园区提供哪些免费办公空间？",
|
||||
"园区有哪些孵化服务？",
|
||||
"园区提供哪些创业辅导？",
|
||||
"园区有哪些基础配套？",
|
||||
"园区可以免费使用哪些资源？",
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "园区企业介绍",
|
||||
"questions": [
|
||||
"介绍一下园区入驻企业",
|
||||
"园区有哪些 AI 科技企业？",
|
||||
"园区有哪些跨境电商企业？",
|
||||
"园区有哪些生物医药企业？",
|
||||
"介绍一下云南派音人工智能科技",
|
||||
"介绍一下米勒克尔蓝宝石珠宝",
|
||||
"介绍一下研X同行者网络",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
AI_OPC_TOOLS = [
|
||||
"DeepSeek", "通义千问", "ChatGPT", "豆包",
|
||||
"Midjourney", "Stable Diffusion", "剪映", "Notion AI",
|
||||
"WPS AI", "GitHub Copilot",
|
||||
]
|
||||
|
||||
|
||||
@router.get("/api/ai/questions")
|
||||
async def ai_questions():
|
||||
"""预设问题列表（围绕园区性质设计，前端右侧面板展示）"""
|
||||
return {
|
||||
"groups": AI_GROUPED_QUESTIONS,
|
||||
"opcTools": AI_OPC_TOOLS,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/ai/chat")
|
||||
async def ai_chat(body: ChatBody):
|
||||
"""大模型对话：工具调用（切页/控制/卡片/通知）经 MQTT 广播到所有大屏
|
||||
返回 {reply, tools, model}；tools 供请求端本地同步执行"""
|
||||
messages = [m.model_dump() for m in body.messages]
|
||||
result = llm_run_chat(messages, body.screen_id or "")
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/api/ai/asr")
|
||||
async def ai_asr(file: UploadFile = File(...), format: str = "m4a"):
|
||||
"""语音识别：上传录音 → 阿里云 paraformer 转写为文本"""
|
||||
data = await file.read()
|
||||
if not data:
|
||||
return JSONResponse({"ok": False, "error": "空音频"}, status_code=400)
|
||||
try:
|
||||
text = asr_transcribe(data, fmt=format)
|
||||
return {"ok": True, "text": text}
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("ASR 转写失败: %s", e)
|
||||
return JSONResponse({"ok": False, "error": str(e)}, status_code=502)
|
||||
|
||||
|
||||
# ==================== 展示控制（管理端 → MQTT） ====================
|
||||
# 全部 MQTT 前端控制命令白名单（与 docs/mqtt-commands.md 保持一致）
|
||||
_VALID_DISPLAY_ACTIONS = {
|
||||
# 页面
|
||||
"navigate", "navigate_rel",
|
||||
# 全局视觉
|
||||
"vision_set",
|
||||
# AI 助手页
|
||||
"ai_input", "ai_preset", "ai_company", "ai_zone",
|
||||
# 语音对话页
|
||||
"voice_start", "voice_stop", "voice_refresh",
|
||||
# 媒体
|
||||
"play", "pause", "next", "prev", "set_mode", "play_target",
|
||||
# 双屏
|
||||
"dual_screen",
|
||||
# 全局
|
||||
"alert", "show_card", "minimize", "settings_changed", "playlist_changed",
|
||||
}
|
||||
|
||||
|
||||
class RegisterBody(BaseModel):
|
||||
device_id: str = "" # 唯一设备标识（MQTT client_id）
|
||||
role: str = "" # main | secondary
|
||||
parent_id: str = "" # 副屏从属的主屏 device_id；主屏为空
|
||||
|
||||
|
||||
@router.post("/api/display/register")
|
||||
async def display_register(body: RegisterBody):
|
||||
"""设备注册：前端上报唯一 device_id + 角色 + 从属主屏，后端维护设备表供精准控制。"""
|
||||
hub.register_device(body.device_id, body.role, body.parent_id)
|
||||
return {"ok": True, "device_id": body.device_id, "role": body.role, "parent_id": body.parent_id}
|
||||
|
||||
|
||||
@router.post("/api/display/command")
|
||||
async def display_command_publish(body: DisplayCommandBody):
|
||||
"""管理端/任意客户端通过 REST 发控制指令 → 后端转 MQTT 广播给所有大屏"""
|
||||
if body.action not in _VALID_DISPLAY_ACTIONS:
|
||||
return JSONResponse({"ok": False, "error": f"不支持的指令: {body.action}"}, status_code=400)
|
||||
# 按屏幕终端路由：screen_id 精确到某客户端；screen_role 按主/副定位；皆空 → 全局广播
|
||||
cmd = hub.publish_command(body.action, body.params, body.screen_id or None, body.screen_role or None)
|
||||
return {"ok": cmd.get("published", False), "cmd_id": cmd["cmd_id"], "action": body.action,
|
||||
"screen_id": body.screen_id or "", "screen_role": body.screen_role or "both",
|
||||
"mqtt_connected": hub.connected}
|
||||
|
||||
|
||||
@router.get("/api/display/state")
|
||||
async def display_state():
|
||||
return hub.status()
|
||||
|
||||
|
||||
# ==================== SSE 兼容通道（MQTT 不可用时前端回退） ====================
|
||||
|
||||
@router.get("/api/events")
|
||||
async def sse_events(request: Request):
|
||||
q = bus.subscribe()
|
||||
|
||||
async def gen():
|
||||
try:
|
||||
while True:
|
||||
if await request.is_disconnected():
|
||||
break
|
||||
try:
|
||||
data = await asyncio.wait_for(q.get(), timeout=15)
|
||||
yield f"data: {data}\n\n"
|
||||
except asyncio.TimeoutError:
|
||||
yield ": keepalive\n\n"
|
||||
finally:
|
||||
bus.unsubscribe(q)
|
||||
|
||||
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||
|
||||
|
||||
# ==================== 视觉识别事件上报（前端摄像头实时识别 → 后端日志/MQTT） ====================
|
||||
class VisionEventBody(BaseModel):
|
||||
event: str # camera_on|camera_off|detecting|facing|triggered|silent|error|model_error
|
||||
faces: int = 0
|
||||
dwell_ms: int = 0
|
||||
detail: str = ""
|
||||
|
||||
@router.post("/api/vision/event")
|
||||
async def vision_event(body: VisionEventBody):
|
||||
"""前端摄像头识别状态/触发上报；后端记录详细日志。
|
||||
主动问候已关闭，故不再对 triggered 广播 alert（避免右上角弹「已主动问候」误导提示）。"""
|
||||
log.info(
|
||||
"vision event=%s faces=%d dwell_ms=%d detail=%s",
|
||||
body.event, body.faces, body.dwell_ms, body.detail,
|
||||
)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ==================== YOLO 人脸检测（前端抽帧 → 后端推理） ====================
|
||||
class VisionFrameBody(BaseModel):
|
||||
image: str = "" # JPEG base64（不含 data: 前缀）
|
||||
conf: float = 0.0 # 可选：覆盖置信度阈值
|
||||
|
||||
@router.post("/api/vision/frame")
|
||||
def vision_frame(body: VisionFrameBody):
|
||||
"""接收前端抽帧 JPEG base64，后端 YOLO 推理返回人脸框（faces/boxes/latency_ms）。
|
||||
|
||||
普通 def 由 FastAPI 线程池执行（推理约 100-200ms），不阻塞事件循环。
|
||||
"""
|
||||
img_b64 = (body.image or "").strip()
|
||||
if not img_b64:
|
||||
log.warning("vision frame: 缺少 image")
|
||||
return {"ok": False, "error": "missing image"}
|
||||
from .vision_yolo import predict_base64
|
||||
return predict_base64(img_b64)
|
||||
|
||||
|
||||
class VisionLlmBody(BaseModel):
|
||||
image: str = "" # JPEG base64
|
||||
prompt: str = "" # 可选，自定义识别指令
|
||||
|
||||
@router.post("/api/vision/llm")
|
||||
def vision_llm(body: VisionLlmBody):
|
||||
"""对话开场画面识别：多模态 LLM（qwen3-vl-flash）识别人数/性别，返回 {ok, people, males, females, desc}。"""
|
||||
img = (body.image or "").strip()
|
||||
if not img:
|
||||
return {"ok": False, "error": "missing image"}
|
||||
try:
|
||||
from .vision_llm import analyze_scene
|
||||
result = analyze_scene(img, body.prompt)
|
||||
log.info("api: /api/vision/llm -> people=%s males=%s females=%s desc=%s",
|
||||
result.get("people"), result.get("males"), result.get("females"),
|
||||
(result.get("desc") or "")[:60])
|
||||
return result
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("vision llm 失败: %s", e)
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
|
||||
# ==================== 智能体工具调用 + 知识库 ====================
|
||||
class ToolsExecBody(BaseModel):
|
||||
name: str = ""
|
||||
args: dict = {}
|
||||
screen_id: str = "" # 发起语音对话的屏幕设备 id（工具结果只发到该屏）
|
||||
|
||||
@router.post("/api/tools/exec")
|
||||
def tools_exec(body: ToolsExecBody):
|
||||
"""执行智能体工具（LLM function calling）：name + args → 结果字符串"""
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
return {"ok": False, "error": "missing tool name"}
|
||||
from .tools import exec_tool
|
||||
result = exec_tool(name, body.args or {}, body.screen_id or "")
|
||||
log.info("api: /api/tools/exec name=%s args=%s screen=%s result=%s", name,
|
||||
json.dumps(body.args or {}, ensure_ascii=False)[:300],
|
||||
body.screen_id or "all", str(result)[:400])
|
||||
return {"ok": True, "result": result}
|
||||
|
||||
@router.get("/api/kb/brief")
|
||||
def kb_brief(max_chars: int = 1200):
|
||||
"""园区知识摘要（前端拼进对话 instructions）"""
|
||||
try:
|
||||
from .rag import brief
|
||||
result = brief(max_chars=max_chars)
|
||||
log.info("api: /api/kb/brief max_chars=%d -> %s", max_chars, str(result)[:300])
|
||||
return {"ok": True, "brief": result}
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("kb brief 失败: %s", e)
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
@router.get("/api/kb/retrieve")
|
||||
def kb_retrieve(q: str = "", top_k: int = 3):
|
||||
"""知识库检索（query → 相关段落）"""
|
||||
if not q.strip():
|
||||
return {"ok": False, "error": "missing q"}
|
||||
try:
|
||||
from .rag import retrieve
|
||||
chunks = retrieve(q, top_k=top_k)
|
||||
log.info("api: /api/kb/retrieve q=%s top_k=%d -> %d 段", q[:100], top_k, len(chunks))
|
||||
return {"ok": True, "chunks": chunks}
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("kb retrieve 失败: %s", e)
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
@router.get("/api/s2s/instructions")
|
||||
def s2s_instructions():
|
||||
"""语音助手基础提示词（服务端统一组装：配置提示词 + 园区知识库）"""
|
||||
try:
|
||||
from .rag import build_instructions
|
||||
instructions = build_instructions()
|
||||
log.info("api: /api/s2s/instructions -> 系统提示词（%d 字）=%s",
|
||||
len(instructions), instructions[:600])
|
||||
return {"ok": True, "instructions": instructions}
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("s2s instructions 构建失败: %s", e)
|
||||
return {"ok": False, "error": str(e)}
|
||||
@@ -0,0 +1,143 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""s2s 实时语音栈（VAD 本地 + 云 ASR/LLM/TTS）桥接
|
||||
|
||||
在 DPM 进程内以后台线程运行 s2s 的 RealtimeServer（默认 :8765 /v1/realtime）。
|
||||
- 复用 live-avatar 云化栈（speech-to-speech v0.2.11 + cloud-migration 补丁）。
|
||||
- 依赖已并入 backend/pyproject.toml；本模块负责按 DPM Settings 组装并启动。
|
||||
- 前端语音对话页通过 WS 直连 ws://<host>:<S2S_PORT>/v1/realtime。
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("dpm.s2s")
|
||||
|
||||
|
||||
def _build_pool(stop_event: threading.Event):
|
||||
"""按 DPM 配置构建 s2s 管线池（含 VAD/silero 加载 + LLM warmup，耗时约 30~60s）。"""
|
||||
import sys
|
||||
|
||||
from speech_to_speech.api.openai_realtime.websocket_router import create_app
|
||||
from speech_to_speech.s2s_pipeline import (
|
||||
_build_realtime_pipeline_unit,
|
||||
parse_arguments,
|
||||
prepare_all_args,
|
||||
)
|
||||
|
||||
# 与 start_backend.sh 等价的参数（api_key 留空 → handler 从 env DASHSCOPE_API_KEY 读取）
|
||||
sys.argv = [
|
||||
"dpm-s2s",
|
||||
"--mode", "realtime",
|
||||
"--stt", "dashscope-asr",
|
||||
"--llm_backend", "chat-completions",
|
||||
"--tts", "qwen3-cloud",
|
||||
"--model_name", settings.S2S_LLM_MODEL,
|
||||
"--responses_api_base_url", settings.LLM_BASE_URL,
|
||||
"--responses_api_api_key", settings.DASHSCOPE_API_KEY,
|
||||
"--dashscope_asr_model", settings.S2S_STT_MODEL,
|
||||
"--qwen3_cloud_model", settings.S2S_TTS_MODEL,
|
||||
"--qwen3_cloud_voice", settings.S2S_TTS_VOICE,
|
||||
"--enable_live_transcription", "false",
|
||||
"--thresh", "0.6",
|
||||
"--min_silence_ms", "500",
|
||||
"--min_speech_ms", "500",
|
||||
"--log_level", settings.S2S_LOG_LEVEL,
|
||||
]
|
||||
args = parse_arguments()
|
||||
prepare_all_args(
|
||||
args.module_kwargs,
|
||||
args.whisper_stt_handler_kwargs,
|
||||
args.paraformer_stt_handler_kwargs,
|
||||
args.faster_whisper_stt_handler_kwargs,
|
||||
args.mlx_audio_whisper_stt_handler_kwargs,
|
||||
args.parakeet_tdt_stt_handler_kwargs,
|
||||
args.dashscope_asr_stt_handler_kwargs,
|
||||
args.language_model_handler_kwargs,
|
||||
args.responses_api_language_model_handler_kwargs,
|
||||
args.chat_tts_handler_kwargs,
|
||||
args.facebook_mms_tts_handler_kwargs,
|
||||
args.pocket_tts_handler_kwargs,
|
||||
args.kokoro_tts_handler_kwargs,
|
||||
args.qwen3_tts_handler_kwargs,
|
||||
args.qwen3_cloud_tts_handler_kwargs,
|
||||
)
|
||||
|
||||
pool = [
|
||||
_build_realtime_pipeline_unit(
|
||||
index=i,
|
||||
stop_event=stop_event,
|
||||
module_kwargs=args.module_kwargs,
|
||||
vad_handler_kwargs=args.vad_handler_kwargs,
|
||||
whisper_stt_handler_kwargs=args.whisper_stt_handler_kwargs,
|
||||
faster_whisper_stt_handler_kwargs=args.faster_whisper_stt_handler_kwargs,
|
||||
paraformer_stt_handler_kwargs=args.paraformer_stt_handler_kwargs,
|
||||
mlx_audio_whisper_stt_handler_kwargs=args.mlx_audio_whisper_stt_handler_kwargs,
|
||||
parakeet_tdt_stt_handler_kwargs=args.parakeet_tdt_stt_handler_kwargs,
|
||||
dashscope_asr_stt_handler_kwargs=args.dashscope_asr_stt_handler_kwargs,
|
||||
language_model_handler_kwargs=args.language_model_handler_kwargs,
|
||||
responses_api_language_model_handler_kwargs=args.responses_api_language_model_handler_kwargs,
|
||||
chat_tts_handler_kwargs=args.chat_tts_handler_kwargs,
|
||||
facebook_mms_tts_handler_kwargs=args.facebook_mms_tts_handler_kwargs,
|
||||
pocket_tts_handler_kwargs=args.pocket_tts_handler_kwargs,
|
||||
kokoro_tts_handler_kwargs=args.kokoro_tts_handler_kwargs,
|
||||
qwen3_tts_handler_kwargs=args.qwen3_tts_handler_kwargs,
|
||||
qwen3_cloud_tts_handler_kwargs=args.qwen3_cloud_tts_handler_kwargs,
|
||||
)
|
||||
for i in range(settings.S2S_NUM_PIPELINES)
|
||||
]
|
||||
|
||||
app = create_app(pool=pool, stop_event=stop_event)
|
||||
return app, pool
|
||||
|
||||
|
||||
def start_s2s_backend(stop_event: threading.Event) -> None:
|
||||
"""后台线程构建管线并启动 RealtimeServer（阻塞调用，放线程里跑）。"""
|
||||
|
||||
def _run() -> None:
|
||||
from speech_to_speech.api.openai_realtime.server import RealtimeServer
|
||||
|
||||
# LLM warmup / 云 ASR/TTS 连接在弱网下会瞬断：整个池构建重试几次
|
||||
last_exc: Exception | None = None
|
||||
log.info(
|
||||
"s2s: 启动配置 STT=%s LLM=%s TTS=%s(%s) pipelines=%d ws://%s:%d/v1/realtime retries=%d",
|
||||
settings.S2S_STT_MODEL, settings.S2S_LLM_MODEL, settings.S2S_TTS_MODEL,
|
||||
settings.S2S_TTS_VOICE, settings.S2S_NUM_PIPELINES,
|
||||
settings.S2S_HOST, settings.S2S_PORT, settings.S2S_BUILD_RETRIES,
|
||||
)
|
||||
for attempt in range(settings.S2S_BUILD_RETRIES):
|
||||
try:
|
||||
log.info(
|
||||
"s2s: 构建语音管线池（%d 路，尝试 %d/%d，首次约 30~60s）…",
|
||||
settings.S2S_NUM_PIPELINES,
|
||||
attempt + 1,
|
||||
settings.S2S_BUILD_RETRIES,
|
||||
)
|
||||
app, pool = _build_pool(stop_event)
|
||||
server = RealtimeServer(
|
||||
stop_event=stop_event,
|
||||
pool=pool,
|
||||
host=settings.S2S_HOST,
|
||||
port=settings.S2S_PORT,
|
||||
)
|
||||
# 关键：与 s2s_pipeline.main() 的 realtime 分支一致——用 ThreadManager
|
||||
# 启动 RealtimeServer + 全部 handler 线程（VAD/STT/LLM/TTS）。
|
||||
# 只跑 server 不启动 handler 线程会导致管线不处理任何音频。
|
||||
from speech_to_speech.utils.thread_manager import ThreadManager
|
||||
|
||||
all_handlers = [server] + [h for unit in pool for h in unit.handlers]
|
||||
thread_manager = ThreadManager(all_handlers)
|
||||
thread_manager.start()
|
||||
log.info("s2s: 实时语音服务 ws://%s:%d/v1/realtime", settings.S2S_HOST, settings.S2S_PORT)
|
||||
thread_manager.wait()
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
last_exc = e
|
||||
log.warning("s2s: 构建尝试 %d/%d 失败：%s", attempt + 1, settings.S2S_BUILD_RETRIES, e)
|
||||
if attempt + 1 < settings.S2S_BUILD_RETRIES:
|
||||
time.sleep(settings.S2S_BUILD_RETRY_DELAY)
|
||||
log.error("s2s: 启动失败（DPM_S2S_ENABLED=0 可关闭）：%s", last_exc)
|
||||
|
||||
threading.Thread(target=_run, daemon=True).start()
|
||||
@@ -0,0 +1,262 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""园区数据引擎（后端）—— 真实数据 + 测算演示
|
||||
数据来源（用户指定口径）：
|
||||
· 《昆明市大学生创业园运行情况统计表（2026年7月）.xls》（官方月报）
|
||||
· 《昆明市创业园宣传册.pdf》
|
||||
测算口径（用户确认）：
|
||||
· TOKEN：真实业务锚点月均 200 亿 tokens → 日均 ≈ 6.67 亿（200亿/30），
|
||||
实时速率 ≈ 7,700 t/s（日均/86400s），近30日累计 = 200 亿
|
||||
· 工具调用：按平均每次调用消耗约 2 万 tokens 折算（今日 ≈ 3.3 万次）
|
||||
其余统计指标为真实静态值；TOKEN/工具仅做轻微演示波动
|
||||
"""
|
||||
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
|
||||
# ---------- 真实锚点（仅来自 xls 统计表 + 附件1 省级材料 + 宣传册） ----------
|
||||
REAL = {
|
||||
"as_of": "2026-07",
|
||||
"token_monthly_yi": 200, # 月均消耗 200 亿 tokens（AI 平台业务锚点，非政府统计口径）
|
||||
"token_daily_wan": 66666.7, # 日均 6.67 亿 tokens（200亿/30 ≈ 66666.7 万）
|
||||
"token_rate": 7700, # 实时 t/s（6.67e8 / 86400 ≈ 7,716）
|
||||
"tools_per_call_tokens": 150000, # 平均每次工具调用消耗 tokens（真实测算：12.5M输入 ÷ 81步 ≈ 15.4万）
|
||||
"occupied": 38, # 实际入驻企业数（官方口径：《2026年7月运行情况统计表》38 家）
|
||||
"capacity": 49, # 园区可容纳企业个数（统计表）
|
||||
"invested": 560, # 开园以来累计投入运营资金（万元，统计表）
|
||||
"jobs": 213, # 带动就业人数（统计表，当年累计口径）
|
||||
"revenue_total": 252.15, # 生产经营总额（万元，统计表，当年累计口径）
|
||||
"revenue_tax": 1.03, # 上缴税利总额（万元，统计表，当年累计口径）
|
||||
"incubated_total": 181, # 累计孵化企业（附件1 省级绩效累计口径，截至2026年5月）
|
||||
"in_incubation": 31, # 当前实有在孵创业实体（附件1）
|
||||
"graduated": 149, # 累计成功孵化出园企业（统计表，开园以来累计）
|
||||
"patents": 171, # 入驻团队发明专利（附件1 累计口径）
|
||||
"jobs_total": 1529, # 累计带动就业（附件1 省级绩效累计口径）
|
||||
"revenue_total_cum": 21911.14, # 入驻团队累计经营收入（万元，附件1 累计口径 ≈ 2.19 亿元）
|
||||
"revenue_tax_cum": 426.2, # 入驻团队累计税收（万元，附件1 累计口径）
|
||||
"mentors": 50, # 创业指导专家团队（附件1）
|
||||
"success_rate": 95.91, # 近3年入孵实体孵化成功率（%）（附件1）
|
||||
"area": 3000, # 园区建筑面积（㎡，统计表+宣传册）
|
||||
"founded": 2009, # 成立年份（宣传册）
|
||||
"province_level": 2012, # 获评省级创业园年份（宣传册）
|
||||
"address": "昆明市民航路229号（人力资源中心1-2楼）",
|
||||
"phone": "18087174891",
|
||||
"email": "714066050@qq.com",
|
||||
}
|
||||
|
||||
# 入驻企业名录（《入驻企业信息表》39 家全称，与在园数统一口径）
|
||||
COMPANIES = [
|
||||
"云南派音人工智能科技有限公司", "米勒克尔蓝宝石珠宝产业化（项目）", "中泰研学合作（项目）",
|
||||
"云南宸中低空经济有限公司", "昆明智海银高文化科技有限公司", "人工智能机器人大模型训练（项目）",
|
||||
"云南廷秀文旅康养服务（项目）", "瀚颖AI+教育信息咨询（项目）", "云南大学AI+联合创业服务平台（项目）",
|
||||
"仰光客厅（项目）", "云南上古绝学文化发展有限公司", "千楣之路（项目）",
|
||||
"中越生物医疗（项目）", "中越文旅交流（项目）", "绿态环保科技项目",
|
||||
"联翼航空技术低空经济应用国际化拓展（项目）", "酷享野农AI农业（项目）", "滇缅国际设计（项目）",
|
||||
"昆明舒诺生物科技有限责任公司", "达岸教育管理（云南）有限公司", "昆明滇油石化有限责任公司",
|
||||
"花仙子园艺肥料（云南）有限公司", "Facebook越南市场普洱茶与建水紫陶壶跨境电商（项目）",
|
||||
"研X——同行者网络（项目）", "鬼才明AI创意工作室（项目）", "朵哈·玫瑰特色产业链（项目）",
|
||||
"昆明云韵体育有限责任公司", "昆明舒护安养老服务有限责任公司",
|
||||
"南菌优培—食用菌专用生物制剂创新与高原特色野生菌高值化全链开发（项目）",
|
||||
"五华区丽裳文化艺术工作室", "云南星瑞航空（项目）", "综合性云南直播私域服务平台",
|
||||
"昆明屿澈电子商务有限公司", "中外青少年研学（项目）", "星杭商务信息咨询昆明有限责任公司",
|
||||
"蓝智科技（项目）", "云品出滇·纸享万家（项目）",
|
||||
"云迹轻旅—昆明本土一站式轻量化文旅服务工作室（项目）", "启元人工智能科技（项目）",
|
||||
]
|
||||
|
||||
# 宣传册四大孵化区域
|
||||
ZONES = [
|
||||
{"name": "OPC创业空间", "desc": "AI 技术支撑 · 数字经济/AI应用/软件开发/文创设计/电商直播等轻资产领域", "period": "拎包入驻 · 最长2年"},
|
||||
{"name": "创业苗圃区", "desc": "创意阶段 · 未注册企业团队 · 开放式共享工位", "period": "最长2年"},
|
||||
{"name": "孵化加速区", "desc": "已工商注册初创企业 · 独立办公空间", "period": "基础2年 · 可延1年（≤3年）"},
|
||||
{"name": "国际创客区", "desc": "归国留学生 · 外籍留学生 · 国际化配套服务", "period": "参照加速区"},
|
||||
]
|
||||
|
||||
# 园区企业行业分布（按《入驻企业信息表》39 家企业简介分类统计）
|
||||
INDUSTRY_MIX = [
|
||||
{"name": "人工智能", "value": 7},
|
||||
{"name": "跨境电商", "value": 5},
|
||||
{"name": "教育培训", "value": 4},
|
||||
{"name": "文旅康养", "value": 4},
|
||||
{"name": "文化创意", "value": 4},
|
||||
{"name": "生物医药", "value": 4},
|
||||
{"name": "低空经济", "value": 3},
|
||||
{"name": "现代农业", "value": 3},
|
||||
{"name": "绿色环保", "value": 2},
|
||||
{"name": "企业服务", "value": 2},
|
||||
{"name": "新材料", "value": 1},
|
||||
]
|
||||
|
||||
# 园区实时动态（真实事件：园区官方公示 + 宣传册活动 + 互联网公开信息，最新在前）
|
||||
FEED = [
|
||||
{"id": 24, "icon": "dna", "text": "园内企业「昆明舒诺生物科技」完成GEO服务体系建设（生成式引擎优化，聚焦宠物营养品牌数字化）", "time": "2026-08"},
|
||||
{"id": 1, "icon": "activity", "text": "2026年7月运行情况统计表填报完成：累计投入运营资金560万元、带动就业213人", "time": "2026-07-31"},
|
||||
{"id": 2, "icon": "radio", "text": "2026年第三期入驻创业企业（项目）评审结果在昆明市人社局官网公示", "time": "2026-07-02"},
|
||||
{"id": 3, "icon": "graduation", "text": "云南师范大学就业创业实践基地落地园区 · 云南旅游职业学院开展就业创业交流", "time": "2026-06-03"},
|
||||
{"id": 23, "icon": "radio", "text": "「PineSound」完成 ICP 备案（滇ICP备2026008517号）与公安备案，注册地址昆明市民航路301-307号", "time": "2026"},
|
||||
{"id": 4, "icon": "rocket", "text": "\"滇水逐梦·雨林创航\"昆明·西双版纳创业路演PK赛成功举办", "time": "2026-05-20"},
|
||||
{"id": 5, "icon": "flag", "text": "昆明市青年企业家协会创新创业基地在园区揭牌", "time": "2026-05-13"},
|
||||
{"id": 6, "icon": "users", "text": "\"聚力赋能·共创未来\"高校创新创业交流暨项目路演活动圆满举行", "time": "2026-05-08"},
|
||||
{"id": 7, "icon": "trophy", "text": "\"创赢未来\"2026创业大赛昆明选拔赛暨马兰花创业培训讲师大赛举行（13个项目参赛）", "time": "2026-04-22"},
|
||||
{"id": 8, "icon": "broadcast", "text": "2026年第二期创业企业（项目）入驻招募公告发布（官网 www.kmyc.gov.cn）", "time": "2026-04-15"},
|
||||
{"id": 21, "icon": "sparkles", "text": "「PineSound」AI驱动音频管理平台上线 pinesound.cn：智能配乐与音效生成，集成100W+音效库、50W+配乐库，全球版权授权（官网）", "time": "2026"},
|
||||
{"id": 22, "icon": "medal", "text": "PineSound发布企业标准 Q/YNPY 001-2026《数字内容创作 配乐通用分类标准》", "time": "2026"},
|
||||
{"id": 9, "icon": "award", "text": "2026年第一期入驻评审结果在昆明市人社局官网公示（23个优质项目正式入驻）", "time": "2026-03-27"},
|
||||
{"id": 10, "icon": "users", "text": "2026年第一期招募集中评审完成：38个申请，23个优质项目正式入驻", "time": "2026-03-24"},
|
||||
{"id": 19, "icon": "sparkles", "text": "云南派音人工智能科技专注多模态音频技术研发：自研Pine系列模型覆盖音频识别、向量嵌入、音效生成、配乐创作", "time": "2026-04"},
|
||||
{"id": 20, "icon": "bolt", "text": "「PineSound」2026年4月注册成立，获北京投资支持并吸纳就业4人，入驻加速区（入驻企业信息表）", "time": "2026-04"},
|
||||
{"id": 11, "icon": "cpu", "text": "云南首个人工智能OPC创新人才基地落地昆明，填补省内个体AI创业培育空白（公开报道）", "time": "2026-03-23"},
|
||||
{"id": 12, "icon": "medal", "text": "第九届\"春城创业荟\"创业创新大赛圆满闭幕，获奖项目名单公布", "time": "2025-09-30"},
|
||||
{"id": 13, "icon": "trophy", "text": "\"春城创业荟\"初赛落幕，147个项目晋级复赛", "time": "2025-04-28"},
|
||||
{"id": 14, "icon": "building", "text": "园内企业「花仙子园艺肥料（云南）」成立，注册资本118万元（公开报道）", "time": "2025-04-28"},
|
||||
{"id": 15, "icon": "radio", "text": "2025年入园项目评审结果公示（昆明市人社局官网）", "time": "2025-04-22"},
|
||||
{"id": 16, "icon": "atom", "text": "研X平台入选\"创客中国\"项目库（一站式研究生学术成长平台）", "time": "2025-01"},
|
||||
{"id": 17, "icon": "leaf", "text": "行业动态：园内「朵哈·玫瑰」原料产地昆明八街食用玫瑰行情上涨，玫瑰经济走强", "time": "2025-01"},
|
||||
{"id": 18, "icon": "heart", "text": "行业动态：「千楣之路」主营檀娜卡——泰国天然护肤品牌Thanaka正式进入中国市场", "time": "2025-01"},
|
||||
]
|
||||
|
||||
|
||||
def _day_secs():
|
||||
"""当日零点起已过秒数（时钟驱动，保证数值单向持续增长、重启连续）"""
|
||||
now = time.time()
|
||||
lt = time.localtime(now)
|
||||
day_start = time.mktime((lt.tm_year, lt.tm_mon, lt.tm_mday, 0, 0, 0, 0, 0, -1))
|
||||
return max(0.0, now - day_start)
|
||||
|
||||
|
||||
def _today_values(prev_noise=None):
|
||||
"""按真实速率推算当日累计值（单向增长 + 工具调用波动）：
|
||||
平台实时速率 = 单会话 157 tok/s × 49 路并发 ≈ 7,700 t/s
|
||||
今日 TOKEN（万）= 秒数 × 7,700 / 10000 = 秒数 × 0.77
|
||||
今日工具调用 = 今日 TOKEN / 15万 tokens 每次（真实基线）
|
||||
+ 随机游走噪声（±10，波浪式波动，不单调）
|
||||
"""
|
||||
secs = _day_secs()
|
||||
today_wan = round(secs * 0.77) # 万 tokens
|
||||
base_tools = secs * 0.77 * 10000 / REAL["tools_per_call_tokens"] # 真实基线（浮点）
|
||||
if prev_noise is None:
|
||||
noise = 0.0
|
||||
else:
|
||||
noise = max(-10.0, min(10.0, prev_noise + (random.random() - 0.5) * 6.0))
|
||||
today_tools = max(0, round(base_tools + noise)) # 次
|
||||
return today_wan, today_tools, noise
|
||||
|
||||
|
||||
def _make_series(base, noise, n=30):
|
||||
"""生成围绕日均锚点的演示曲线（单位：万 tokens），尾点=今日实时值"""
|
||||
out = []
|
||||
for _ in range(n - 1):
|
||||
v = base + (random.random() - 0.5) * noise
|
||||
out.append(round(max(1, v), 1))
|
||||
out.append(round(base, 1))
|
||||
return out
|
||||
|
||||
|
||||
def _live_values(prev=None):
|
||||
"""TOKEN 面板实时运行指标（围绕真实基准动态波动，会话持续推进）"""
|
||||
r = random.random
|
||||
steps = (prev or {}).get("steps", 81)
|
||||
steps += 1 if r() < 0.3 else 0
|
||||
return {
|
||||
"firstToken": round(1.3 + (r() - 0.5) * 0.2, 2), # 首 token 平均 1.3s 基准 ±0.1
|
||||
"throughput": round(157 + (r() - 0.5) * 14), # 吞吐 157 tok/s 基准 ±7
|
||||
"cacheHit": round(98 + (r() - 0.5) * 1.2, 1), # 缓存命中 98% 基准 ±0.6
|
||||
"inputM": round((prev or {}).get("inputM", 12.5) + 0.004 + r() * 0.008, 2), # 会话输入持续推进
|
||||
"outputM": round((prev or {}).get("outputM", 2.0) + 0.001 + r() * 0.002, 2), # 会话输出持续推进
|
||||
"rounds": max(2, steps // 40), # 每 40 步一轮
|
||||
"steps": steps, # 步数持续增长
|
||||
}
|
||||
|
||||
|
||||
def init_snapshot():
|
||||
today_wan, today_tools, noise = _today_values()
|
||||
return {
|
||||
"as_of": REAL["as_of"],
|
||||
"_toolsNoise": noise,
|
||||
"live": _live_values(),
|
||||
"token": {
|
||||
"today": today_wan, # 万 tokens（当日累计，实时增长）
|
||||
"total": REAL["token_monthly_yi"] * 10000 + today_wan, # 万 tokens（近30日 = 200亿 + 今日增量）
|
||||
"rate": REAL["token_rate"], # t/s（157 tok/s × 49 并发 ≈ 7,700）
|
||||
"series": _make_series(REAL["token_daily_wan"], REAL["token_daily_wan"] * 0.16),
|
||||
},
|
||||
"tools": {
|
||||
"today": today_tools, # 次（随 TOKEN 实时联动）
|
||||
"total": round(REAL["token_monthly_yi"] * 100000000 / REAL["tools_per_call_tokens"]) + today_tools, # 次
|
||||
"success": 98.5, # 成功率（演示）
|
||||
"nodes": 12, # AI 服务节点（演示）
|
||||
},
|
||||
"projects": {
|
||||
"inPark": REAL["occupied"], # 实际入驻 38 家（2026年7月统计表官方口径）
|
||||
"capacity": REAL["capacity"], # 可容纳 49 个
|
||||
"invested": REAL["invested"], # 累计投入 560 万元
|
||||
},
|
||||
"jobs": {"total": REAL["jobs"]}, # 带动就业 213 人（当年累计口径）
|
||||
"revenue": {"total": REAL["revenue_total"], "tax": REAL["revenue_tax"]}, # 万元（当年累计口径）
|
||||
"cumulative": { # 省级绩效累计口径（附件1，截至2026年5月）
|
||||
"incubated": REAL["incubated_total"], # 累计孵化企业 181 家
|
||||
"inIncubation": REAL["in_incubation"], # 当前实有在孵 31 家
|
||||
"graduated": REAL["graduated"], # 累计成功出园 149 家
|
||||
"patents": REAL["patents"], # 发明专利 171 项
|
||||
"jobs": REAL["jobs_total"], # 累计带动就业 1,529 人
|
||||
"revenue": REAL["revenue_total_cum"], # 累计经营收入 21,911.14 万元
|
||||
"tax": REAL["revenue_tax_cum"], # 累计税收 426.2 万元
|
||||
"mentors": REAL["mentors"], # 创业指导专家团队 50 人
|
||||
"successRate": REAL["success_rate"], # 近3年孵化成功率 95.91%
|
||||
},
|
||||
"park": {
|
||||
"area": REAL["area"],
|
||||
"founded": REAL["founded"],
|
||||
"provinceLevel": REAL["province_level"],
|
||||
"address": REAL["address"],
|
||||
"phone": REAL["phone"],
|
||||
"email": REAL["email"],
|
||||
},
|
||||
"zones": ZONES,
|
||||
"companies": COMPANIES,
|
||||
"industryMix": INDUSTRY_MIX,
|
||||
"feed": FEED,
|
||||
}
|
||||
|
||||
|
||||
def next_snapshot(s):
|
||||
"""时钟驱动：按真实速率重算当日累计（TOKEN 单向增长，工具调用波浪波动），其余真实静态"""
|
||||
today_wan, today_tools, noise = _today_values(s.get("_toolsNoise"))
|
||||
token = {
|
||||
"today": today_wan,
|
||||
"total": REAL["token_monthly_yi"] * 10000 + today_wan,
|
||||
"rate": REAL["token_rate"],
|
||||
"series": s["token"]["series"][:-1] + [today_wan],
|
||||
}
|
||||
tools = {
|
||||
"today": today_tools,
|
||||
"total": round(REAL["token_monthly_yi"] * 100000000 / REAL["tools_per_call_tokens"]) + today_tools,
|
||||
"success": 98.5,
|
||||
"nodes": 12,
|
||||
}
|
||||
return {
|
||||
**s,
|
||||
"_toolsNoise": noise,
|
||||
"live": _live_values(s.get("live")),
|
||||
"token": token,
|
||||
"tools": tools,
|
||||
}
|
||||
|
||||
|
||||
class SimEngine:
|
||||
"""带锁的快照引擎：单例供 API 与 MQTT tick 共用"""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.RLock()
|
||||
self._snap = init_snapshot()
|
||||
|
||||
def snapshot(self):
|
||||
with self._lock:
|
||||
return dict(self._snap)
|
||||
|
||||
def tick(self):
|
||||
with self._lock:
|
||||
self._snap = next_snapshot(self._snap)
|
||||
return dict(self._snap)
|
||||
|
||||
|
||||
sim_engine = SimEngine()
|
||||
@@ -0,0 +1,115 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""JSON 持久化：设置 / 播放列表 / URL 媒体（对齐原 Rust storage 语义）"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .config import settings
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"username": "admin",
|
||||
"password": "123456",
|
||||
"volume": 80,
|
||||
"sfx_volume": 60,
|
||||
"auto_play": True,
|
||||
"play_mode": "sequential",
|
||||
"image_duration": 5,
|
||||
"fullscreen": True,
|
||||
"autostart": False,
|
||||
}
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, data_file: Path):
|
||||
self.data_file = data_file
|
||||
self._lock = threading.RLock()
|
||||
self._data = self._load()
|
||||
|
||||
# ---------- 底层 ----------
|
||||
def _load(self):
|
||||
if self.data_file.exists():
|
||||
try:
|
||||
data = json.loads(self.data_file.read_text("utf-8"))
|
||||
# 规范化：URL 来源自动标记（兼容旧数据 source=None 的 URL 条目）
|
||||
for it in data.setdefault("playlist", []):
|
||||
p = it.get("path", "")
|
||||
if p.startswith(("http://", "https://")) and not it.get("source"):
|
||||
it["source"] = "url"
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
return {"settings": dict(DEFAULT_SETTINGS), "playlist": [], "url_media": []}
|
||||
|
||||
def _save(self):
|
||||
self.data_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self.data_file.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(self._data, ensure_ascii=False, indent=2), "utf-8")
|
||||
tmp.replace(self.data_file)
|
||||
|
||||
# ---------- 设置 ----------
|
||||
def get_settings(self):
|
||||
with self._lock:
|
||||
s = dict(DEFAULT_SETTINGS)
|
||||
s.update(self._data.get("settings", {}))
|
||||
return s
|
||||
|
||||
def update_settings(self, **kw):
|
||||
with self._lock:
|
||||
s = dict(DEFAULT_SETTINGS)
|
||||
s.update(self._data.get("settings", {}))
|
||||
for k, v in kw.items():
|
||||
if v is not None and k in s:
|
||||
s[k] = v
|
||||
self._data["settings"] = s
|
||||
self._save()
|
||||
|
||||
# ---------- 播放列表 ----------
|
||||
def get_playlist(self):
|
||||
with self._lock:
|
||||
return list(self._data.get("playlist", []))
|
||||
|
||||
def add_to_playlist(self, path):
|
||||
with self._lock:
|
||||
items = self._data.setdefault("playlist", [])
|
||||
if not any(it.get("path") == path for it in items):
|
||||
# URL 来源自动标记，避免被当作本地文件过滤
|
||||
source = "url" if path.startswith(("http://", "https://")) else None
|
||||
items.append({"path": path, "source": source, "name": None})
|
||||
self._save()
|
||||
|
||||
def remove_from_playlist(self, path):
|
||||
with self._lock:
|
||||
self._data.setdefault("playlist", [])
|
||||
self._data["playlist"] = [it for it in self._data["playlist"] if it.get("path") != path]
|
||||
self._save()
|
||||
|
||||
# ---------- URL 媒体 ----------
|
||||
def get_url_media(self):
|
||||
with self._lock:
|
||||
return list(self._data.get("url_media", []))
|
||||
|
||||
def add_url_media(self, url, name, media_type):
|
||||
"""返回 True 表示新增，False 表示已存在（重复）"""
|
||||
with self._lock:
|
||||
items = self._data.setdefault("url_media", [])
|
||||
if any(it.get("url") == url for it in items):
|
||||
return False
|
||||
items.append({
|
||||
"url": url, "name": name, "type": media_type,
|
||||
"added_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
})
|
||||
self._save()
|
||||
return True
|
||||
|
||||
def delete_media(self, path):
|
||||
with self._lock:
|
||||
self._data.setdefault("url_media", [])
|
||||
self._data["url_media"] = [it for it in self._data["url_media"] if it.get("url") != path]
|
||||
self._data.setdefault("playlist", [])
|
||||
self._data["playlist"] = [it for it in self._data["playlist"] if it.get("path") != path]
|
||||
self._save()
|
||||
|
||||
|
||||
storage = Storage(settings.DATA_FILE)
|
||||
@@ -0,0 +1,145 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""智能体工具注册表 —— LLM function calling 的后端执行器
|
||||
|
||||
- TOOLS：工具名 → 描述 + JSON Schema 参数（给 LLM 声明用）
|
||||
- exec_tool(name, args)：执行工具，返回给 LLM 的字符串结果（JSON）
|
||||
- 供 POST /api/tools/exec 调用；前端 s2s 客户端收到 toolcall 事件后转发到此执行
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
log = logging.getLogger("dpm.tools")
|
||||
|
||||
|
||||
def _trunc(s, n=800):
|
||||
"""日志用截断：超长内容只留前 n 字并标注总长。"""
|
||||
s = str(s)
|
||||
return s if len(s) <= n else s[:n] + f"...（共{len(s)}字）"
|
||||
|
||||
|
||||
# ── 工具实现 ────────────────────────────────────────────────────────────
|
||||
|
||||
def _get_park_overview(args):
|
||||
"""园区实时概览（官方统计口径 + 省级绩效累计口径 + AI 平台运行指标）"""
|
||||
from .sim_engine import sim_engine
|
||||
d = sim_engine.snapshot()
|
||||
return json.dumps({
|
||||
"数据口径": "官方统计（《2026年7月运行情况统计表》）；累计口径（附件1省级材料，截至2026年5月）；AI平台指标：TOKEN/工具调用",
|
||||
"实际入驻企业数(家)": d["projects"]["inPark"],
|
||||
"园区可容纳企业数(个)": d["projects"]["capacity"],
|
||||
"开园以来累计投入运营资金(万元)": d["projects"]["invested"],
|
||||
"带动就业人数(人,当年累计)": d["jobs"]["total"],
|
||||
"生产经营总额(万元,当年累计)": d["revenue"]["total"],
|
||||
"上缴税利总额(万元,当年累计)": d["revenue"]["tax"],
|
||||
"累计孵化企业(家,省级绩效累计)": d["cumulative"]["incubated"],
|
||||
"当前实有在孵实体(家,省级绩效累计)": d["cumulative"]["inIncubation"],
|
||||
"累计成功孵化出园企业(家,开园以来累计)": d["cumulative"]["graduated"],
|
||||
"入驻团队发明专利(项,累计)": d["cumulative"]["patents"],
|
||||
"累计带动就业(人,省级绩效累计)": d["cumulative"]["jobs"],
|
||||
"入驻团队累计经营收入(万元)": d["cumulative"]["revenue"],
|
||||
"入驻团队累计税收(万元)": d["cumulative"]["tax"],
|
||||
"创业指导专家团队(人)": d["cumulative"]["mentors"],
|
||||
"近3年入孵实体孵化成功率(%)": d["cumulative"]["successRate"],
|
||||
"园区建筑面积(㎡)": d["park"]["area"],
|
||||
"TOKEN月均消耗(亿)": 200,
|
||||
"TOKEN实时速率(t/s)": d["token"]["rate"],
|
||||
"今日TOKEN消耗(万)": d["token"]["today"],
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _query_companies(args):
|
||||
"""园区入驻企业名录查询（关键词过滤）"""
|
||||
from .sim_engine import COMPANY_NAMES
|
||||
q = (args.get("keyword") or "").strip()
|
||||
names = [n for n in COMPANY_NAMES if q in n] if q else list(COMPANY_NAMES)
|
||||
return json.dumps({
|
||||
"total": len(names),
|
||||
"keyword": q,
|
||||
"companies": names[:20],
|
||||
}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _control_display(args, screen_id=""):
|
||||
"""大屏显示控制：切页 / 弹通知 / 媒体播放暂停（经 MQTT 按发起屏幕下发）"""
|
||||
from .mqtt import hub
|
||||
action = (args.get("action") or "").strip()
|
||||
target = (args.get("target") or "").strip()
|
||||
if action not in ("switch_page", "alert", "media_play", "media_pause"):
|
||||
return json.dumps({"error": f"不支持的 action: {action}"}, ensure_ascii=False)
|
||||
params = {"target": target} if target else {}
|
||||
ok = hub.publish_command(action, params, screen_id=screen_id).get("published", False)
|
||||
return json.dumps({"ok": ok, "action": action, "target": target}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _get_time(args):
|
||||
"""当前日期时间"""
|
||||
return json.dumps({"datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}, ensure_ascii=False)
|
||||
|
||||
|
||||
# ── 注册表 ──────────────────────────────────────────────────────────────
|
||||
|
||||
TOOLS = {
|
||||
"get_park_overview": {
|
||||
"fn": _get_park_overview,
|
||||
"description": "获取园区实时运营概览：在园项目数、累计孵化企业、带动就业、营收（今日/累计）、设备在线率、在园人数、能耗等。回答园区数据类问题时使用。",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
"query_companies": {
|
||||
"fn": _query_companies,
|
||||
"description": "查询园区入驻企业名录，支持按企业名关键词过滤。回答'有哪些企业/某企业是否入驻'时使用。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"keyword": {"type": "string", "description": "企业名关键词，可为空字符串"}},
|
||||
},
|
||||
},
|
||||
"control_display": {
|
||||
"fn": _control_display,
|
||||
"description": "控制大屏显示：switch_page 切换页面、alert 弹通知、media_play/media_pause 控制媒体播放。用户要求控制大屏时使用。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "enum": ["switch_page", "alert", "media_play", "media_pause"]},
|
||||
"target": {"type": "string", "description": "目标：页面路径（如 /、/twin、/ai、/voice）或通知文本"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"get_time": {
|
||||
"fn": _get_time,
|
||||
"description": "获取当前日期时间。用户问'现在几点/今天几号'时使用。",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def tool_schemas():
|
||||
"""给 LLM 的 tools 定义（OpenAI function calling 格式）"""
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"name": name,
|
||||
"description": t["description"],
|
||||
"parameters": t["parameters"],
|
||||
}
|
||||
for name, t in TOOLS.items()
|
||||
]
|
||||
|
||||
|
||||
def exec_tool(name: str, args: dict, screen_id: str = ""):
|
||||
"""执行工具 → 返回给 LLM 的字符串结果（JSON）。
|
||||
控制类工具（control_display）按发起屏幕 screen_id 只下发到该屏。"""
|
||||
t = TOOLS.get(name)
|
||||
if not t:
|
||||
log.warning("tools: 未知工具 %s", name)
|
||||
return json.dumps({"error": f"未知工具: {name}"}, ensure_ascii=False)
|
||||
try:
|
||||
if name == "control_display":
|
||||
result = _control_display(args or {}, screen_id)
|
||||
else:
|
||||
result = t["fn"](args or {})
|
||||
log.info("tools: 执行工具 %s args=%s screen=%s", name, json.dumps(args, ensure_ascii=False), screen_id or "all")
|
||||
log.info("tools: 结果 -> %s", _trunc(result, 1000))
|
||||
return result
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("tools: %s 执行失败 %s", name, e, exc_info=True)
|
||||
return json.dumps({"error": f"{name} 执行失败: {e}"}, ensure_ascii=False)
|
||||
@@ -0,0 +1,83 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""对话开场画面识别 —— 用多模态 LLM（qwen3-vl-flash）识别实时画面中的人数、性别等，
|
||||
并把结果作为上下文注入对话，让 AI 了解当前在场观众。
|
||||
|
||||
与 vision_yolo（YOLO 数人脸/姿态）互补：这里用 LLM 做语义级理解（人数、性别构成、场景描述）。
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
import urllib.request
|
||||
|
||||
import certifi
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("dpm.vision")
|
||||
|
||||
_SSL_CTX = ssl.create_default_context(cafile=certifi.where())
|
||||
|
||||
_DEFAULT_PROMPT = (
|
||||
"请识别这张实时画面，只输出一个 JSON 对象（不要输出任何其他文字）："
|
||||
'{"people": 画面中人数(int), "males": 其中男性人数(int), "females": 其中女性人数(int), '
|
||||
'"desc": 一句话中文描述画面（含大致人数、性别构成、人物大致状态，如年龄/坐站/是否看屏幕）。'
|
||||
"若画面无人或不确定，则 people=0、males=0、females=0，desc='画面中暂时没有人'。"
|
||||
)
|
||||
|
||||
|
||||
def analyze_scene(jpeg_b64: str, prompt: str = "") -> dict:
|
||||
"""调用多模态 LLM 识别画面中人数/性别，返回 {ok, people, males, females, desc}。"""
|
||||
payload = {
|
||||
"model": settings.VISION_LLM_MODEL,
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt or _DEFAULT_PROMPT},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg_b64}"}},
|
||||
],
|
||||
}],
|
||||
}
|
||||
req = urllib.request.Request(
|
||||
f"{settings.LLM_BASE_URL}/chat/completions",
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {settings.DASHSCOPE_API_KEY}",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
content = data["choices"][0]["message"].get("content") or ""
|
||||
return _parse(content)
|
||||
|
||||
|
||||
def _parse(text: str) -> dict:
|
||||
"""从模型输出中抽取 JSON 对象并规范化为结果 dict。"""
|
||||
text = (text or "").strip()
|
||||
if text.startswith("```"):
|
||||
lines = [l for l in text.splitlines() if not l.strip().startswith("```")]
|
||||
text = "\n".join(lines).strip()
|
||||
obj: dict = {}
|
||||
i, j = text.find("{"), text.rfind("}")
|
||||
if i != -1 and j > i:
|
||||
try:
|
||||
obj = json.loads(text[i:j + 1])
|
||||
except Exception: # noqa: BLE001
|
||||
log.warning("vision llm: 返回 JSON 解析失败: %s", text[:120])
|
||||
people = _as_int(obj.get("people"))
|
||||
males = _as_int(obj.get("males"))
|
||||
females = _as_int(obj.get("females"))
|
||||
return {
|
||||
"ok": True,
|
||||
"people": people,
|
||||
"males": males,
|
||||
"females": females,
|
||||
"desc": str(obj.get("desc") or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def _as_int(v):
|
||||
try:
|
||||
return int(float(v))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
@@ -0,0 +1,204 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""YOLO 人脸检测 + 姿态估计（后端推理）
|
||||
|
||||
链路：前端摄像头抽帧 → JPEG base64 → POST /api/vision/frame → 本模块推理 → 返回人脸/姿态/举手特征。
|
||||
模型：
|
||||
- backend/models/yolov8n-face.pt （lindevs/yolov8-face，WIDERFACE 训练，torch 6.3MB）
|
||||
- backend/models/yolov8n-pose.pt （ultralytics 官方，COCO 17 关键点，torch 6.8MB）
|
||||
推理：ultralytics 加载（torch 原生，自动 letterbox + NMS），CPU 设备。
|
||||
|
||||
手势判定分工：
|
||||
- 后端只做"单帧几何特征"（举手 = 手腕高于肩），无状态
|
||||
- 前端做时序判定（举手持续 1.5s → 开始对话；举手后手腕摆动 → 挥手结束对话）
|
||||
"""
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
log = logging.getLogger("dpm.vision")
|
||||
|
||||
MODEL_DIR = Path(__file__).resolve().parent.parent / "models"
|
||||
FACE_MODEL_PATH = MODEL_DIR / "yolov8n-face.pt"
|
||||
POSE_MODEL_PATH = MODEL_DIR / "yolov8n-pose.pt"
|
||||
CONF_THRESHOLD = 0.25 # 远距离小脸可调低（如 0.15）
|
||||
IMGSZ = 640
|
||||
|
||||
# COCO 17 关键点索引
|
||||
KP_LEFT_SHOULDER = 5
|
||||
KP_RIGHT_SHOULDER = 6
|
||||
KP_LEFT_ELBOW = 7
|
||||
KP_RIGHT_ELBOW = 8
|
||||
KP_LEFT_WRIST = 9
|
||||
KP_RIGHT_WRIST = 10
|
||||
RAISE_LIFT = 0.06 # 举手：手腕高于对应肩 6% 画面高（降低阈值，更易稳定激活）
|
||||
FIST_ELBOW_LIFT = 0.05 # 举拳：手腕高于肘 5% 画面高（前臂上举）
|
||||
FIST_SHOULDER_GAP = 0.15 # 举拳：拳不高于肩 15% 画面高（收在胸前）
|
||||
FIST_CHEST_DIST = 0.25 # 举拳：拳与肩水平距离 < 25% 画面宽（贴近躯干）
|
||||
KP_CONF_MIN = 0.3 # 关键点置信度过滤
|
||||
|
||||
# ── 扩展手势几何阈值（基于 COCO 17 关键点） ──
|
||||
POINT_REACH_X = 0.30 # 指向：手腕水平伸出距肩 ≥30% 画面宽（手臂向前/侧伸）
|
||||
POINT_Y_RANGE = 0.28 # 指向：手腕与肩同高 ±28% 画面高（排除高举/下垂）
|
||||
HANDS_CLOSE_DIST = 0.18 # 双手合十/靠近：双腕欧氏距离 <18% 画面宽
|
||||
|
||||
_face_model = None
|
||||
_pose_model = None
|
||||
|
||||
|
||||
def get_face_model():
|
||||
global _face_model
|
||||
if _face_model is None:
|
||||
from ultralytics import YOLO
|
||||
log.info("vision: 加载 YOLO 人脸模型 %s（首次约 2-5s）", FACE_MODEL_PATH)
|
||||
_face_model = YOLO(str(FACE_MODEL_PATH), task="detect")
|
||||
log.info("vision: YOLO 人脸模型就绪")
|
||||
return _face_model
|
||||
|
||||
|
||||
def get_pose_model():
|
||||
global _pose_model
|
||||
if _pose_model is None:
|
||||
from ultralytics import YOLO
|
||||
log.info("vision: 加载 YOLO 姿态模型 %s（首次约 2-5s）", POSE_MODEL_PATH)
|
||||
_pose_model = YOLO(str(POSE_MODEL_PATH), task="pose")
|
||||
log.info("vision: YOLO 姿态模型就绪")
|
||||
return _pose_model
|
||||
|
||||
|
||||
def _parse_pose(results, W, H):
|
||||
"""解析姿态结果 → persons + raised(举手) + fists(举拳)
|
||||
+ both_up(双臂举起) + pointing(指向) + hands_close(双手合十)
|
||||
+ hands(每手几何摘要，供前端挥手/手势跟随时序判定)
|
||||
注意：ultralytics keypoints.data 为【原图像素坐标】，
|
||||
此处统一归一化为 [0,1]（x/W, y/H）后再判定阈值与输出
|
||||
"""
|
||||
persons = []
|
||||
raised = []
|
||||
fists = []
|
||||
pointing = []
|
||||
hands = []
|
||||
for r in results:
|
||||
if r.keypoints is None:
|
||||
continue
|
||||
kps = r.keypoints.data # [N,17,3] 像素坐标
|
||||
for i in range(kps.shape[0]):
|
||||
kp = kps[i]
|
||||
person = [
|
||||
[round(float(kp[j][0]), 3), round(float(kp[j][1]), 3), round(float(kp[j][2]), 3)]
|
||||
for j in range(17)
|
||||
]
|
||||
persons.append(person)
|
||||
# 每只手臂：举手（腕明显高于肩）或 举拳（前臂上举、拳收胸前），两者互斥
|
||||
arm = {}
|
||||
for side, wrist_i, elbow_i, shoulder_i in (
|
||||
("left", KP_LEFT_WRIST, KP_LEFT_ELBOW, KP_LEFT_SHOULDER),
|
||||
("right", KP_RIGHT_WRIST, KP_RIGHT_ELBOW, KP_RIGHT_SHOULDER),
|
||||
):
|
||||
w = kp[wrist_i]
|
||||
e = kp[elbow_i]
|
||||
s = kp[shoulder_i]
|
||||
if float(w[2]) < KP_CONF_MIN or float(s[2]) < KP_CONF_MIN or float(e[2]) < KP_CONF_MIN:
|
||||
continue
|
||||
# 像素 → 归一化 [0,1]
|
||||
wx, wy = float(w[0]) / W, float(w[1]) / H
|
||||
sx, sy = float(s[0]) / W, float(s[1]) / H
|
||||
ey = float(e[1]) / H
|
||||
is_raised = wy < sy - RAISE_LIFT
|
||||
is_fist = (
|
||||
wy < ey - FIST_ELBOW_LIFT
|
||||
and wy >= sy - FIST_SHOULDER_GAP
|
||||
and abs(wx - sx) < FIST_CHEST_DIST
|
||||
)
|
||||
if is_raised:
|
||||
raised.append({"side": side, "x": round(wx, 4), "y": round(wy, 4)})
|
||||
elif is_fist:
|
||||
fists.append({"side": side, "x": round(wx, 4), "y": round(wy, 4)})
|
||||
# 指向：手腕水平伸出距肩较远、且与肩同高区间（前伸/侧伸，排除高举与下垂）
|
||||
if (
|
||||
abs(wx - sx) > POINT_REACH_X
|
||||
and abs(wy - sy) < POINT_Y_RANGE
|
||||
):
|
||||
pointing.append({"side": side, "x": round(wx, 4), "y": round(wy, 4)})
|
||||
arm[side] = {
|
||||
"side": side,
|
||||
"wx": round(wx, 4), "wy": round(wy, 4),
|
||||
"sx": round(sx, 4), "sy": round(sy, 4),
|
||||
"raised": is_raised, "fist": is_fist,
|
||||
}
|
||||
if arm:
|
||||
hands.append(arm)
|
||||
# 双臂举起：左、右腕都高于各自肩
|
||||
both_up = False
|
||||
if len(hands) >= 1:
|
||||
h0 = hands[0]
|
||||
if "left" in h0 and "right" in h0:
|
||||
both_up = h0["left"]["raised"] and h0["right"]["raised"]
|
||||
# 双手合十/靠近：同一人的双腕欧氏距离 < 阈值
|
||||
hands_close = False
|
||||
if len(hands) >= 1:
|
||||
h0 = hands[0]
|
||||
if "left" in h0 and "right" in h0:
|
||||
dx = h0["left"]["wx"] - h0["right"]["wx"]
|
||||
dy = h0["left"]["wy"] - h0["right"]["wy"]
|
||||
hands_close = (dx * dx + dy * dy) ** 0.5 < HANDS_CLOSE_DIST
|
||||
return persons, raised, fists, both_up, pointing, hands_close, hands
|
||||
|
||||
|
||||
def predict_jpeg(jpeg_bytes: bytes):
|
||||
"""人脸检测 + 姿态估计 → {faces, boxes, pose, raised, fists, both_up, pointing, hands_close, hands, latency_ms}"""
|
||||
t0 = time.time()
|
||||
img = Image.open(io.BytesIO(jpeg_bytes)).convert("RGB")
|
||||
|
||||
# 人脸
|
||||
face_res = get_face_model().predict(img, conf=CONF_THRESHOLD, imgsz=IMGSZ, verbose=False, device="cpu")
|
||||
boxes = []
|
||||
for r in face_res:
|
||||
if r.boxes is None:
|
||||
continue
|
||||
for b in r.boxes:
|
||||
xyxy = [round(float(v), 1) for v in b.xyxy[0].tolist()]
|
||||
conf = round(float(b.conf[0]), 3)
|
||||
boxes.append({"box": xyxy, "conf": conf})
|
||||
|
||||
# 姿态
|
||||
W, H = img.size # 原图尺寸（关键点像素坐标 → 归一化基准）
|
||||
pose_res = get_pose_model().predict(img, conf=CONF_THRESHOLD, imgsz=IMGSZ, verbose=False, device="cpu")
|
||||
persons, raised, fists, both_up, pointing, hands_close, hands = _parse_pose(pose_res, W, H)
|
||||
|
||||
latency_ms = round((time.time() - t0) * 1000, 1)
|
||||
log.info(
|
||||
"vision: 人脸 %d 姿态 %d 举手 %d 举拳 %d 指向 %d 合十 %s · %.0fms",
|
||||
len(boxes), len(persons), len(raised), len(fists), len(pointing), hands_close, latency_ms,
|
||||
)
|
||||
return {
|
||||
"faces": len(boxes),
|
||||
"boxes": boxes,
|
||||
"pose": persons,
|
||||
"raised": raised,
|
||||
"fists": fists,
|
||||
"both_up": both_up,
|
||||
"pointing": pointing,
|
||||
"hands_close": hands_close,
|
||||
"hands": hands,
|
||||
"latency_ms": latency_ms,
|
||||
}
|
||||
|
||||
|
||||
def predict_base64(b64: str):
|
||||
"""入口：base64 JPEG → 检测结果 dict（含 ok 标记，失败时带 error）"""
|
||||
try:
|
||||
jpeg = base64.b64decode(b64)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("vision: base64 解码失败 %s", e)
|
||||
return {"ok": False, "error": f"bad base64: {e}"}
|
||||
try:
|
||||
data = predict_jpeg(jpeg)
|
||||
data["ok"] = True
|
||||
return data
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("vision: 推理失败 %s", e, exc_info=True)
|
||||
return {"ok": False, "error": str(e)}
|
||||
Reference in New Issue
Block a user