Pine
064b06ecf0
迁入 app/park：llm(对话)、rag(双路向量知识库)、tools/ai_tools(智能体工具)、 asr(语音识别)、s2s_bridge(实时语音桥)、vision_yolo/vision_llm(人脸/多模态)、 knowledge/*.md、vendor/s2s-cloud(s2s 云化栈)；routers 补 /api/ai|kb|asr|vision| s2s|tools 端点。智能体提示词/企业名录改读 park_config 主数据源。pyproject 加重依赖 (dashscope/numpy/openai/torch/transformers/ultralytics/websockets/soundfile/scipy/ nltk/jinja2)。TestClient 冒烟：ai/chat(无 Key 走本地规则)、kb、s2s、tools、display 均 200。
84 lines
3.0 KiB
Python
84 lines
3.0 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""对话开场画面识别 —— 用多模态 LLM（qwen3-vl-flash）识别实时画面中的人数、性别等，
|
||
并把结果作为上下文注入对话，让 AI 了解当前在场观众。
|
||
|
||
与 vision_yolo（YOLO 数人脸/姿态）互补：这里用 LLM 做语义级理解（人数、性别构成、场景描述）。
|
||
"""
|
||
import json
|
||
import logging
|
||
import ssl
|
||
import urllib.request
|
||
|
||
import certifi
|
||
|
||
from .config import settings
|
||
|
||
log = logging.getLogger("dpm.vision")
|
||
|
||
_SSL_CTX = ssl.create_default_context(cafile=certifi.where())
|
||
|
||
_DEFAULT_PROMPT = (
|
||
"请识别这张实时画面，只输出一个 JSON 对象（不要输出任何其他文字）："
|
||
'{"people": 画面中人数(int), "males": 其中男性人数(int), "females": 其中女性人数(int), '
|
||
'"desc": 一句话中文描述画面（含大致人数、性别构成、人物大致状态，如年龄/坐站/是否看屏幕）。'
|
||
"若画面无人或不确定，则 people=0、males=0、females=0，desc='画面中暂时没有人'。"
|
||
)
|
||
|
||
|
||
def analyze_scene(jpeg_b64: str, prompt: str = "") -> dict:
|
||
"""调用多模态 LLM 识别画面中人数/性别，返回 {ok, people, males, females, desc}。"""
|
||
payload = {
|
||
"model": settings.VISION_LLM_MODEL,
|
||
"messages": [{
|
||
"role": "user",
|
||
"content": [
|
||
{"type": "text", "text": prompt or _DEFAULT_PROMPT},
|
||
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{jpeg_b64}"}},
|
||
],
|
||
}],
|
||
}
|
||
req = urllib.request.Request(
|
||
f"{settings.LLM_BASE_URL}/chat/completions",
|
||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {settings.DASHSCOPE_API_KEY}",
|
||
},
|
||
)
|
||
with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as resp:
|
||
data = json.loads(resp.read().decode("utf-8"))
|
||
content = data["choices"][0]["message"].get("content") or ""
|
||
return _parse(content)
|
||
|
||
|
||
def _parse(text: str) -> dict:
|
||
"""从模型输出中抽取 JSON 对象并规范化为结果 dict。"""
|
||
text = (text or "").strip()
|
||
if text.startswith("```"):
|
||
lines = [l for l in text.splitlines() if not l.strip().startswith("```")]
|
||
text = "\n".join(lines).strip()
|
||
obj: dict = {}
|
||
i, j = text.find("{"), text.rfind("}")
|
||
if i != -1 and j > i:
|
||
try:
|
||
obj = json.loads(text[i:j + 1])
|
||
except Exception: # noqa: BLE001
|
||
log.warning("vision llm: 返回 JSON 解析失败: %s", text[:120])
|
||
people = _as_int(obj.get("people"))
|
||
males = _as_int(obj.get("males"))
|
||
females = _as_int(obj.get("females"))
|
||
return {
|
||
"ok": True,
|
||
"people": people,
|
||
"males": males,
|
||
"females": females,
|
||
"desc": str(obj.get("desc") or "").strip(),
|
||
}
|
||
|
||
|
||
def _as_int(v):
|
||
try:
|
||
return int(float(v))
|
||
except (TypeError, ValueError):
|
||
return 0
|