feat: 大屏后端（FastAPI+MQTT+sim_engine）
- REST :10085 / MQTT 控制 / SSE 回退 / AI+语音 - 数据源规划：接核心服务端 + 算力引擎真实数据
This commit is contained in:
@@ -0,0 +1,143 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""s2s 实时语音栈（VAD 本地 + 云 ASR/LLM/TTS）桥接
|
||||
|
||||
在 DPM 进程内以后台线程运行 s2s 的 RealtimeServer（默认 :8765 /v1/realtime）。
|
||||
- 复用 live-avatar 云化栈（speech-to-speech v0.2.11 + cloud-migration 补丁）。
|
||||
- 依赖已并入 backend/pyproject.toml；本模块负责按 DPM Settings 组装并启动。
|
||||
- 前端语音对话页通过 WS 直连 ws://<host>:<S2S_PORT>/v1/realtime。
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("dpm.s2s")
|
||||
|
||||
|
||||
def _build_pool(stop_event: threading.Event):
|
||||
"""按 DPM 配置构建 s2s 管线池（含 VAD/silero 加载 + LLM warmup，耗时约 30~60s）。"""
|
||||
import sys
|
||||
|
||||
from speech_to_speech.api.openai_realtime.websocket_router import create_app
|
||||
from speech_to_speech.s2s_pipeline import (
|
||||
_build_realtime_pipeline_unit,
|
||||
parse_arguments,
|
||||
prepare_all_args,
|
||||
)
|
||||
|
||||
# 与 start_backend.sh 等价的参数（api_key 留空 → handler 从 env DASHSCOPE_API_KEY 读取）
|
||||
sys.argv = [
|
||||
"dpm-s2s",
|
||||
"--mode", "realtime",
|
||||
"--stt", "dashscope-asr",
|
||||
"--llm_backend", "chat-completions",
|
||||
"--tts", "qwen3-cloud",
|
||||
"--model_name", settings.S2S_LLM_MODEL,
|
||||
"--responses_api_base_url", settings.LLM_BASE_URL,
|
||||
"--responses_api_api_key", settings.DASHSCOPE_API_KEY,
|
||||
"--dashscope_asr_model", settings.S2S_STT_MODEL,
|
||||
"--qwen3_cloud_model", settings.S2S_TTS_MODEL,
|
||||
"--qwen3_cloud_voice", settings.S2S_TTS_VOICE,
|
||||
"--enable_live_transcription", "false",
|
||||
"--thresh", "0.6",
|
||||
"--min_silence_ms", "500",
|
||||
"--min_speech_ms", "500",
|
||||
"--log_level", settings.S2S_LOG_LEVEL,
|
||||
]
|
||||
args = parse_arguments()
|
||||
prepare_all_args(
|
||||
args.module_kwargs,
|
||||
args.whisper_stt_handler_kwargs,
|
||||
args.paraformer_stt_handler_kwargs,
|
||||
args.faster_whisper_stt_handler_kwargs,
|
||||
args.mlx_audio_whisper_stt_handler_kwargs,
|
||||
args.parakeet_tdt_stt_handler_kwargs,
|
||||
args.dashscope_asr_stt_handler_kwargs,
|
||||
args.language_model_handler_kwargs,
|
||||
args.responses_api_language_model_handler_kwargs,
|
||||
args.chat_tts_handler_kwargs,
|
||||
args.facebook_mms_tts_handler_kwargs,
|
||||
args.pocket_tts_handler_kwargs,
|
||||
args.kokoro_tts_handler_kwargs,
|
||||
args.qwen3_tts_handler_kwargs,
|
||||
args.qwen3_cloud_tts_handler_kwargs,
|
||||
)
|
||||
|
||||
pool = [
|
||||
_build_realtime_pipeline_unit(
|
||||
index=i,
|
||||
stop_event=stop_event,
|
||||
module_kwargs=args.module_kwargs,
|
||||
vad_handler_kwargs=args.vad_handler_kwargs,
|
||||
whisper_stt_handler_kwargs=args.whisper_stt_handler_kwargs,
|
||||
faster_whisper_stt_handler_kwargs=args.faster_whisper_stt_handler_kwargs,
|
||||
paraformer_stt_handler_kwargs=args.paraformer_stt_handler_kwargs,
|
||||
mlx_audio_whisper_stt_handler_kwargs=args.mlx_audio_whisper_stt_handler_kwargs,
|
||||
parakeet_tdt_stt_handler_kwargs=args.parakeet_tdt_stt_handler_kwargs,
|
||||
dashscope_asr_stt_handler_kwargs=args.dashscope_asr_stt_handler_kwargs,
|
||||
language_model_handler_kwargs=args.language_model_handler_kwargs,
|
||||
responses_api_language_model_handler_kwargs=args.responses_api_language_model_handler_kwargs,
|
||||
chat_tts_handler_kwargs=args.chat_tts_handler_kwargs,
|
||||
facebook_mms_tts_handler_kwargs=args.facebook_mms_tts_handler_kwargs,
|
||||
pocket_tts_handler_kwargs=args.pocket_tts_handler_kwargs,
|
||||
kokoro_tts_handler_kwargs=args.kokoro_tts_handler_kwargs,
|
||||
qwen3_tts_handler_kwargs=args.qwen3_tts_handler_kwargs,
|
||||
qwen3_cloud_tts_handler_kwargs=args.qwen3_cloud_tts_handler_kwargs,
|
||||
)
|
||||
for i in range(settings.S2S_NUM_PIPELINES)
|
||||
]
|
||||
|
||||
app = create_app(pool=pool, stop_event=stop_event)
|
||||
return app, pool
|
||||
|
||||
|
||||
def start_s2s_backend(stop_event: threading.Event) -> None:
|
||||
"""后台线程构建管线并启动 RealtimeServer（阻塞调用，放线程里跑）。"""
|
||||
|
||||
def _run() -> None:
|
||||
from speech_to_speech.api.openai_realtime.server import RealtimeServer
|
||||
|
||||
# LLM warmup / 云 ASR/TTS 连接在弱网下会瞬断：整个池构建重试几次
|
||||
last_exc: Exception | None = None
|
||||
log.info(
|
||||
"s2s: 启动配置 STT=%s LLM=%s TTS=%s(%s) pipelines=%d ws://%s:%d/v1/realtime retries=%d",
|
||||
settings.S2S_STT_MODEL, settings.S2S_LLM_MODEL, settings.S2S_TTS_MODEL,
|
||||
settings.S2S_TTS_VOICE, settings.S2S_NUM_PIPELINES,
|
||||
settings.S2S_HOST, settings.S2S_PORT, settings.S2S_BUILD_RETRIES,
|
||||
)
|
||||
for attempt in range(settings.S2S_BUILD_RETRIES):
|
||||
try:
|
||||
log.info(
|
||||
"s2s: 构建语音管线池（%d 路，尝试 %d/%d，首次约 30~60s）…",
|
||||
settings.S2S_NUM_PIPELINES,
|
||||
attempt + 1,
|
||||
settings.S2S_BUILD_RETRIES,
|
||||
)
|
||||
app, pool = _build_pool(stop_event)
|
||||
server = RealtimeServer(
|
||||
stop_event=stop_event,
|
||||
pool=pool,
|
||||
host=settings.S2S_HOST,
|
||||
port=settings.S2S_PORT,
|
||||
)
|
||||
# 关键：与 s2s_pipeline.main() 的 realtime 分支一致——用 ThreadManager
|
||||
# 启动 RealtimeServer + 全部 handler 线程（VAD/STT/LLM/TTS）。
|
||||
# 只跑 server 不启动 handler 线程会导致管线不处理任何音频。
|
||||
from speech_to_speech.utils.thread_manager import ThreadManager
|
||||
|
||||
all_handlers = [server] + [h for unit in pool for h in unit.handlers]
|
||||
thread_manager = ThreadManager(all_handlers)
|
||||
thread_manager.start()
|
||||
log.info("s2s: 实时语音服务 ws://%s:%d/v1/realtime", settings.S2S_HOST, settings.S2S_PORT)
|
||||
thread_manager.wait()
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
last_exc = e
|
||||
log.warning("s2s: 构建尝试 %d/%d 失败：%s", attempt + 1, settings.S2S_BUILD_RETRIES, e)
|
||||
if attempt + 1 < settings.S2S_BUILD_RETRIES:
|
||||
time.sleep(settings.S2S_BUILD_RETRY_DELAY)
|
||||
log.error("s2s: 启动失败（DPM_S2S_ENABLED=0 可关闭）：%s", last_exc)
|
||||
|
||||
threading.Thread(target=_run, daemon=True).start()
|
||||
Reference in New Issue
Block a user