Pine
4d693e895d
useMqttControl 扩展命令分发（opc/display/command）： - vision_set：全局人物识别开/关（dpm:vision-control → 语音页 visionEnabled） - ai_input / ai_preset / ai_company / ai_zone：AI 助手页操作（dpm:ai-action） - voice_start / voice_stop / voice_refresh：语音对话页操作（dpm:voice-control） - 页面状态即时上报 opc/frontend/state（路由变化即发，心跳外补充） 页面监听： - AiChatPanel：输入并发送 / 选择预设问题发送 - ParkSidePanel：切换分区（加速区/国际区/成长区）/ 切换展示企业（next/prev） - VoiceAssistant：启动/关闭对话 / 刷新页面；vision 全局开关 - MediaScreen：play/pause/next/prev 已有覆盖 切页（navigate/navigate_rel）、心跳上报沿用既有通道。
329 lines
13 KiB
React
329 lines
13 KiB
React
import { useState, useRef, useEffect, useCallback } from 'react';
|
||
import Icon from './Icons';
|
||
import { API_BASE } from '../config';
|
||
import { getMqttStatus } from '../utils/mqtt';
|
||
|
||
/* =========================================================
|
||
AI 智能助手对话面板 —— 文字 / 语音双模式
|
||
· 文字：后端 /api/ai/chat（阿里云通义千问 + 工具调用）
|
||
· 语音：本地录音 → 后端 /api/ai/asr（阿里云 paraformer）→ 进入对话流
|
||
后端不可用时自动回退本地模拟（离线兜底）
|
||
========================================================= */
|
||
|
||
const QUICK_QUESTIONS = [
|
||
'如何申请入驻园区？',
|
||
'园区有哪些政策扶持？',
|
||
'展示入驻企业卡片',
|
||
'AI 赋能服务有哪些？',
|
||
'切换到数字孪生页面',
|
||
];
|
||
|
||
/* 离线兜底回复（后端不可用时） */
|
||
const FALLBACK_REPLIES = [
|
||
'园区提供「空间 + 孵化 + 融资 + 政策 + 资源 + AI 赋能 + 综合服务」七位一体服务，在园项目 158 家，累计带动就业 2,186 人，累计营收 2.08 亿元。',
|
||
'入驻流程：提交申请 → 项目评审 → 签订协议 → 安排场地并对接导师服务。在园项目 158 家，累计孵化 208 余家。',
|
||
'园区政策覆盖场地租金减免、创业补贴、税费优惠与高企认定申报辅导（在园高企 32 家）。',
|
||
'园区按成长阶段分三个区域：加速区（A1–A12）、国际区（I1–I7）、成长区（G1–G8），共 39 家入驻企业。',
|
||
];
|
||
const pick = (arr) => arr[Math.floor(Math.random() * arr.length)];
|
||
|
||
/* 本地兜底识别题库（ASR 不可用时） */
|
||
const VOICE_SAMPLES = QUICK_QUESTIONS;
|
||
|
||
/* ---------- 消息 ---------- */
|
||
let msgId = 0;
|
||
const nextId = () => ++msgId;
|
||
|
||
/** 执行后端返回的工具调用。
|
||
后端已通过 MQTT 广播（发布成功时所有大屏都会收到，本地跳过避免重复）；
|
||
仅当 MQTT 发布失败 / 后端不可达时本地执行，保证当前屏幕也能响应 */
|
||
function executeLocalTools(tools, mqttPublished) {
|
||
if (!Array.isArray(tools) || tools.length === 0) return;
|
||
if (mqttPublished !== false) return;
|
||
for (const tool of tools) {
|
||
const type = tool.type;
|
||
const params = tool.params || {};
|
||
if (type === 'navigate' || type === 'control' || type === 'alert' || type === 'show_card') {
|
||
window.dispatchEvent(new CustomEvent('dpm:mqtt-local', { detail: { action: type, params } }));
|
||
}
|
||
}
|
||
}
|
||
|
||
export default function AiChatPanel() {
|
||
const [msgs, setMsgs] = useState(() => [{ id: nextId(), role: 'ai', text: '您好，我是 OPC 园区 AI 智能助手（阿里云通义千问）。\n\n支持文字与语音提问，可以为您解答入驻流程、政策扶持、场地服务、AI 赋能等问题，还可以通过语音或文字让我切换页面、展示企业卡片。' }]);
|
||
const [input, setInput] = useState('');
|
||
const [typing, setTyping] = useState(false);
|
||
const [mode, setMode] = useState('text');
|
||
const [voiceState, setVoiceState] = useState('idle'); // idle | recording | transcribing
|
||
const scrollRef = useRef(null);
|
||
const timerRef = useRef(null);
|
||
const recorderRef = useRef(null);
|
||
const chunksRef = useRef([]);
|
||
const voiceStateRef = useRef(voiceState);
|
||
|
||
useEffect(() => { voiceStateRef.current = voiceState; }, [voiceState]);
|
||
|
||
useEffect(() => {
|
||
const el = scrollRef.current;
|
||
if (el) el.scrollTop = el.scrollHeight;
|
||
}, [msgs, typing, mode, voiceState]);
|
||
|
||
useEffect(() => {
|
||
return () => {
|
||
clearTimeout(timerRef.current);
|
||
if (recorderRef.current && recorderRef.current.state !== 'inactive') {
|
||
try { recorderRef.current.stop(); } catch { /* ignore */ }
|
||
}
|
||
};
|
||
}, []);
|
||
|
||
/** 发送消息 → 后端 LLM */
|
||
const send = useCallback(async (raw, viaVoice = false) => {
|
||
const text = String(raw || '').trim();
|
||
if (!text || typing) return;
|
||
setInput('');
|
||
setMsgs((prev) => [...prev, { id: nextId(), role: 'me', text, voice: viaVoice }]);
|
||
setTyping(true);
|
||
|
||
let reply = null;
|
||
let tools = [];
|
||
try {
|
||
const res = await fetch(`${API_BASE}/api/ai/chat`, {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/json' },
|
||
body: JSON.stringify({ messages: [{ role: 'user', content: text }] }),
|
||
});
|
||
if (res.ok) {
|
||
const data = await res.json();
|
||
reply = data.reply;
|
||
tools = data.tools || [];
|
||
}
|
||
} catch { /* 后端不可用 */ }
|
||
|
||
if (!reply) reply = pick(FALLBACK_REPLIES);
|
||
setTyping(false);
|
||
setMsgs((prev) => [...prev, { id: nextId(), role: 'ai', text: reply }]);
|
||
executeLocalTools(tools);
|
||
}, [typing]);
|
||
|
||
const clear = useCallback(() => {
|
||
clearTimeout(timerRef.current);
|
||
setTyping(false);
|
||
setMsgs([{ id: nextId(), role: 'ai', text: '您好，我是 OPC 园区 AI 智能助手。有什么可以帮您？' }]);
|
||
}, []);
|
||
|
||
/* ================= 语音对话（录音 → 后端 ASR） ================= */
|
||
|
||
const pickMimeType = () => {
|
||
const types = ['audio/mp4', 'audio/webm;codecs=opus', 'audio/ogg;codecs=opus', 'audio/webm'];
|
||
for (const t of types) {
|
||
if (window.MediaRecorder && MediaRecorder.isTypeSupported(t)) return t;
|
||
}
|
||
return '';
|
||
};
|
||
|
||
const startRecording = useCallback(() => {
|
||
const mime = pickMimeType();
|
||
if (!window.MediaRecorder || !navigator.mediaDevices?.getUserMedia) {
|
||
// 无录音能力：直接模拟
|
||
setVoiceState('transcribing');
|
||
timerRef.current = setTimeout(() => {
|
||
setVoiceState('idle');
|
||
send(pick(VOICE_SAMPLES), true);
|
||
}, 1200);
|
||
return;
|
||
}
|
||
navigator.mediaDevices.getUserMedia({ audio: true })
|
||
.then((stream) => {
|
||
const rec = new MediaRecorder(stream, mime ? { mimeType: mime } : undefined);
|
||
chunksRef.current = [];
|
||
rec.ondataavailable = (e) => { if (e.data && e.data.size > 0) chunksRef.current.push(e.data); };
|
||
rec.onstop = async () => {
|
||
stream.getTracks().forEach((t) => t.stop());
|
||
setVoiceState('transcribing');
|
||
const blob = new Blob(chunksRef.current, { type: mime || 'audio/webm' });
|
||
try {
|
||
const fmt = (mime || 'webm').split(';')[0].split('/')[1] || 'm4a';
|
||
const form = new FormData();
|
||
form.append('file', blob, `recording.${fmt === 'mp4' ? 'm4a' : fmt}`);
|
||
form.append('format', fmt === 'mp4' ? 'm4a' : fmt);
|
||
const res = await fetch(`${API_BASE}/api/ai/asr`, { method: 'POST', body: form });
|
||
const data = await res.json();
|
||
const text = (data && data.text || '').trim();
|
||
if (text) {
|
||
send(text, true);
|
||
} else {
|
||
setVoiceState('idle');
|
||
setMsgs((prev) => [...prev, { id: nextId(), role: 'ai', text: '抱歉，没有听清您的问题，请再试一次，或改用文字输入。' }]);
|
||
}
|
||
} catch {
|
||
// 后端不可用：回退本地模拟识别
|
||
setVoiceState('transcribing');
|
||
timerRef.current = setTimeout(() => {
|
||
setVoiceState('idle');
|
||
send(pick(VOICE_SAMPLES), true);
|
||
}, 1000);
|
||
}
|
||
};
|
||
rec.onerror = () => {
|
||
stream.getTracks().forEach((t) => t.stop());
|
||
setVoiceState('idle');
|
||
send(pick(VOICE_SAMPLES), true);
|
||
};
|
||
recorderRef.current = rec;
|
||
rec.start();
|
||
setVoiceState('recording');
|
||
})
|
||
.catch(() => {
|
||
// 麦克风权限被拒：模拟
|
||
setVoiceState('transcribing');
|
||
timerRef.current = setTimeout(() => {
|
||
setVoiceState('idle');
|
||
send(pick(VOICE_SAMPLES), true);
|
||
}, 1200);
|
||
});
|
||
}, [send]);
|
||
|
||
// MQTT 调度器 → AI 页操作：输入并发送 / 选择预设问题
|
||
useEffect(() => {
|
||
const onAiAction = (e) => {
|
||
const { type, text, index } = e.detail || {};
|
||
if (type === 'input' && text && String(text).trim()) {
|
||
setInput(String(text).trim());
|
||
void send(String(text).trim());
|
||
} else if (type === 'preset') {
|
||
const q = QUICK_QUESTIONS[Number(index)];
|
||
if (q) void send(q);
|
||
}
|
||
};
|
||
window.addEventListener('dpm:ai-action', onAiAction);
|
||
return () => window.removeEventListener('dpm:ai-action', onAiAction);
|
||
}, [send]);
|
||
|
||
const toggleVoice = useCallback(() => {
|
||
if (voiceStateRef.current === 'recording') {
|
||
const rec = recorderRef.current;
|
||
if (rec && rec.state === 'recording') {
|
||
rec.stop(); // 触发 onstop → 上传识别
|
||
} else {
|
||
setVoiceState('idle');
|
||
}
|
||
} else if (voiceStateRef.current === 'idle') {
|
||
startRecording();
|
||
}
|
||
}, [startRecording]);
|
||
|
||
const switchMode = useCallback((m) => {
|
||
setMode(m);
|
||
if (voiceStateRef.current === 'recording') {
|
||
try { recorderRef.current && recorderRef.current.stop(); } catch { /* ignore */ }
|
||
setVoiceState('idle');
|
||
}
|
||
}, []);
|
||
|
||
const voiceHint = {
|
||
idle: '点击麦克风，说出你的问题',
|
||
recording: '正在聆听… 再次点击结束',
|
||
transcribing: '正在识别语音…',
|
||
}[voiceState];
|
||
|
||
return (
|
||
<section className="bd-chat">
|
||
{/* 头部 */}
|
||
<div className="bd-chat-head">
|
||
<span className="bd-chat-logo"><Icon name="robot" size={17} /></span>
|
||
<div className="bd-chat-titles">
|
||
<b>AI 智能助手</b>
|
||
<span>阿里云通义千问 · 语音识别</span>
|
||
</div>
|
||
<div className="bd-chat-modes">
|
||
<button className={mode === 'text' ? 'active' : ''} onClick={() => switchMode('text')} title="文字对话">
|
||
<Icon name="message-circle" size={13} />文字
|
||
</button>
|
||
<button className={mode === 'voice' ? 'active' : ''} onClick={() => switchMode('voice')} title="语音对话">
|
||
<Icon name="microphone" size={13} />语音
|
||
</button>
|
||
</div>
|
||
<span className="bd-chat-model"><i />通义千问</span>
|
||
<button className="bd-chat-clear" title="清空对话" onClick={clear}>
|
||
<Icon name="eraser" size={15} />
|
||
</button>
|
||
</div>
|
||
|
||
{/* 消息区 */}
|
||
<div className="bd-chat-msgs" ref={scrollRef}>
|
||
{msgs.map((m) => (
|
||
<div className={`bd-msg ${m.role}`} key={m.id}>
|
||
{m.role === 'ai' && <span className="bd-msg-avatar"><Icon name="robot" size={13} /></span>}
|
||
<div className="bd-msg-bubble">
|
||
{m.voice && <span className="bd-msg-voice-tag"><Icon name="microphone" size={10} />语音</span>}
|
||
{m.text}
|
||
</div>
|
||
</div>
|
||
))}
|
||
{typing && (
|
||
<div className="bd-msg ai">
|
||
<span className="bd-msg-avatar"><Icon name="robot" size={13} /></span>
|
||
<div className="bd-msg-bubble bd-typing"><i /><i /><i /></div>
|
||
</div>
|
||
)}
|
||
</div>
|
||
|
||
{/* 文字模式 */}
|
||
{mode === 'text' && (
|
||
<>
|
||
<div className="bd-chat-quick">
|
||
{QUICK_QUESTIONS.map((q) => (
|
||
<button key={q} onClick={() => send(q)}>
|
||
<Icon name="sparkles" size={12} />
|
||
{q}
|
||
</button>
|
||
))}
|
||
</div>
|
||
<div className="bd-chat-input">
|
||
<input
|
||
value={input}
|
||
placeholder="输入你的问题，如：展示入驻企业卡片"
|
||
onChange={(e) => setInput(e.target.value)}
|
||
onKeyDown={(e) => {
|
||
if (e.key === 'Enter' && !e.nativeEvent.isComposing) {
|
||
e.preventDefault();
|
||
send(input);
|
||
}
|
||
}}
|
||
/>
|
||
<button className="bd-chat-send" disabled={!input.trim() || typing} onClick={() => send(input)} title="发送">
|
||
<Icon name="send" size={16} />
|
||
</button>
|
||
</div>
|
||
</>
|
||
)}
|
||
|
||
{/* 语音模式 */}
|
||
{mode === 'voice' && (
|
||
<div className="bd-chat-voice">
|
||
<div className={`bd-chat-voice-hint ${voiceState}`}>
|
||
{voiceState === 'recording' && <span className="bd-voice-wave"><i /><i /><i /><i /><i /></span>}
|
||
{voiceHint}
|
||
</div>
|
||
<button
|
||
className={`bd-chat-mic ${voiceState}`}
|
||
onClick={toggleVoice}
|
||
title={voiceState === 'recording' ? '结束并发送' : '开始语音提问'}
|
||
>
|
||
{voiceState === 'recording' && <span className="bd-mic-ring r1" />}
|
||
{voiceState === 'recording' && <span className="bd-mic-ring r2" />}
|
||
<Icon
|
||
name={voiceState === 'recording' ? 'microphone-filled' : 'microphone'}
|
||
size={30}
|
||
/>
|
||
</button>
|
||
<div className="bd-chat-voice-tip">
|
||
{voiceState === 'recording' ? '再次点击结束并识别' : '点击开始说话 · 阿里云语音识别'}
|
||
</div>
|
||
</div>
|
||
)}
|
||
</section>
|
||
);
|
||
}
|