feat: 更新语音助手名称为昆小创，修改欢迎语和提示词
This commit is contained in:
Pine
@@ -90,7 +90,7 @@ class Settings:
|
||||
# 语音助手基础提示词（.env 设 DPM_S2S_INSTRUCTIONS 可覆盖；知识库内容由 rag 自动拼接）
|
||||
S2S_INSTRUCTIONS = _env(
|
||||
"DPM_S2S_INSTRUCTIONS",
|
||||
"你是昆小园，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、"
|
||||
"你是昆小创，昆明市大学生创业园的专属智能语音助手。可用工具：get_park_overview（园区实时数据）、"
|
||||
"query_companies（企业名录）、control_display（大屏控制）、get_time（时间）。"
|
||||
"涉及园区数据/企业/大屏控制时务必调用工具获取准确信息。请用简洁专业的中文回答，不超过三句话。",
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ const pick = (arr) => arr[Math.floor(Math.random() * arr.length)];
|
||||
const VOICE_SAMPLES = QUICK_QUESTIONS;
|
||||
|
||||
/* 默认欢迎语（新对话第一条消息，与初始一致） */
|
||||
const WELCOME_TEXT = '您好，我是「问问小园」，昆明市大学生创业园的专属 AI 助手（通义千问）。\n\n可以为您解答入驻申请、创业政策、创业担保贷款、OPC 概念、AI 工具等问题，也可以介绍园区入驻企业。试试右侧的预设问题吧。';
|
||||
const WELCOME_TEXT = '您好，我是昆小创，昆明市大学生创业园的专属 AI 助手。\n\n可以为您解答入驻申请、创业政策、创业担保贷款、OPC 概念、AI 工具等问题，也可以介绍园区入驻企业。试试右侧的预设问题吧。';
|
||||
|
||||
/* ---------- 消息 ---------- */
|
||||
let msgId = 0;
|
||||
@@ -320,9 +320,9 @@ export default function AiChatPanel() {
|
||||
<span className="bd-chat-logo"><Icon name="robot" size={17} /></span>
|
||||
<div className="bd-chat-titles">
|
||||
<b>问问小园</b>
|
||||
<span>昆明市大学生创业园 · 通义千问</span>
|
||||
<span>昆明市大学生创业园 · 昆小创</span>
|
||||
</div>
|
||||
<span className="bd-chat-model"><i />通义千问</span>
|
||||
<span className="bd-chat-model"><i />昆小创</span>
|
||||
<button className="bd-chat-clear" title="清空对话" onClick={clear}>
|
||||
<Icon name="eraser" size={15} />
|
||||
</button>
|
||||
|
||||
+22
-48
@@ -1,4 +1,10 @@
|
||||
// @ts-check
|
||||
// AudioWorklet processor 源码（构建期内联，运行时直接 Blob → addModule）。
|
||||
// 打包环境（Tauri v2 + WebView2）无法可靠地从 tauri.localhost 自定义协议或
|
||||
// 跨域后端 URL 用 addModule 加载 worklet 模块；源码内联后零网络、零 CORS、零后端依赖。
|
||||
import micWorkletSrc from "./worklets/mic-capture.js?raw";
|
||||
import playbackWorkletSrc from "./worklets/audio-playback.js?raw";
|
||||
|
||||
/**
|
||||
* Minimal WebSocket client for the Hugging Face speech-to-speech load balancer.
|
||||
*
|
||||
@@ -479,57 +485,25 @@ export class S2sWsRealtimeClient extends EventTarget {
|
||||
}
|
||||
|
||||
// AudioWorklet 加载（WebView2/打包兼容）：
|
||||
// 1) 先直接 addModule(http 地址) —— 浏览器/常规环境快路径
|
||||
// 2) 失败则 fetch 源码 → Blob URL → addModule —— 打包 WebView2 下
|
||||
// tauri.localhost 自定义协议与跨域均受限，Blob 同源最稳
|
||||
// 3) 多个候选基址（后端 http / 页面源 / 相对路径）
|
||||
const candidates = [
|
||||
window.VOICE_WORKLETS_BASE,
|
||||
`${window.location.origin}/voice-worklets/`,
|
||||
"/voice-worklets/",
|
||||
].filter(Boolean);
|
||||
let workletErr = null;
|
||||
let workletOk = false;
|
||||
|
||||
const addViaBlob = async (name) => {
|
||||
for (const base of candidates) {
|
||||
try {
|
||||
const res = await fetch(base + name, { cache: "no-store" });
|
||||
if (!res.ok) continue;
|
||||
const src = await res.text();
|
||||
const url = URL.createObjectURL(new Blob([src], { type: "application/javascript" }));
|
||||
try {
|
||||
await ctx.audioWorklet.addModule(url);
|
||||
return true;
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
} catch (err) {
|
||||
workletErr = err;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
for (const base of candidates) {
|
||||
// 源码已由 Vite `?raw` 内联进 bundle，运行时直接从内存字符串建 Blob → addModule。
|
||||
// 彻底摆脱运行时 fetch/网络/CORS/后端可达性依赖，任何 WebView2 都能加载。
|
||||
const addWorkletFromSrc = async (name, src) => {
|
||||
const url = URL.createObjectURL(new Blob([src], { type: "text/javascript" }));
|
||||
try {
|
||||
await ctx.audioWorklet.addModule(base + "mic-capture.js?v=3");
|
||||
await ctx.audioWorklet.addModule(base + "audio-playback.js?v=3");
|
||||
workletOk = true;
|
||||
break;
|
||||
} catch (err) {
|
||||
workletErr = err;
|
||||
console.warn("[ws] worklet 直接加载失败，改走 Blob 方案:", base, err);
|
||||
await ctx.audioWorklet.addModule(url);
|
||||
return true;
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
};
|
||||
const workletOk =
|
||||
(await addWorkletFromSrc("mic-capture", micWorkletSrc)) &&
|
||||
(await addWorkletFromSrc("audio-playback", playbackWorkletSrc));
|
||||
if (workletOk) {
|
||||
console.info("[ws] AudioWorklet 已通过内联源码 Blob URL 加载（WebView2 兼容模式）");
|
||||
} else {
|
||||
throw new Error("AudioWorklet 模块加载失败（内联源码 Blob addModule 失败）");
|
||||
}
|
||||
if (!workletOk) {
|
||||
// 回退：fetch → Blob → addModule（WebView2 打包环境最稳）
|
||||
const micOk = await addViaBlob("mic-capture.js?v=3");
|
||||
const playOk = await addViaBlob("audio-playback.js?v=3");
|
||||
workletOk = micOk && playOk;
|
||||
if (workletOk) console.info("[ws] AudioWorklet 已通过 Blob URL 加载（WebView2 兼容模式）");
|
||||
}
|
||||
if (!workletOk) throw workletErr || new Error("AudioWorklet 模块加载失败");
|
||||
|
||||
const captureNode = new AudioWorkletNode(ctx, "mic-capture", {
|
||||
numberOfInputs: 1,
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* AudioWorkletProcessor that plays back Float32 mono samples received from
|
||||
* the main thread, upsampling whatever incoming rate the server uses
|
||||
* (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz).
|
||||
*
|
||||
* Lifecycle / messaging:
|
||||
*
|
||||
* main -> worklet:
|
||||
* { kind: "config", inputRate: 16000, muted: false } one-shot at startup
|
||||
* { kind: "audio", samples: Float32Array } per chunk
|
||||
* { kind: "clear" } wipe queue (barge-in)
|
||||
*
|
||||
* worklet -> main:
|
||||
* { kind: "stats", queuedMs, played } every ~250 ms
|
||||
* { kind: "underrun" } every time the queue
|
||||
* runs dry mid-playback
|
||||
*
|
||||
* `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the
|
||||
* same audio, so the page mutes its own playback to avoid double audio. When
|
||||
* the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page
|
||||
* plays the backend audio itself.
|
||||
*
|
||||
* IMPORTANT: the read position only advances while audio is actually being
|
||||
* played (`_playing`). Advancing during idle would drift `_readIdx` far past
|
||||
* the next chunk's length and turn every read into NaN (silence) once audio
|
||||
* arrives.
|
||||
*/
|
||||
|
||||
const STATS_INTERVAL_FRAMES = 12000;
|
||||
const FADE_FRAMES = 32;
|
||||
|
||||
class AudioPlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this._inputRate = 16000;
|
||||
this._stepRatio = this._inputRate / sampleRate;
|
||||
this._muted = false;
|
||||
this._queue = [];
|
||||
this._readIdx = 0;
|
||||
this._fracPos = 0;
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this._framesSinceStats = 0;
|
||||
this._totalPlayed = 0;
|
||||
this._dbgReceived = 0;
|
||||
this._dbgOutPeak = 0;
|
||||
this._fadeIn = 0;
|
||||
this._fadeOut = 0;
|
||||
|
||||
this.port.onmessage = (e) => {
|
||||
const data = e.data;
|
||||
if (!data || typeof data !== "object") return;
|
||||
switch (data.kind) {
|
||||
case "config":
|
||||
if (typeof data.inputRate === "number" && data.inputRate > 0) {
|
||||
this._inputRate = data.inputRate;
|
||||
this._stepRatio = this._inputRate / sampleRate;
|
||||
}
|
||||
if (typeof data.muted === "boolean") this._muted = data.muted;
|
||||
break;
|
||||
case "audio": {
|
||||
// Accept Float32Array, a transferred ArrayBuffer, or a plain
|
||||
// {buffer, byteOffset, length} descriptor: cross-realm structured
|
||||
// cloning can deliver the samples in any of these shapes.
|
||||
let arr = data.samples;
|
||||
let f32 = null;
|
||||
if (arr instanceof Float32Array) {
|
||||
f32 = arr;
|
||||
} else if (arr instanceof ArrayBuffer) {
|
||||
f32 = new Float32Array(arr);
|
||||
} else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) {
|
||||
f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2);
|
||||
}
|
||||
if (f32 && f32.length > 0) {
|
||||
this._queue.push(f32);
|
||||
this._dbgReceived += 1;
|
||||
if (!this._playing) {
|
||||
this._playing = true;
|
||||
this._fadeIn = FADE_FRAMES;
|
||||
this._fadeOut = 0;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "clear":
|
||||
this._queue.length = 0;
|
||||
this._readIdx = 0;
|
||||
this._fracPos = 0;
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this._fadeOut = FADE_FRAMES;
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
_queuedSamples() {
|
||||
let total = -this._readIdx;
|
||||
for (const buf of this._queue) total += buf.length;
|
||||
return Math.max(0, total);
|
||||
}
|
||||
|
||||
/** Linear-interp read at the current fractional position. */
|
||||
_readInterpolated() {
|
||||
if (this._queue.length === 0) return null;
|
||||
const head = this._queue[0];
|
||||
const idx = this._readIdx;
|
||||
const frac = this._fracPos;
|
||||
|
||||
let a = head[idx];
|
||||
let b;
|
||||
if (idx + 1 < head.length) {
|
||||
b = head[idx + 1];
|
||||
} else if (this._queue.length > 1) {
|
||||
b = this._queue[1][0];
|
||||
} else {
|
||||
b = a;
|
||||
}
|
||||
return a + (b - a) * frac;
|
||||
}
|
||||
|
||||
/** Advance the read position by `stepRatio`; pop consumed buffers. */
|
||||
_advance() {
|
||||
this._fracPos += this._stepRatio;
|
||||
while (this._fracPos >= 1) {
|
||||
this._fracPos -= 1;
|
||||
this._readIdx += 1;
|
||||
}
|
||||
while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) {
|
||||
this._readIdx -= this._queue[0].length;
|
||||
this._queue.shift();
|
||||
}
|
||||
}
|
||||
|
||||
process(_, outputs) {
|
||||
const channels = outputs[0];
|
||||
if (!channels || channels.length === 0) return true;
|
||||
const out = channels[0];
|
||||
const stereo = channels.length > 1 ? channels[1] : null;
|
||||
|
||||
// Muted mode (Live Avatar WebRTC carries the audio): output silence.
|
||||
if (this._muted) {
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
out[i] = 0;
|
||||
if (stereo) stereo[i] = 0;
|
||||
}
|
||||
this._framesSinceStats += out.length;
|
||||
return true;
|
||||
}
|
||||
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
let sample = 0;
|
||||
|
||||
if (this._playing) {
|
||||
const v = this._readInterpolated();
|
||||
if (v === null) {
|
||||
// Underrun: ramp out cleanly to avoid clicks.
|
||||
sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES);
|
||||
this._lastSample = sample;
|
||||
if (Math.abs(sample) < 1e-4) {
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
this.port.postMessage({ kind: "underrun" });
|
||||
}
|
||||
} else {
|
||||
sample = v;
|
||||
this._lastSample = v;
|
||||
this._advance();
|
||||
}
|
||||
|
||||
if (this._fadeIn > 0) {
|
||||
const gain = 1 - this._fadeIn / FADE_FRAMES;
|
||||
sample *= gain;
|
||||
this._fadeIn -= 1;
|
||||
}
|
||||
if (this._fadeOut > 0) {
|
||||
const gain = this._fadeOut / FADE_FRAMES;
|
||||
sample *= gain;
|
||||
this._fadeOut -= 1;
|
||||
if (this._fadeOut === 0) {
|
||||
this._playing = false;
|
||||
this._lastSample = 0;
|
||||
}
|
||||
}
|
||||
|
||||
this._totalPlayed += 1;
|
||||
const abs = sample < 0 ? -sample : sample;
|
||||
if (abs > this._dbgOutPeak) this._dbgOutPeak = abs;
|
||||
}
|
||||
|
||||
out[i] = sample;
|
||||
if (stereo) stereo[i] = sample;
|
||||
}
|
||||
|
||||
this._framesSinceStats += out.length;
|
||||
if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) {
|
||||
this.port.postMessage({
|
||||
kind: "stats",
|
||||
queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000),
|
||||
played: this._totalPlayed,
|
||||
received: this._dbgReceived,
|
||||
outPeak: this._dbgOutPeak,
|
||||
});
|
||||
this._framesSinceStats = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("audio-playback", AudioPlaybackProcessor);
|
||||
@@ -0,0 +1,159 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz)
|
||||
* down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it
|
||||
* back to the main thread in fixed-size chunks.
|
||||
*
|
||||
* The Hugging Face speech-to-speech WebSocket route expects the
|
||||
* `input_audio_buffer.append` payload at 16 kHz PCM16 mono.
|
||||
*
|
||||
* Design notes:
|
||||
* - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a
|
||||
* cheap low-pass before decimating. Good enough for voice STT; we lose
|
||||
* a tiny bit of >8 kHz content which the pipeline discards anyway.
|
||||
* - Output frames are emitted at the cadence dictated by `chunkMs`
|
||||
* (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime
|
||||
* server batches incoming audio so the cadence is flexible; 20-100 ms
|
||||
* is the sweet spot.
|
||||
* - Float -> Int16 saturates to [-1, 1] before scaling.
|
||||
* - Optional noise gate: per-chunk RMS decides open/closed against a
|
||||
* threshold; the gain ramps (fast attack, hold, slow release) so word
|
||||
* onsets aren't clipped and quiet tails don't click. The gate only
|
||||
* affects the audio we SEND; the main-thread visualiser taps the raw
|
||||
* mic separately. We post the chunk RMS up every frame so the Settings
|
||||
* mic meter can show the live level against the threshold.
|
||||
*/
|
||||
|
||||
const TARGET_RATE = 16000;
|
||||
const DEFAULT_CHUNK_MS = 40;
|
||||
// Gate envelope timing (fixed; only the threshold is user-tunable).
|
||||
const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive
|
||||
const GATE_HOLD_MS = 250; // stay open this long after the level drops back under
|
||||
const GATE_RELEASE_MS = 80; // then fade closed over this long (no click)
|
||||
|
||||
class MicCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor(options) {
|
||||
super();
|
||||
const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS;
|
||||
this._inputRate = sampleRate;
|
||||
this._ratio = this._inputRate / TARGET_RATE;
|
||||
this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000);
|
||||
this._scratch = new Float32Array(0);
|
||||
this._decimated = new Float32Array(this._chunkSamples16k);
|
||||
this._enabled = true;
|
||||
|
||||
// Noise gate state. Disabled by default (pure passthrough).
|
||||
this._gateEnabled = false;
|
||||
this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open
|
||||
this._gateGain = 1; // smoothed gain currently applied
|
||||
this._holdRemaining = 0; // samples left before the gate may start closing
|
||||
this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE));
|
||||
this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE));
|
||||
this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE);
|
||||
|
||||
this.port.onmessage = (e) => {
|
||||
const data = e.data;
|
||||
if (data?.kind === "enable") this._enabled = !!data.value;
|
||||
else if (data?.kind === "gate") {
|
||||
this._gateEnabled = !!data.enabled;
|
||||
// dB -> linear amplitude. When off, threshold 0 keeps the gate open.
|
||||
this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Append `incoming` to the internal scratch buffer, then emit as many
|
||||
* full output chunks as we have material for.
|
||||
* @param {Float32Array} incoming
|
||||
*/
|
||||
_ingest(incoming) {
|
||||
if (incoming.length === 0) return;
|
||||
const next = new Float32Array(this._scratch.length + incoming.length);
|
||||
next.set(this._scratch, 0);
|
||||
next.set(incoming, this._scratch.length);
|
||||
this._scratch = next;
|
||||
this._maybeEmit();
|
||||
}
|
||||
|
||||
_maybeEmit() {
|
||||
const r = this._ratio;
|
||||
const n = this._chunkSamples16k;
|
||||
const needIn = Math.ceil(n * r);
|
||||
const dec = this._decimated;
|
||||
while (this._scratch.length >= needIn) {
|
||||
// 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter.
|
||||
let sumSq = 0;
|
||||
if (Math.abs(r - 3) < 1e-6) {
|
||||
// 48 kHz -> 16 kHz fast path with boxcar lowpass.
|
||||
for (let i = 0; i < n; i++) {
|
||||
const idx = i * 3;
|
||||
const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3;
|
||||
dec[i] = s;
|
||||
sumSq += s * s;
|
||||
}
|
||||
} else {
|
||||
// Generic path: linear interpolation. Slower but works at any rate
|
||||
// (e.g. some Windows boxes report sampleRate=44100).
|
||||
for (let i = 0; i < n; i++) {
|
||||
const srcPos = i * r;
|
||||
const idx = Math.floor(srcPos);
|
||||
const frac = srcPos - idx;
|
||||
const a = this._scratch[idx];
|
||||
const b = this._scratch[idx + 1] ?? a;
|
||||
const s = a + (b - a) * frac;
|
||||
dec[i] = s;
|
||||
sumSq += s * s;
|
||||
}
|
||||
}
|
||||
const rms = Math.sqrt(sumSq / n);
|
||||
|
||||
// 2. Decide the gate target for this chunk, then ramp sample-by-sample.
|
||||
let target = 1;
|
||||
if (this._gateEnabled) {
|
||||
if (rms >= this._thresholdLin) {
|
||||
this._holdRemaining = this._holdSamples; // re-arm the hold
|
||||
} else if (this._holdRemaining > 0) {
|
||||
this._holdRemaining -= n; // coasting through the hold window
|
||||
} else {
|
||||
target = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Apply the (smoothed) gain and pack to Int16.
|
||||
const out = new Int16Array(n);
|
||||
let gain = this._gateGain;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const coef = target > gain ? this._attackCoef : this._releaseCoef;
|
||||
gain = target + (gain - target) * coef;
|
||||
const s = dec[i] * gain;
|
||||
const clamped = s < -1 ? -1 : s > 1 ? 1 : s;
|
||||
out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
|
||||
}
|
||||
this._gateGain = gain;
|
||||
|
||||
// Shift the scratch buffer to keep only the trailing unused samples.
|
||||
const consumed = Math.floor(n * r);
|
||||
this._scratch = this._scratch.slice(consumed);
|
||||
|
||||
// Live input level for the Settings meter (raw RMS, pre-gate).
|
||||
this.port.postMessage({ kind: "level", rms });
|
||||
|
||||
if (this._enabled) {
|
||||
this.port.postMessage(out.buffer, [out.buffer]);
|
||||
}
|
||||
// When disabled (mic muted) we silently consume input so the worklet
|
||||
// stays alive and the buffer never grows unbounded.
|
||||
}
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || input.length === 0 || !input[0]) return true;
|
||||
const mono = input[0];
|
||||
if (mono.length > 0) this._ingest(mono);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("mic-capture", MicCaptureProcessor);
|
||||
Reference in New Issue
Block a user