refactor(backend): 启动环境初始化与资源项目化(零跨项目依赖)
- main.py:uv run main.py 一键启动——读取 backend/.env、TORCH_HOME/NLTK_DATA 指向项目内缓存、certifi SSL + NO_PROXY 网络直连、默认开启 s2s,启动横幅 - vendor/:补丁后 s2s 云化栈移入项目(backend/vendor/s2s-cloud),pyproject sources 改相对路径 - .gitignore:忽略 .torch-cache / nltk_data 缓存 - AGENTS.md:进程管理铁律(AI 不启动/重启服务,由用户操作)
This commit is contained in:
+57
@@ -0,0 +1,57 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM
|
||||
* conversion for the audio frames on the wire, transcript extraction from a
|
||||
* `response.done` payload, and a tiny URL helper. Kept separate from the client
|
||||
* so the protocol/state logic stays readable.
|
||||
*/
|
||||
|
||||
/** @param {string} url */
|
||||
export function trimTrailingSlash(url) {
|
||||
return url.endsWith("/") ? url.slice(0, -1) : url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the assistant transcript out of a `response.done` payload. The text
|
||||
* lives in `response.output[].content[].transcript` (audio) or `.text`. Used as
|
||||
* the source of truth for interrupted replies, where the dedicated
|
||||
* `*.transcript.done` event may never arrive.
|
||||
* @param {any} response
|
||||
* @returns {string}
|
||||
*/
|
||||
export function extractResponseTranscript(response) {
|
||||
const output = response?.output;
|
||||
if (!Array.isArray(output)) return "";
|
||||
/** @type {string[]} */
|
||||
const parts = [];
|
||||
for (const item of output) {
|
||||
for (const part of item?.content ?? []) {
|
||||
const text = part?.transcript ?? part?.text;
|
||||
if (typeof text === "string" && text.trim()) parts.push(text.trim());
|
||||
}
|
||||
}
|
||||
return parts.join(" ").trim();
|
||||
}
|
||||
|
||||
/** @param {ArrayBuffer} buf */
|
||||
export function base64FromArrayBuffer(buf) {
|
||||
const bytes = new Uint8Array(buf);
|
||||
// Chunked encoding so we don't blow up the call stack on long buffers.
|
||||
let binary = "";
|
||||
const chunk = 0x8000;
|
||||
for (let i = 0; i < bytes.length; i += chunk) {
|
||||
binary += String.fromCharCode.apply(null, /** @type {number[]} */ (
|
||||
/** @type {unknown} */ (bytes.subarray(i, i + chunk))
|
||||
));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/** @param {string} b64 */
|
||||
export function base64ToBytes(b64) {
|
||||
const binary = atob(b64);
|
||||
const len = binary.length;
|
||||
const out = new Uint8Array(len);
|
||||
for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// @ts-check
|
||||
/**
|
||||
* Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the
|
||||
* mic input and the TTS output) and maps the low-frequency speech energy onto
|
||||
* the orb's CSS custom properties:
|
||||
* - `--bar0`..`--bar4` the 5-band level meter
|
||||
* - `--ai-audio-level` the global "Reachy talks" glow / scale pulse
|
||||
*
|
||||
* The bottom of the FFT is where speech energy lives, so the band edges stay
|
||||
* low — that keeps the bars dancing on voice rather than on noise. While the AI
|
||||
* is speaking we source the bars from the OUTPUT analyser so the orb pulses with
|
||||
* Reachy's voice instead of sitting dead while the user is silent.
|
||||
*/
|
||||
|
||||
// Exported so the client can size its AnalyserNodes to match our buffer.
|
||||
export const VIS_FFT_SIZE = 256;
|
||||
const VIS_BAND_COUNT = 5;
|
||||
const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52];
|
||||
const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy)
|
||||
const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade)
|
||||
|
||||
export class OrbVisualiser {
|
||||
/**
|
||||
* @param {AnalyserNode} micAnalyser
|
||||
* @param {AnalyserNode} outAnalyser
|
||||
* @param {() => boolean} isAiSpeaking Source the bars from the AI output when
|
||||
* true, otherwise from the mic.
|
||||
*/
|
||||
constructor(micAnalyser, outAnalyser, isAiSpeaking) {
|
||||
this._mic = micAnalyser;
|
||||
this._out = outAnalyser;
|
||||
this._isAiSpeaking = isAiSpeaking;
|
||||
this._buf = new Uint8Array(micAnalyser.frequencyBinCount);
|
||||
this._bands = new Float32Array(VIS_BAND_COUNT);
|
||||
this._aiLevel = 0;
|
||||
/** @type {number | null} */
|
||||
this._frame = null;
|
||||
}
|
||||
|
||||
/** Begin the rAF loop (idempotent). */
|
||||
start() {
|
||||
if (this._frame !== null) return;
|
||||
const root = document.documentElement;
|
||||
const tick = () => {
|
||||
this._frame = requestAnimationFrame(tick);
|
||||
this._update(root);
|
||||
};
|
||||
this._frame = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
/** Stop the loop and clear the CSS vars so the orb returns to rest. */
|
||||
stop() {
|
||||
if (this._frame !== null) {
|
||||
cancelAnimationFrame(this._frame);
|
||||
this._frame = null;
|
||||
}
|
||||
const root = document.documentElement;
|
||||
for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`);
|
||||
root.style.removeProperty("--ai-audio-level");
|
||||
}
|
||||
|
||||
/** @param {HTMLElement} root */
|
||||
_update(root) {
|
||||
// Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars.
|
||||
const source = this._isAiSpeaking() ? this._out : this._mic;
|
||||
source.getByteFrequencyData(this._buf);
|
||||
|
||||
for (let b = 0; b < VIS_BAND_COUNT; b++) {
|
||||
const lo = VIS_BAND_EDGES[b];
|
||||
const hi = VIS_BAND_EDGES[b + 1];
|
||||
let sum = 0;
|
||||
let n = 0;
|
||||
for (let i = lo; i < hi && i < this._buf.length; i++) {
|
||||
sum += this._buf[i];
|
||||
n += 1;
|
||||
}
|
||||
const target = n > 0 ? sum / (n * 255) : 0;
|
||||
const prev = this._bands[b];
|
||||
const k = target > prev ? VIS_ATTACK : VIS_RELEASE;
|
||||
const next = prev + (target - prev) * k;
|
||||
this._bands[b] = next;
|
||||
root.style.setProperty(`--bar${b}`, next.toFixed(3));
|
||||
}
|
||||
|
||||
// Global AI audio level: peak of the output analyser, used by the CSS to
|
||||
// make the orb's glow / scale react to Reachy's voice.
|
||||
this._out.getByteFrequencyData(this._buf);
|
||||
let peak = 0;
|
||||
const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]);
|
||||
for (let i = 0; i < limit; i++) {
|
||||
if (this._buf[i] > peak) peak = this._buf[i];
|
||||
}
|
||||
const aiTarget = peak / 255;
|
||||
const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE;
|
||||
this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k;
|
||||
root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3));
|
||||
}
|
||||
}
|
||||
+1176
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user