diff --git a/public/voice-demo/config.js b/public/voice-demo/config.js deleted file mode 100644 index 85a427c..0000000 --- a/public/voice-demo/config.js +++ /dev/null @@ -1,3 +0,0 @@ -// voice-demo 语音对话页 WS 地址：自动取自当前页面 hostname（与 DPM 后端同机/同网段） -// 如服务器 IP 是 192.168.1.9 → ws://192.168.1.9:8765/v1/realtime -window.VOICE_WS_URL = window.VOICE_WS_URL || ("ws://" + location.hostname + ":8765/v1/realtime"); diff --git a/public/voice-demo/index.html b/public/voice-demo/index.html deleted file mode 100644 index 8e6975a..0000000 --- a/public/voice-demo/index.html +++ /dev/null @@ -1,353 +0,0 @@ - - - - - - Minimal Conversation · S2S backend (WebSocket) - - - - - - - - - - -
-
-
- - - - - - -
-
- -
-
-
- - -
- - - - -
- -

Tap to start

- - - - -
- - -
-
- - -
- - -
- - - - - -
-
-
-
-

Conversation

- -
-
-
-
- - -
-
-

About

- -
- - -
-

Speech-to-speech is Hugging Face's open framework for real-time voice agents. Rather than one end-to-end model, it chains four open models from the Hub (speech detection, transcription, a vision-language model, and synthesis), so any stage can be swapped or run locally. This demo wires that pipeline to hosted inference.

- View the project on GitHub -
- - -
-

The pipeline

-
    -
  1. You speak
    2. -
  2. - VAD - detects speech - silero-vad -
    3. -
  3. - STT - transcribes it - nvidia/parakeet-tdt-1.1b -
    4. -
  4. - VLM - analyses and composes the reply · via Cerebras - google/gemma-4-31B-it -
    5. -
  5. - TTS - speaks it back - Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice -
    6. -
  6. The orb replies
    7. -
-
-
- - - -
-
-

Settings

- -
- -
-
- - -
- -
- -
- - Noise gate - Off - -
- -
- Off - −3 dB -
-
- - Mutes the mic below the handle so room noise isn't sent. Slide fully - left to turn it off; the bar shows your live input while in a call. - -
- - - -
- - Connect first, then come back here to apply live changes. -
-
-
- - -
- - - -
-
-

Tools

- -
- -

Let the assistant act during the conversation. Changes apply live.

- -
-
-
- Web search - Look things up on Google, via Serper. -
- -
- - -
-
- Camera - Let the assistant see through your webcam. -
- -
- A live preview shows bottom-left while the camera is on. -
-
- - - - -
- - -

That's a wrap for now

-

- -

-
- - - - - - diff --git a/public/voice-demo/main.js b/public/voice-demo/main.js deleted file mode 100644 index 1e6f701..0000000 --- a/public/voice-demo/main.js +++ /dev/null @@ -1,1503 +0,0 @@ -// @ts-check -/** - * Minimal voice conversation app, talking to a Hugging Face speech-to-speech - * backend over **WebSocket** (drop-in alternative to the WebRTC variant). - * - * Click the orb -> we ask for the mic, POST a session on the LB, open a - * WebSocket on the routed compute endpoint, push session.update + mic - * audio, play back the TTS audio. The orb visually reflects the live - * state (idle, connecting, listening, user-speaking, processing, - * ai-speaking). - * - * The only meaningful difference vs. the WebRTC main.js is that the - * client owns its own AudioContext (no `attachOutputTrack`), so we hand - * it the MediaStream directly. - * - * @typedef {"idle" | "connecting" | "queued" | "your-turn" | "listening" | "user-speaking" | "processing" | "ai-speaking" | "error"} AppState - */ - -import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js"; -import { $, truncateError, DEBUG } from "./ui/dom.js"; -import { ChatView } from "./ui/chat.js"; -import { Account } from "./ui/account.js"; - -const DEFAULT_VOICE = "Aiden"; -const DEFAULT_INSTRUCTIONS = - "你是一个友好的中文语音助手。请用中文简洁地回答用户的问题，语气温暖自然。"; - -// Appended to the user's instructions whenever at least one tool is enabled. -// Stops the model from announcing capabilities ("Yes, I can search") and then -// idling for the next turn — it should act immediately in the same response. -const TOOL_USE_HINT = - " When the user's request calls for one of your tools, do not describe your " + - "capabilities or say you can do it and wait for another turn. Instead, say " + - 'a brief acknowledgement like "Let me search for that..." and call the tool ' + - "right away in the same response."; - -const STORAGE_KEYS = { - // Direct s2s server URL, used only when the deploy has no LOAD_BALANCER_URL - // (in LB mode the browser never learns the LB address — it POSTs /api/session). - directUrl: "s2s.ws.directUrl", - voice: "s2s.ws.voice", - instructions: "s2s.ws.instructions", - tools: "s2s.ws.tools", - searchKey: "s2s.ws.searchKey", - noiseGate: "s2s.ws.noiseGate", -}; - -// ── Noise gate ────────────────────────────────────────────────────────────── -// The Settings cursor sets the gate's open threshold in dBFS. Its leftmost -// position is an OFF detent (gate disabled, pure passthrough); the rest of the -// travel is the active threshold. The cursor shares the meter's dB axis, so the -// handle sits on the level bar — raise it until room noise stops lighting it up. -// The slider range IS the shared axis: the live meter fill and the threshold -// thumb both map across [GATE_OFF_DB, GATE_MAX_DB], so the thumb sits exactly -// where the gate cuts on the same scale as the level bar. -const GATE_OFF_DB = -66; // slider minimum = off / bottom of the meter axis -const GATE_MAX_DB = -3; // slider maximum = most aggressive / top of the meter axis -const GATE_DEFAULT_DB = -50; // first-run default: a gentle gate, enabled - -/** @param {number} thresholdDb @returns {import("./ws/s2s-ws-client.js").NoiseGate} */ -function gateParams(thresholdDb) { - return { enabled: thresholdDb > GATE_OFF_DB, thresholdDb }; -} - -// ── Tools ───────────────────────────────────────────────────────────────── -// Function tools we declare to the backend. The model decides when to call -// one; the executor below runs it and returns the result (see runTool). -/** @type {Record} */ -const TOOL_DEFS = { - web_search: { - type: "function", - name: "web_search", - description: - "Search the web for current or factual information you don't already know " + - "(news, prices, facts, documentation). Returns the top results with titles, " + - "snippets and URLs.", - parameters: { - type: "object", - properties: { query: { type: "string", description: "The search query." } }, - required: ["query"], - }, - }, - camera_snapshot: { - type: "function", - name: "camera_snapshot", - description: - "Capture the current frame from the user's webcam so you can see what they " + - "are showing you. Use it whenever the user refers to something visual or " + - "asks you to look.", - parameters: { type: "object", properties: {}, required: [] }, - }, -}; - -/** Longest edge of the snapshot sent to the VLM, in px (keeps payload sane). */ -const SNAPSHOT_MAX_EDGE = 768; -const SNAPSHOT_QUALITY = 0.7; - -function loadSettings() { - return { - directUrl: localStorage.getItem(STORAGE_KEYS.directUrl) || "", - voice: localStorage.getItem(STORAGE_KEYS.voice) || DEFAULT_VOICE, - instructions: localStorage.getItem(STORAGE_KEYS.instructions) || DEFAULT_INSTRUCTIONS, - noiseGate: loadGateThreshold(), - }; -} - -/** Stored gate threshold (dBFS), clamped to the slider range. Defaults to a - * gentle enabled gate (GATE_DEFAULT_DB) when the user hasn't set one yet. */ -function loadGateThreshold() { - const stored = localStorage.getItem(STORAGE_KEYS.noiseGate); - // getItem returns null when unset, and Number(null) === 0 (finite!), so guard - // the missing/empty case explicitly before coercing — otherwise the default - // never fires and 0 clamps to the slider max. - if (stored === null || stored === "") return GATE_DEFAULT_DB; - const raw = Number(stored); - if (!Number.isFinite(raw)) return GATE_DEFAULT_DB; - return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(raw))); -} - -/** @param {ReturnType} s */ -function saveSettings(s) { - localStorage.setItem(STORAGE_KEYS.directUrl, s.directUrl); - localStorage.setItem(STORAGE_KEYS.voice, s.voice); - localStorage.setItem(STORAGE_KEYS.instructions, s.instructions); - localStorage.setItem(STORAGE_KEYS.noiseGate, String(s.noiseGate)); -} - -/** @returns {{ web_search: boolean, camera_snapshot: boolean }} */ -function loadTools() { - try { - const raw = JSON.parse(localStorage.getItem(STORAGE_KEYS.tools) || "{}"); - // Both tools default ON (web search still only activates when a key exists). - // We never call getUserMedia on page load — the camera only actually starts - // on a user gesture (conversation start), so a default-on flag doesn't - // silently resume the webcam; an explicit saved `false` is respected. - return { - web_search: raw.web_search ?? true, - camera_snapshot: raw.camera_snapshot ?? false, - }; - } catch { - return { web_search: true, camera_snapshot: true }; - } -} - -function saveTools() { - localStorage.setItem(STORAGE_KEYS.tools, JSON.stringify(toolsEnabled)); -} - -/** @type {Record} */ -const STATE_VIEWS = { - idle: { caption: "Tap to start", disabled: false }, - connecting: { caption: "Connecting", disabled: true }, - queued: { caption: "Finding you a spot…", disabled: true }, - "your-turn": { caption: "You're up! 🎉", disabled: true }, - listening: { caption: "", disabled: false }, - "user-speaking": { caption: "", disabled: false }, - processing: { caption: "", disabled: false }, - "ai-speaking": { caption: "", disabled: false }, - error: { caption: "Tap to retry", disabled: false }, -}; - -/** @type {Record} */ -const STATE_CLASS = { - idle: "state-idle", - connecting: "state-connecting", - queued: "state-queued", - "your-turn": "state-your-turn", - listening: "state-listening", - "user-speaking": "state-user-speaking", - processing: "state-processing", - "ai-speaking": "state-ai-speaking", - error: "state-error", -}; - -/** @type {ReadonlySet} */ -const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]); - -/** @type {HTMLButtonElement} */ -const circleBtn = $("#main-circle"); -/** @type {HTMLParagraphElement} */ -const circleCaption = $("#circle-caption"); -/** @type {HTMLParagraphElement} */ -const circleSubcaption = $("#circle-subcaption"); -/** @type {HTMLElement} */ -const orbWrap = $(".orb-wrap"); -/** @type {HTMLButtonElement} */ -const micBtn = $("#mic-btn"); -/** @type {HTMLButtonElement} */ -const stopBtn = $("#stop-btn"); -/** @type {HTMLElement} */ -const queueActions = $("#queue-actions"); -/** @type {HTMLButtonElement} */ -const joinQueueBtn = $("#join-queue-btn"); -/** @type {HTMLButtonElement} */ -const leaveQueueBtn = $("#leave-queue-btn"); - -/** @type {HTMLButtonElement} */ -const settingsBtn = $("#settings-btn"); -/** @type {HTMLDialogElement} */ -const settingsModal = $("#settings-modal"); - -/** @type {HTMLDialogElement} */ -const aboutModal = $("#about-modal"); -/** @type {HTMLButtonElement} */ -const aboutClose = $("#about-close"); - -/** @type {HTMLButtonElement} */ -const toolsBtn = $("#tools-btn"); -/** @type {HTMLDialogElement} */ -const toolsModal = $("#tools-modal"); -/** @type {HTMLButtonElement} */ -const toolsClose = $("#tools-close"); -/** @type {HTMLInputElement} */ -const toolWebSwitch = $("#tool-web"); -/** @type {HTMLInputElement} */ -const toolCamSwitch = $("#tool-cam"); -/** @type {HTMLElement} */ -const toolWebRow = $("#tool-web-row"); -/** @type {HTMLElement} */ -const toolWebHint = $("#tool-web-hint"); -/** @type {HTMLElement} */ -const toolCamHint = $("#tool-cam-hint"); -/** @type {HTMLInputElement} */ -const searchKeyInput = $("#search-key"); -/** @type {HTMLElement} */ -const camPip = $("#cam-pip"); -/** @type {HTMLVideoElement} */ -const camVideo = $("#cam-video"); - -/** @type {HTMLInputElement} */ -const inputLbUrl = $("#lb-url"); -/** @type {HTMLElement} */ -const connField = $("#conn-field"); -/** @type {HTMLElement} */ -const connHint = $("#conn-hint"); -/** @type {HTMLSelectElement} */ -const inputVoice = $("#voice"); -/** @type {HTMLInputElement} */ -const inputVoiceUpload = $("#voice-upload"); -/** @type {HTMLElement} */ -const voiceUploadBtn = $("#voice-upload-btn"); -/** @type {HTMLTextAreaElement} */ -const inputInstructions = $("#instructions"); - -// Voice upload: clicking the Upload button opens the file picker -voiceUploadBtn.addEventListener("click", () => inputVoiceUpload.click()); -voiceUploadBtn.addEventListener("keydown", (e) => { - if (e.key === "Enter" || e.key === " ") inputVoiceUpload.click(); -}); -// After selecting a file, upload it and set as current voice -inputVoiceUpload.addEventListener("change", async () => { - const file = inputVoiceUpload.files?.[0]; - if (!file) return; - try { - voiceUploadBtn.textContent = "⏳ Uploading..."; - const form = new FormData(); - form.append("file", file); - const resp = await fetch("/api/upload_voice", { method: "POST", body: form }); - if (!resp.ok) throw new Error(`Upload failed: ${resp.status}`); - const data = await resp.json(); - // Add uploaded voice as a selectable option - const opt = document.createElement("option"); - opt.value = data.path; - opt.textContent = `📁 ${data.filename}`; - inputVoice.insertBefore(opt, inputVoice.firstChild); - inputVoice.value = data.path; - voiceUploadBtn.textContent = "✓ Uploaded"; - setCaption(`Voice sample "${data.filename}" uploaded — restart to apply`, "muted"); - } catch (err) { - setCaption(`Upload error: ${err.message}`, "error"); - voiceUploadBtn.textContent = "📤 Upload"; - } - inputVoiceUpload.value = ""; - setTimeout(() => { voiceUploadBtn.textContent = "📤 Upload"; }, 3000); -}); -/** @type {HTMLInputElement} */ -const inputNoiseGate = $("#noise-gate"); -/** @type {HTMLElement} */ -const gateValue = $("#gate-value"); -/** @type {HTMLElement} */ -const gateMeterFill = $("#gate-meter-fill"); -/** @type {HTMLElement} */ -const micGate = $("#mic-gate"); -const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc")); -const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track")); -const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill")); -const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit")); -const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle")); - -/** @type {HTMLButtonElement} */ -const restartBtn = $("#restart-conversation"); -/** @type {HTMLElement} */ -const restartHint = $("#restart-hint"); -const settingsForm = /** @type {HTMLFormElement} */ (settingsModal.querySelector("form")); - -/** @type {AppState} */ -let currentState = "idle"; -let settings = loadSettings(); - -// ── Connection target ──────────────────────────────────────────────────────── -// Three modes, decided by the deploy via /api/config: -// • SPEECH_TO_SPEECH_URL set -> direct mode pinned by the deploy: the browser -// connects straight to that URL, shown read-only in Settings. Overrides the -// load balancer entirely. -// • LOAD_BALANCER_URL set -> original flow: POST the same-origin /api/session -// proxy (the server forwards to the LB; the LB address is never sent here). -// • neither (allowDirect) -> the user sets a speech-to-speech server URL and -// the browser connects to it directly (no load balancer, no /session). -let lbMode = false; -// Fail open: direct entry is allowed unless /api/config reports an LB URL. This -// way a missing/unreachable config (e.g. static hosting) leaves the field -// usable rather than locked. -let allowDirect = true; -// Deploy-pinned s2s URL (SPEECH_TO_SPEECH_URL). Non-empty -> locked direct -// mode: the field displays it read-only and the saved user URL is untouched. -let pinnedUrl = ""; - -// ── Tool state ────────────────────────────────────────────────────────────── -let toolsEnabled = loadTools(); -// Whether the server holds a Serper key (learned from /api/config on load). -let serverSearchKey = false; -// A user-supplied key (fallback when the deploy has none). localStorage only. -let userSearchKey = localStorage.getItem(STORAGE_KEYS.searchKey) || ""; -/** @type {MediaStream | null} */ -let cameraStream = null; - -/** Search is usable if the server has a key or the user supplied one. */ -function searchAvailable() { - return serverSearchKey || !!userSearchKey; -} - -/** Tool definitions for the currently-enabled (and usable) tools. */ -function activeToolDefs() { - const defs = []; - if (toolsEnabled.web_search && searchAvailable()) defs.push(TOOL_DEFS.web_search); - if (toolsEnabled.camera_snapshot) defs.push(TOOL_DEFS.camera_snapshot); - return defs; -} - -/** Instructions plus the hidden tool-use hint when any tool is active. */ -function effectiveInstructions() { - const base = settings.instructions; - return activeToolDefs().length ? base + TOOL_USE_HINT : base; -} - -/** Push the active tool set to a live session so toggles apply mid-call. */ -function pushToolsToSession() { - if (!client || !LIVE_STATES.has(currentState)) return; - client.setTools(activeToolDefs()); - // The hidden tool-use hint depends on whether any tool is active, so refresh - // instructions alongside the tool set. - client.updateSession({ instructions: effectiveInstructions() }); -} - -// ── Chat view ─────────────────────────────────────────────────────────────── -// Owns the history panel, the ephemeral bubbles, and all transcript/tool -// streaming state. The client's events are forwarded to its on* methods. -const chat = new ChatView(); - -// ── Account / limiter ───────────────────────────────────────────────────── -// Login chip + daily-limit modal (inert unless the deploy is in LB mode). The -// server meters conversation time; the client just heartbeats a live session -// and tears down when the server reports the budget is spent. -const account = new Account(); -let limiterOn = false; -let heartbeatTimer = 0; -let trackedSessionId = ""; -let trackedTier = ""; -// The waiting-queue ticket id while we're in line (else ""). Used to leave the -// queue on teardown / tab-close so we don't hold a phantom place. -let queuedTicketId = ""; - -/** @type {S2sWsRealtimeClient | null} */ -let client = null; -/** @type {MediaStream | null} */ -let micStream = null; -let micMuted = false; - -/** @param {AppState} next */ -function setState(next) { - currentState = next; - const view = STATE_VIEWS[next]; - circleBtn.disabled = view.disabled; - circleBtn.className = `circle ${STATE_CLASS[next]}`; - if (next !== "error") setCaption(view.caption); - - const live = LIVE_STATES.has(next); - orbWrap.classList.toggle("live", live); - micBtn.setAttribute("aria-hidden", live ? "false" : "true"); - stopBtn.setAttribute("aria-hidden", live ? "false" : "true"); - micBtn.tabIndex = live ? 0 : -1; - stopBtn.tabIndex = live ? 0 : -1; - - // Queue affordances: "Leave queue" whenever we're in line; "Join now" only once - // it's our turn (a slot is held for us). Both live under #queue-actions. - const yourTurn = next === "your-turn"; - const inLine = next === "queued" || yourTurn; - queueActions.hidden = !inLine; - joinQueueBtn.hidden = !yourTurn; - joinQueueBtn.tabIndex = yourTurn ? 0 : -1; - leaveQueueBtn.hidden = !inLine; - leaveQueueBtn.tabIndex = inLine ? 0 : -1; - if (!yourTurn) stopJoinCountdown(); - - // Warm reassurance under the terse position, only while waiting in line. - if (next === "queued") { - circleSubcaption.textContent = - "Sorry, we overhugged! 🤗 Every slot is busy, so we saved you a spot. Hang tight, you're moving up."; - circleSubcaption.hidden = false; - } else { - circleSubcaption.hidden = true; - } - - updateRestartAvailability(); -} - -function updateRestartAvailability() { - // Restart works from any settled state — it tears down a live call (if any) - // and reconnects with the current settings. Only block while mid-connect or - // while waiting in the queue (restarting from there would just re-queue). - restartBtn.disabled = - currentState === "connecting" || currentState === "queued" || currentState === "your-turn"; - restartHint.hidden = false; - restartHint.textContent = LIVE_STATES.has(currentState) - ? "Reconnects now with the settings above." - : "Starts a conversation with the settings above."; -} - -/** - * @param {string} text - * @param {"" | "error" | "muted"} [kind] - */ -function setCaption(text, kind = "") { - const trimmed = text.trim(); - circleCaption.textContent = trimmed; - circleCaption.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; -} - -function openSettings() { - syncConnectionUi(); - inputVoice.value = settings.voice; - inputInstructions.value = settings.instructions; - syncGateUi(); - updateRestartAvailability(); - settingsModal.showModal(); -} - -/** dB position (clamped to the slider axis) as a 0..1 fraction of the track. - * @param {number} db */ -function dbToFraction(db) { - const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); - return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); -} - -/** @param {number} f @returns {number} dB at a 0..1 position on the gate axis. */ -function fractionToDb(f) { - const clamped = Math.min(1, Math.max(0, f)); - return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB)); -} - -// ── Radial gate arc (around the mic button, live during a call) ───────────── -// A 270° arc with the gap facing the orb (right). Fraction 0 (=Off) sits at the -// bottom-ish start; 1 (=max) at the top-ish end. The level fill and the -// threshold handle ride this same axis, mirroring the Settings widget. -const ARC_R = 40; -// A ~200° arc centred on the left (180°) so the wide gap faces the orb (right). -const ARC_SPAN_DEG = 200; -const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; // lower-left start; Off end - -/** Point at fraction f (0..1) and radius r, in the 0..100 viewBox. - * @param {number} f @param {number} [r] */ -function arcPoint(f, r = ARC_R) { - const deg = ARC_START_DEG + f * ARC_SPAN_DEG; - const rad = (deg * Math.PI) / 180; - return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; -} - -/** SVG path `d` for the full 0..1 arc (clockwise). */ -function fullArcD() { - const a = arcPoint(0); - const b = arcPoint(1); - const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0; - return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`; -} - -/** One-time geometry: track, fill (dash-revealed) and the transparent hit band. */ -function initGateArc() { - const d = fullArcD(); - mgaTrack.setAttribute("d", d); - mgaFill.setAttribute("d", d); - mgaHit.setAttribute("d", d); - // pathLength 100 lets us reveal the fill by fraction via dashoffset. - mgaFill.setAttribute("pathLength", "100"); - mgaFill.style.strokeDasharray = "100 100"; - mgaFill.style.strokeDashoffset = "100"; // empty until levels arrive - renderGateHandle(); -} - -/** Place the threshold bead on the arc at the stored threshold; flag off state. */ -function renderGateHandle() { - const off = settings.noiseGate <= GATE_OFF_DB; - const p = arcPoint(dbToFraction(settings.noiseGate)); - mgaHandle.setAttribute("cx", String(p.x)); - mgaHandle.setAttribute("cy", String(p.y)); - micGate.classList.toggle("gate-off", off); -} - -/** Paint a 0..1 live level onto the arc fill (and the Settings meter if open). - * Brightens the tick when the level crosses the threshold — i.e. the gate is - * actually open — but only when gating is enabled. - * @param {number} rms */ -function paintInputLevel(rms) { - const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; - const f = dbToFraction(db); - mgaFill.style.strokeDashoffset = String(100 * (1 - f)); - if (settingsModal.open) gateMeterFill.style.width = `${f * 100}%`; - const enabled = settings.noiseGate > GATE_OFF_DB; - micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(settings.noiseGate)); -} - -/** The single place that commits a new gate threshold: updates both controls, - * persists, and applies live to the running session. - * @param {number} db */ -function setGateThreshold(db) { - settings.noiseGate = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db))); - const off = settings.noiseGate <= GATE_OFF_DB; - inputNoiseGate.value = String(settings.noiseGate); - gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; - renderGateHandle(); - localStorage.setItem(STORAGE_KEYS.noiseGate, String(settings.noiseGate)); - if (client && LIVE_STATES.has(currentState)) { - client.setNoiseGate(gateParams(settings.noiseGate)); - } -} - -/** Reflect the stored gate threshold into the slider, label and arc handle. */ -function syncGateUi() { - inputNoiseGate.value = String(settings.noiseGate); - const off = settings.noiseGate <= GATE_OFF_DB; - gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; - renderGateHandle(); -} - -// Drag along the arc band to set the threshold (a tap on the glyph still mutes). -let gateDragging = false; -/** @param {PointerEvent} e */ -function gatePointerToDb(e) { - const rect = mgaArc.getBoundingClientRect(); - const cx = rect.left + rect.width / 2; - const cy = rect.top + rect.height / 2; - let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI; - if (deg < 0) deg += 360; - // Map the on-arc angle to a fraction; angles in the right-side gap fall - // outside [0,1] and fractionToDb clamps them to the nearest end (just-below - // start -> Off, just-past end -> max). - const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG; - return fractionToDb(f); -} -mgaHit.addEventListener("pointerdown", (e) => { - gateDragging = true; - mgaHit.setPointerCapture(e.pointerId); - setGateThreshold(gatePointerToDb(e)); -}); -mgaHit.addEventListener("pointermove", (e) => { - if (gateDragging) setGateThreshold(gatePointerToDb(e)); -}); -const endGateDrag = (/** @type {PointerEvent} */ e) => { - if (!gateDragging) return; - gateDragging = false; - try { mgaHit.releasePointerCapture(e.pointerId); } catch {} -}; -mgaHit.addEventListener("pointerup", endGateDrag); -mgaHit.addEventListener("pointercancel", endGateDrag); - -settingsBtn.addEventListener("click", openSettings); - -// About panel: native , Esc closes for free; also close on the X and -// on a click in the backdrop (a click whose target is the dialog itself). -$("#about-btn-m").addEventListener("click", () => aboutModal.showModal()); -aboutClose.addEventListener("click", () => aboutModal.close()); -aboutModal.addEventListener("click", (e) => { - if (e.target === aboutModal) aboutModal.close(); -}); - -// ── Tools panel ─────────────────────────────────────────────────────────── - -/** Reflect the current tool state into the panel controls. */ -function syncToolsUi() { - const avail = searchAvailable(); - toolWebSwitch.checked = toolsEnabled.web_search && avail; - toolWebSwitch.disabled = !avail; - toolWebRow.classList.toggle("disabled", !avail); - toolCamSwitch.checked = toolsEnabled.camera_snapshot; - - if (serverSearchKey) { - // Key lives server-side: show it as configured, never expose it. - searchKeyInput.value = ""; - searchKeyInput.placeholder = "•••••••• · provided by the server"; - searchKeyInput.disabled = true; - toolWebHint.textContent = "Ready. The search key is held server-side and never sent to your browser."; - } else { - searchKeyInput.disabled = false; - searchKeyInput.value = userSearchKey; - searchKeyInput.placeholder = "Paste a Serper key to enable web search"; - toolWebHint.textContent = userSearchKey - ? "Using your key — stored in this browser only." - : "No server key configured. Add your own Serper key to enable web search."; - } -} - -toolsBtn.addEventListener("click", () => { syncToolsUi(); toolsModal.showModal(); }); -toolsClose.addEventListener("click", () => toolsModal.close()); -toolsModal.addEventListener("click", (e) => { - if (e.target === toolsModal) toolsModal.close(); -}); - -toolWebSwitch.addEventListener("change", () => { - if (toolWebSwitch.checked && !searchAvailable()) { - toolWebSwitch.checked = false; // guard: can't enable without a key - return; - } - toolsEnabled.web_search = toolWebSwitch.checked; - saveTools(); - pushToolsToSession(); -}); - -toolCamSwitch.addEventListener("change", async () => { - if (toolCamSwitch.checked) { - try { - // Flipping the switch always re-requests the camera, so a permission that - // was only dismissed earlier is asked again here. - await enableCamera(); - } catch (err) { - toolCamSwitch.checked = false; - const denied = err instanceof Error && (err.name === "NotAllowedError" || err.name === "SecurityError"); - toolCamHint.textContent = denied - ? "Camera blocked. Allow it from the camera icon in your browser's address bar — it switches on automatically." - : `Camera unavailable${err instanceof Error ? `: ${err.message}` : ""}`; - return; - } - toolsEnabled.camera_snapshot = true; - toolCamHint.textContent = "Camera on. The assistant can take a snapshot when it needs to see."; - } else { - disableCamera(); - toolsEnabled.camera_snapshot = false; - toolCamHint.textContent = "Let the assistant see through your webcam."; - } - saveTools(); - pushToolsToSession(); -}); - -searchKeyInput.addEventListener("input", () => { - if (serverSearchKey) return; - userSearchKey = searchKeyInput.value.trim(); - if (userSearchKey) localStorage.setItem(STORAGE_KEYS.searchKey, userSearchKey); - else localStorage.removeItem(STORAGE_KEYS.searchKey); - - const avail = searchAvailable(); - toolWebSwitch.disabled = !avail; - toolWebRow.classList.toggle("disabled", !avail); - // Losing the key disables a previously-enabled tool. - if (!avail && toolsEnabled.web_search) { - toolsEnabled.web_search = false; - toolWebSwitch.checked = false; - saveTools(); - pushToolsToSession(); - } - toolWebHint.textContent = userSearchKey - ? "Using your key — stored in this browser only." - : "No server key configured. Add your own Serper key to enable web search."; -}); - -// ── Camera ────────────────────────────────────────────────────────────────── - -async function enableCamera() { - if (cameraStream) return; - cameraStream = await navigator.mediaDevices.getUserMedia({ - video: { facingMode: "user" }, - audio: false, - }); - camVideo.srcObject = cameraStream; - try { await camVideo.play(); } catch { /* autoplay quirks; muted video is fine */ } - camPip.classList.add("visible"); - camPip.setAttribute("aria-hidden", "false"); - // Lets the footer reflow to the bottom-right (and hide on mobile) while the - // webcam preview occupies the bottom of the stage. - document.body.classList.add("cam-on"); -} - -function disableCamera() { - if (cameraStream) { - for (const t of cameraStream.getTracks()) t.stop(); - cameraStream = null; - } - camVideo.srcObject = null; - camPip.classList.remove("visible"); - camPip.setAttribute("aria-hidden", "true"); - document.body.classList.remove("cam-on"); -} - -/** Auto-start the webcam on arrival (the camera tool is on by default). If the - * user declines the permission, switch the tool off and reflect it in the UI - * rather than nagging. */ -async function autoStartCamera() { - if (!toolsEnabled.camera_snapshot || cameraStream) return; - try { - await enableCamera(); - } catch (err) { - console.warn("[main] camera auto-start declined/failed:", err); - toolsEnabled.camera_snapshot = false; - saveTools(); - syncToolsUi(); - } -} - -/** Track the browser's camera permission so a later re-grant (e.g. the user - * unblocks it from the address bar after a denial) turns the camera back on - * without another toggle, and a revoke turns it off. Best-effort: the - * Permissions API doesn't support "camera" everywhere (e.g. Safari). */ -async function watchCameraPermission() { - try { - const status = await navigator.permissions?.query?.({ name: /** @type {any} */ ("camera") }); - if (!status) return; - status.addEventListener("change", () => { - if (status.state === "granted") { - if (!toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = true; saveTools(); } - void autoStartCamera(); - syncToolsUi(); - } else if (status.state === "denied") { - disableCamera(); - if (toolsEnabled.camera_snapshot) { toolsEnabled.camera_snapshot = false; saveTools(); } - syncToolsUi(); - } - }); - } catch { - // Permissions API unavailable for "camera" — the toggle still re-asks. - } -} - -/** - * Grab the current webcam frame as a downscaled JPEG data URL. The preview is - * mirrored in CSS for a natural self-view, but we draw the raw (un-mirrored) - * video here so the model sees the scene in its true orientation. - * @returns {string | null} - */ -function captureSnapshot() { - if (!cameraStream || !camVideo.videoWidth) return null; - const vw = camVideo.videoWidth; - const vh = camVideo.videoHeight; - const scale = Math.min(1, SNAPSHOT_MAX_EDGE / Math.max(vw, vh)); - const w = Math.max(1, Math.round(vw * scale)); - const h = Math.max(1, Math.round(vh * scale)); - const canvas = document.createElement("canvas"); - canvas.width = w; - canvas.height = h; - const ctx = canvas.getContext("2d"); - if (!ctx) return null; - ctx.drawImage(camVideo, 0, 0, w, h); - return canvas.toDataURL("image/jpeg", SNAPSHOT_QUALITY); -} - -/** Brief shutter flash on the preview so the user sees a snapshot was taken. */ -function flashPreview() { - camPip.classList.remove("flash"); - void camPip.offsetWidth; // reflow so the animation restarts - camPip.classList.add("flash"); -} - -// ── Tool executor ───────────────────────────────────────────────────────── -// Runs the function the model called, returns the result, and asks for a -// response so the model speaks it. Errors come back as the tool output too, so -// the model can recover gracefully instead of the turn stalling. - -/** - * Run the function the model called, return its result to the backend, and ask - * for a follow-up response. We also hand the result back to the caller so it - * can be shown in the conversation once the tool has actually run. - * @param {string} name @param {string} argsJson @param {string} callId - * @returns {Promise<{ output: string, image?: string }>} - */ -async function runTool(name, argsJson, callId) { - if (!client) return { output: "" }; - let args = /** @type {Record} */ ({}); - try { args = JSON.parse(argsJson || "{}"); } catch { /* keep {} */ } - - if (DEBUG) console.debug(`[tool] run name=${name} callId=${JSON.stringify(callId)} args=${argsJson}`); - if (!callId) console.warn("[tool] empty call_id — the backend didn't tag the call, can't return a function_call_output"); - - /** @type {{ output: string, image?: string }} */ - let result = { output: "" }; - try { - if (name === "web_search") { - const query = typeof args.query === "string" ? args.query : ""; - result.output = await execWebSearch(query); - // Return the result and let the bare response.create (below) trigger the - // spoken answer. - client.sendToolOutput(callId, result.output); - } else if (name === "camera_snapshot") { - const dataUrl = captureSnapshot(); - if (dataUrl) { - if (DEBUG) console.debug(`[tool] camera_snapshot captured frame (${dataUrl.length} chars), sending image + output`); - result = { output: "Snapshot captured from the webcam and attached as an image.", image: dataUrl }; - // Return the tool output; the frame itself rides along with the - // response.create below (sent right before it), so the model sees the - // snapshot in the very response it's about to speak. - client.sendToolOutput(callId, result.output); - flashPreview(); - } else { - console.warn("[tool] camera_snapshot: no frame — camera off or not ready"); - result.output = "The camera is not available right now."; - client.sendToolOutput(callId, result.output); - } - } else { - result.output = `Unknown tool: ${name}`; - client.sendToolOutput(callId, result.output); - } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - result.output = `Tool failed: ${msg}`; - client.sendToolOutput(callId, result.output); - } - if (DEBUG) console.debug(`[tool] requesting model response after ${name}`); - // Camera: the captured frame rides with the response.create (sent just before - // it) so it's in context for the reply. Other tools: a bare create. - client.requestResponse(result.image ? { image: result.image } : undefined); - return result; -} - -/** @param {string} query @returns {Promise} */ -async function execWebSearch(query) { - if (!query) return "No query provided."; - /** @type {Record} */ - const body = { query }; - // Only send a user key when there's no server key (server prefers its own). - if (!serverSearchKey && userSearchKey) body.key = userSearchKey; - - const res = await fetch("api/search", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(body), - }); - if (!res.ok) { - let detail = String(res.status); - try { const j = await res.json(); if (j.detail) detail = j.detail; } catch {} - throw new Error(`search error (${detail})`); - } - const json = await res.json(); - // Date-stamp the header so the model treats these as fresh realtime facts - // rather than its (older) training knowledge. - const today = new Date().toISOString().slice(0, 10); - /** @type {string[]} */ - const lines = [`Google search result from ${today}:`]; - if (json.answer) lines.push(`Answer: ${json.answer}`); - for (const r of json.results || []) { - lines.push(`- ${r.title}: ${r.snippet} (${r.url})`); - } - return lines.length > 1 ? lines.join("\n") : `${lines[0]}\nNo results found.`; -} - -/** Learn server config (search key + connection target), then refresh the UI. */ -async function fetchConfig() { - try { - const res = await fetch("api/config"); - if (res.ok) { - const json = await res.json(); - serverSearchKey = !!json.search; - lbMode = !!json.lb; - // Lock to LB mode only when the deploy reports a load balancer. - allowDirect = json.allowDirect ?? !lbMode; - // Deploy-pinned direct URL (overrides the LB server-side already). - pinnedUrl = (json.s2sUrl || "").trim(); - // The conversation-time limiter rides on the LB being present. - limiterOn = lbMode; - } - // Non-OK response: leave the fail-open default (allowDirect = true). - } catch { - // Config endpoint unreachable (e.g. static hosting): keep direct entry. - } - if (DEBUG) console.debug(`[ui] config: allowDirect=${allowDirect} lbMode=${lbMode}`); - // Login chip + remaining-budget (no-op / hidden when the limiter is off). - void account.refresh(); - syncToolsUi(); - syncConnectionUi(); -} - -/** - * Resolve where to connect, per the deploy's mode: - * • LB mode -> `{ sessionUrl }`, the client POSTs the same-origin /api/session - * proxy and the server forwards to the LB (its address stays server-side). - * • direct -> `{ directUrl }`, connect straight to the s2s WebSocket. - * Throws a user-facing error if direct mode is on but no URL was entered. - * @returns {{ sessionUrl: string } | { directUrl: string }} - */ -function connectionTarget() { - if (!allowDirect) { - return { sessionUrl: "api/session" }; - } - const directUrl = buildDirectWsUrl(window.VOICE_WS_URL || pinnedUrl || settings.directUrl); - if (!directUrl) { - throw new Error("Enter a speech-to-speech server URL in Settings."); - } - return { directUrl }; -} - -/** - * Normalise a user-typed server address into a realtime WebSocket URL. - * Accepts bare hosts (`localhost:8080`), http(s) URLs, or ws(s) URLs, and adds - * the `/v1/realtime` path when none is given. A full connect URL (with path - * and/or query) is preserved as-is. - * @param {string} raw @returns {string} - */ -function buildDirectWsUrl(raw) { - let s = (raw || "").trim(); - if (!s) return ""; - if (!/^wss?:\/\//i.test(s)) { - if (/^https?:\/\//i.test(s)) { - s = s.replace(/^http/i, "ws"); // http→ws, https→wss - } else { - const isLocal = /^(localhost|127\.0\.0\.1|\[::1\])(:|\/|$)/i.test(s); - s = (isLocal ? "ws://" : "wss://") + s; - } - } - try { - const u = new URL(s); - if (u.pathname === "" || u.pathname === "/") u.pathname = "/v1/realtime"; - return u.toString(); - } catch { - return s; - } -} - -/** Create + resume an AudioContext synchronously (must run inside the user - * gesture so iOS lets it start). Returns null if construction fails. */ -function createResumedAudioContext() { - try { - const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext; - const ctx = new Ctx({ latencyHint: "interactive" }); - if (ctx.state === "suspended") void ctx.resume().catch(() => {}); - return /** @type {AudioContext} */ (ctx); - } catch (err) { - console.warn("[main] AudioContext init failed:", err); - return null; - } -} - -/** Read the editable settings out of the form. The URL field is only honoured - * in free direct mode — in LB mode it's hidden, and when the deploy pins a - * URL it's read-only, so the user's saved URL survives either way. */ -function readSettingsFromForm() { - return { - directUrl: allowDirect && !pinnedUrl ? inputLbUrl.value.trim() : settings.directUrl, - voice: inputVoice.value || DEFAULT_VOICE, - instructions: inputInstructions.value.trim() || DEFAULT_INSTRUCTIONS, - noiseGate: readGateThreshold(), - }; -} - -/** Gate threshold (dBFS) currently shown on the slider, clamped to range. */ -function readGateThreshold() { - const v = Math.round(Number(inputNoiseGate.value)); - if (!Number.isFinite(v)) return GATE_OFF_DB; - return Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, v)); -} - -/** Adapt the connection field to the mode learned from /api/config. */ -function syncConnectionUi() { - if (pinnedUrl) { - // Deploy-pinned URL: show it, but locked — the deployment owns it. - connField.hidden = false; - inputLbUrl.value = pinnedUrl; - inputLbUrl.readOnly = true; - connHint.classList.remove("error"); - connHint.textContent = "Speech-to-speech server URL pinned by this deployment."; - } else if (allowDirect) { - // Direct mode: the user sets their own s2s server URL. - connField.hidden = false; - inputLbUrl.value = settings.directUrl; - inputLbUrl.readOnly = false; - inputLbUrl.placeholder = "http://localhost:port"; - connHint.classList.remove("error"); - connHint.textContent = - "URL of your speech-to-speech server, e.g. http://localhost:8080 (the app adds /v1/realtime)."; - } else { - // LB mode: the load balancer URL is deployment-owned — hide it entirely so - // its address is never exposed in Settings. - connField.hidden = true; - } -} - -/** True when the user must supply a server URL before connecting (direct mode - * with nothing set). */ -function missingServerUrl() { - return allowDirect && !pinnedUrl && !buildDirectWsUrl(settings.directUrl); -} - -/** Open Settings and point the user at the empty server-URL field. */ -function promptServerUrl() { - if (settingsModal.open) syncConnectionUi(); - else openSettings(); - connHint.textContent = "Set the speech-to-speech server URL to start."; - connHint.classList.add("error"); - inputLbUrl.focus(); -} - -settingsForm.addEventListener("submit", (event) => { - const submitter = /** @type {HTMLButtonElement | null} */ ((/** @type {SubmitEvent} */ (event)).submitter); - if (submitter?.value !== "save") return; - - settings = readSettingsFromForm(); - saveSettings(settings); - - // Voice + instructions can apply to a live session without reconnecting; a - // changed connection URL only takes effect on the next restart. - if (client && LIVE_STATES.has(currentState)) { - client.updateSession({ voice: settings.voice, instructions: effectiveInstructions() }); - } -}); - -// The noise gate applies live (worklet param), so tune it without a restart: -// update the label/marker, persist, and push straight to the running client. -inputNoiseGate.addEventListener("input", () => { - setGateThreshold(readGateThreshold()); -}); - -restartBtn.addEventListener("click", async () => { - if (currentState === "connecting") return; // a connect is already underway - settings = readSettingsFromForm(); - saveSettings(settings); - if (missingServerUrl()) { promptServerUrl(); return; } // keep settings open - settingsModal.close(); - // Grab the AudioContext NOW, inside the click gesture — teardown() awaits, and - // creating it afterwards would fall outside the gesture (silent on iOS). - const audioContext = createResumedAudioContext(); - try { - if (client) await teardown(); - await doStart(audioContext); - } catch (err) { - await handleStartError(err); - } -}); - -circleBtn.addEventListener("click", async () => { - try { - if (currentState === "idle" || currentState === "error") { - if (missingServerUrl()) { promptServerUrl(); return; } - await doStart(); - } - } catch (err) { - await handleStartError(err); - } -}); - -/** A failed start is either the daily limit (show the modal, return to idle) or - * a real fault (surface it). doStart already closed any orphan AudioContext. - * @param {any} err */ -async function handleStartError(err) { - if (err && err.code === "limit") { - await teardown(); - account.showLimit(err.tier); - return; - } - // The user left the queue (close() aborted the wait): teardown already reset - // the UI to idle, so there's nothing to report. - if (err && err.code === "aborted") return; - // The whole waiting line is full: a warm, reassuring modal rather than an error. - if (err && err.code === "queue-full") { - await teardown(); - account.showBusy(); - return; - } - // Our place lapsed (ticket reaped, or the join window ran out). Recoverable, not - // a fault: land on the retry state with a kind, plain-language reason. - if (err && (err.code === "queue-expired" || err.code === "join-expired")) { - await teardown(); - setState("error"); - setCaption( - err.code === "join-expired" - ? "Your spot expired. Tap to rejoin." - : "That took a while. Tap to rejoin.", - "error", - ); - return; - } - onFatalError(err); -} - -micBtn.addEventListener("click", () => { - if (!micStream || !client) return; - micMuted = !micMuted; - for (const track of micStream.getAudioTracks()) { - track.enabled = !micMuted; - } - client.setMuted(micMuted); - micBtn.classList.toggle("muted", micMuted); - micBtn.setAttribute("aria-label", micMuted ? "Unmute" : "Mute"); - micBtn.title = micMuted ? "Unmute" : "Mute"; -}); - -stopBtn.addEventListener("click", async () => { - await teardown(); -}); - -// "Leave queue": tear down the pending connect (aborts the poll wait) and drop -// our place in line. Same teardown path as stopping a live call. -leaveQueueBtn.addEventListener("click", async () => { - await teardown(); -}); - -// "Join now": accept the held slot. The click is a user gesture, so the client -// re-resumes the AudioContext here (iOS) before dialing. -joinQueueBtn.addEventListener("click", () => { - stopJoinCountdown(); - if (client) client.join(); -}); - -const MIC_CONSTRAINTS = { - audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, -}; - -/** Prompt for mic permission up front, then immediately release the tracks so no - * recording indicator lingers during a queue wait. Throws a friendly error if the - * user denies. */ -async function primeMicPermission() { - try { - const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); - for (const track of s.getTracks()) track.stop(); - } catch (err) { - throw new Error( - `Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`, - ); - } -} - -/** Acquire the live capture stream once a slot is granted. Permission was primed - * in the tap gesture, so this is silent. Stored module-side for mute + teardown. */ -async function acquireMicStream() { - micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); - return micStream; -} - -/** @param {number} position Update the queued caption ("You're #N in line"). */ -function onQueuePosition(position) { - const n = Number(position) || 0; - setCaption(n > 0 ? `You're #${n} in line` : "Finding you a spot…", "muted"); -} - -// ── "Your turn" join countdown ────────────────────────────────────────────── -// While a slot is held for us, show how long is left to accept it. The client's -// join gate expires just before the load balancer reclaims the slot. -let joinCountdownTimer = 0; - -/** @param {number} sec */ -function startJoinCountdown(sec) { - stopJoinCountdown(); - let left = Math.max(0, Math.floor(sec)); - const paint = () => { - joinQueueBtn.textContent = left > 0 ? `Join now (${left}s)` : "Join now"; - }; - paint(); - joinCountdownTimer = window.setInterval(() => { - left -= 1; - if (left <= 0) { - stopJoinCountdown(); - joinQueueBtn.textContent = "Join now"; - return; - } - paint(); - }, 1000); -} - -function stopJoinCountdown() { - if (joinCountdownTimer) { - clearInterval(joinCountdownTimer); - joinCountdownTimer = 0; - } -} - -/** - * Start a conversation. Pass a pre-created AudioContext when the caller already - * made one inside the tap/click gesture (required on iOS); otherwise one is - * created here, which is still inside the gesture for a direct orb tap. - * @param {AudioContext | null} [audioContext] - */ -async function doStart(audioContext = null) { - // Resolve the target before touching mic/audio so a misconfiguration (e.g. - // direct mode with no URL) fails fast with a clear message. - const target = connectionTarget(); - - chat.clear(); - chat.reset(); - setState("connecting"); - setCaption("Asking for mic…", "muted"); - - // Create + resume the AudioContext SYNCHRONOUSLY, still inside the gesture. - // iOS Safari only starts an AudioContext from a user gesture; if we waited - // until after the getUserMedia / session-creation awaits below, it would stay - // suspended and the whole pipeline would be silent. - if (!audioContext) audioContext = createResumedAudioContext(); - - // Prime the mic permission now (get the prompt out of the way up front), then - // release it. The real capture stream is acquired only once a slot is granted - // (see acquireMicStream), so the mic 'in use' indicator never lights while we - // sit in the queue. Permission persists, so the later acquire is silent. - try { - await primeMicPermission(); - } catch (err) { - if (audioContext) void audioContext.close().catch(() => {}); - throw err; - } - - // The webcam is started on arrival (autoStartCamera), so nothing to do here; - // a still-pending grant just means the snapshot tool isn't ready yet. - - const c = new S2sWsRealtimeClient({ - ...target, - voice: settings.voice, - instructions: effectiveInstructions(), - acquireMic: acquireMicStream, - tools: activeToolDefs(), - noiseGate: gateParams(settings.noiseGate), - ...(audioContext ? { audioContext } : {}), - }); - client = c; - - c.addEventListener("queue", (e) => { - const { position, queueId } = /** @type {CustomEvent<{ position: number; queueId: string }>} */ (e).detail; - if (queueId) queuedTicketId = queueId; - onQueuePosition(position); - }); - - c.addEventListener("ready-to-join", (e) => { - const { info, expiresSec } = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo; expiresSec: number }>} */ (e).detail; - // A slot is held for us. We're out of the queue now, so drop the ticket ref. - // Track the granted session id already so that leaving (or letting the timer - // lapse) refunds the budget the server reserved at claim, even before we dial. - queuedTicketId = ""; - if (info?.sessionId) { - trackedSessionId = info.sessionId; - trackedTier = info.tier || "anon"; - } - startJoinCountdown(expiresSec); - }); - - c.addEventListener("status", (e) => { - const detail = /** @type {CustomEvent<{ status: string }>} */ (e).detail; - onClientStatus(detail.status); - }); - c.addEventListener("transcript", (e) => { - const d = /** @type {CustomEvent<{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }>} */ (e).detail; - chat.onTranscript(d); - }); - - c.addEventListener("response-finished", (e) => { - const detail = /** @type {CustomEvent<{ responseId: string; status: string; audible?: boolean; transcript?: string }>} */ (e).detail; - chat.onResponseFinished(detail); - }); - - c.addEventListener("toolcall", (e) => { - const { name, arguments: args, callId } = /** @type {CustomEvent<{ name: string; arguments: string; callId: string }>} */ (e).detail; - chat.onToolCall(name); - // Execute the tool, then push it to the conversation once the result is in, - // so the toggle shows both the call input and its output together. - void runTool(name, args, callId).then(({ output, image }) => { - chat.onToolResult(name, args, output, image); - }); - }); - c.addEventListener("error", (e) => { - const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail; - onFatalError(detail.error); - }); - c.addEventListener("server-error", (e) => { - // Non-fatal: the backend reported an error mid-session. Log it, keep the - // socket and the conversation alive (the model can recover on its own). - const detail = /** @type {CustomEvent<{ error: unknown }>} */ (e).detail; - const msg = detail.error instanceof Error ? detail.error.message : String(detail.error); - console.warn("[main] server error (non-fatal):", msg); - }); - c.addEventListener("session", (e) => { - const info = /** @type {CustomEvent<{ info: import("./ws/s2s-ws-client.js").WsSessionInfo }>} */ (e).detail.info; - console.log("[ws] session created:", info.sessionId); - // A slot was granted — we're out of the queue; drop the ticket reference so - // teardown doesn't try to leave a line we already left. - queuedTicketId = ""; - // A metered tier (anon / free): heartbeat so the server can extend the - // reservation and tell us when the daily budget runs out. PRO isn't limited. - if (info.limited && info.sessionId) { - trackedSessionId = info.sessionId; - trackedTier = info.tier || "anon"; - startHeartbeat(info.heartbeatSec || 5); - } - }); - c.addEventListener("input-level", (e) => { - const { rms } = /** @type {CustomEvent<{ rms: number }>} */ (e).detail; - paintInputLevel(rms); - }); - - try { - await c.connect(); - } catch (err) { - // The grant can be refused (402 → limit) or the dial can fail. In LB mode - // the AudioContext hasn't been adopted by the client yet (the session POST - // runs first), so close the one we created here to avoid leaking it. - if (audioContext) void audioContext.close().catch(() => {}); - throw err; - } -} - -// ── Conversation-time heartbeat ───────────────────────────────────────────── - -/** Ping the server every `sec` seconds so it can meter the live session; when - * it reports the daily budget is spent, cut the call and show the limit modal. - * @param {number} sec */ -function startHeartbeat(sec) { - stopHeartbeat(); - heartbeatTimer = window.setInterval(async () => { - if (!trackedSessionId) return; - try { - const res = await fetch("api/session/heartbeat", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ sessionId: trackedSessionId }), - keepalive: true, - }); - const json = await res.json().catch(() => ({})); - if (json.expired) await onLimitReached(); - } catch (err) { - // A transient network blip shouldn't kill the call; the next tick retries. - if (DEBUG) console.debug("[ui] heartbeat failed:", err); - } - }, Math.max(1, sec) * 1000); -} - -function stopHeartbeat() { - if (heartbeatTimer) { - clearInterval(heartbeatTimer); - heartbeatTimer = 0; - } -} - -/** The server cut the live session: tear down and explain why. */ -async function onLimitReached() { - const tier = trackedTier; - stopHeartbeat(); - await teardown(); - account.showLimit(tier); -} - -/** Tell the server a session ended so it reconciles + refunds the unused chunk. - * Uses sendBeacon so it still fires when the tab is closing. */ -function endTrackedSession() { - if (!trackedSessionId) return; - const body = JSON.stringify({ sessionId: trackedSessionId }); - try { - const blob = new Blob([body], { type: "application/json" }); - if (!navigator.sendBeacon("api/session/end", blob)) { - void fetch("api/session/end", { - method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, - }).catch(() => {}); - } - } catch { - // Best-effort; the server sweep reaps the session anyway. - } - trackedSessionId = ""; - trackedTier = ""; -} - -/** Leave the waiting queue so the LB frees our place. sendBeacon so it still - * fires on tab close; the LB also reaps the ticket on TTL as a backstop. */ -function endQueueTicket() { - if (!queuedTicketId) return; - const body = JSON.stringify({ queueId: queuedTicketId }); - try { - const blob = new Blob([body], { type: "application/json" }); - if (!navigator.sendBeacon("api/queue/end", blob)) { - void fetch("api/queue/end", { - method: "POST", headers: { "Content-Type": "application/json" }, body, keepalive: true, - }).catch(() => {}); - } - } catch { - // Best-effort; the LB reaps the ticket on TTL anyway. - } - queuedTicketId = ""; -} - -/** @param {string} status */ -function onClientStatus(status) { - switch (status) { - case "creating-session": - case "connecting": - setState("connecting"); - break; - case "queued": - setState("queued"); - break; - case "your-turn": - setState("your-turn"); - break; - case "connected": - setState("listening"); - break; - case "user-speaking": - setState("user-speaking"); - break; - case "processing": - setState("processing"); - break; - case "ai-speaking": - setState("ai-speaking"); - break; - case "closed": - // teardown() will move us to idle - break; - case "error": - setState("error"); - break; - } -} - -async function teardown() { - stopHeartbeat(); - stopJoinCountdown(); - endTrackedSession(); - endQueueTicket(); - chat.reset({ dismiss: true }); - if (client) { - try { - await client.close(); - } catch (err) { - console.warn("[main] error closing client:", err); - } - client = null; - } - if (micStream) { - for (const track of micStream.getTracks()) track.stop(); - micStream = null; - } - // The webcam is independent of the call lifecycle (it runs while the user is - // on the page), so we leave it on here — only the camera toggle stops it. - micMuted = false; - micBtn.classList.remove("muted"); - setState("idle"); - // Refresh the chip's remaining-today after the budget moved. - if (limiterOn) void account.refresh(); -} - -/** @param {unknown} err */ -function onFatalError(err) { - console.error("[main] fatal:", err); - setState("error"); - const message = err instanceof Error ? err.message : String(err); - setCaption(truncateError(message), "error"); - void teardown().catch(() => { - setState("error"); - setCaption(truncateError(message), "error"); - }); -} - -setState("idle"); -chat.renderEmptyState(); -initGateArc(); -void fetchConfig(); - -// ── Text input (set up early, before camera/permission calls) ─────────────── -try { - const textInput = /** @type {HTMLInputElement} */ ($("#text-input")); - const sendBtn = /** @type {HTMLButtonElement} */ ($("#text-send-btn")); - if (textInput && sendBtn) { - const doSendText = (/** @type {string} */ text) => { - if (!text) return; - if (!client) { - setCaption("Start a conversation first", ""); - return; - } - client.sendUserText(text); - chat.onTranscript({ role: "user", text, partial: false }); - textInput.value = ""; - }; - const onTextSend = () => doSendText(textInput.value.trim()); - textInput.addEventListener("keydown", (e) => { - if (e.key === "Enter") { - e.preventDefault(); - onTextSend(); - } - }); - sendBtn.addEventListener("click", onTextSend); - } else { - console.error("[main] text input elements not found"); - } -} catch (err) { - console.error("[main] text input setup failed:", err); -} - -// Start the webcam as soon as the user lands (camera tool defaults on), and -// react to later permission changes (re-grant after a denial re-enables it). -void autoStartCamera(); -void watchCameraPermission(); - -// Reconcile a live session if the tab is closed/hidden mid-call (no teardown). -window.addEventListener("pagehide", () => { endTrackedSession(); endQueueTicket(); }); - -requestAnimationFrame(() => { - document.body.classList.remove("booting"); -}); diff --git a/public/voice-demo/style.css b/public/voice-demo/style.css deleted file mode 100644 index 5ecb56c..0000000 --- a/public/voice-demo/style.css +++ /dev/null @@ -1,2773 +0,0 @@ -:root { - --bg: #0a0b10; - --bg-elev: #13151c; - --bg-elev-2: #1b1e29; - --border: rgba(255, 255, 255, 0.08); - --border-strong: rgba(255, 255, 255, 0.16); - --text: #f5f6fa; - --text-dim: rgba(245, 246, 250, 0.65); - --text-faint: rgba(245, 246, 250, 0.42); - - --accent: #8b7dff; - --accent-2: #22d3ee; - --listening: #22d3ee; - --speaking: #8b7dff; - --processing: #f59e0b; - --error: #ff6a75; - --success: #34d399; - - /* Mono is the "machine voice": reserved for system text the app emits — - * the orb's status caption, tool calls, the transport tag. Body/UI stays - * Inter. */ - --font-mono: "Geist Mono", ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace; - - /* Thesis: color belongs to the voice. The chrome is monochrome; saturated - * hue appears only on the orb and on these tiny role echoes, which mirror - * the orb's own state colors (you listening = cyan, assistant = violet, - * tool = amber) so the transcript reads in the same color language. */ - --voice-user: var(--accent-2); - --voice-assistant: var(--accent); - --voice-tool: var(--processing); - - /* Smoothed mic RMS in [0..1], updated every frame from JS while a session - * is active. Used by audio-reactive circle states. */ - --audio-level: 0; - /* Five log-spaced frequency bands extracted from the mic analyser; - * drive each bar's height independently for a real "spectrum" feel. */ - --bar0: 0; - --bar1: 0; - --bar2: 0; - --bar3: 0; - --bar4: 0; - - --radius-sm: 8px; - --radius-md: 14px; - --radius-lg: 22px; - - --shadow-soft: 0 10px 40px rgba(0, 0, 0, 0.35); - - color-scheme: dark; -} - -* { - box-sizing: border-box; -} - -html, -body { - margin: 0; - padding: 0; - height: 100%; -} - -body { - font-family: "Inter", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; - background: radial-gradient(ellipse at 50% 20%, #1a1c28 0%, var(--bg) 60%); - color: var(--text); - min-height: 100vh; - overflow-x: hidden; - -webkit-font-smoothing: antialiased; -} - -button { - font-family: inherit; -} - -a { - color: var(--text); - text-decoration: none; - border-bottom: 1px solid var(--border-strong); -} -a:hover { - border-bottom-color: var(--text); -} - -#app { - position: relative; - z-index: 1; - display: grid; - grid-template-rows: auto 1fr auto; - min-height: 100vh; -} - -.hidden { - display: none !important; -} - -/* ─── Topbar ──────────────────────────────────────────────────────────── */ - -/* Topbar is just a floating row of controls over the stage - no - * separator line, no background. Same story for the footer. Keeps the - * app feeling like one continuous canvas. */ -.topbar { - display: flex; - /* Top-align so the right control cluster sits up with the title instead of - * floating to the vertical middle of the tall identity block. */ - align-items: flex-start; - justify-content: space-between; - padding: 18px 28px; - /* Default stacking (below the conversation panel, z 200) so the panel drawer - * covers the controls when it's open. */ -} - -.brand { - display: flex; - align-items: center; - gap: 10px; - font-weight: 600; - font-size: 14px; - letter-spacing: 0.01em; - color: var(--text-dim); -} - -/* Transport tag rides the wordmark as a system label, not prose — mono, - * dimmed, the only mono in the topbar. */ -.brand-tag { - font-family: var(--font-mono); - font-weight: 500; - font-size: 11px; - letter-spacing: 0.02em; - opacity: 0.55; -} - -.brand .brand-logo { - width: 26px; - height: 26px; - object-fit: contain; - flex: none; - filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.45)); - transition: transform 0.2s ease; -} -.brand .brand-logo:hover { - transform: translateY(-1px) rotate(-2deg); -} - -/* Narrow viewports (typically the mobile shell iframe and phone-sized - browser windows): the full product name pushes the topbar's right - cluster off-screen. Keep the logo as the brand cue and drop the - wordmark - the orb already gives enough context. */ -@media (max-width: 600px) { - /* The full identity stack is too tall for a phone topbar. Show only the - * title + (i); the popup carries the blurb, credits and pipeline. - * `.brand` prefix raises specificity so this beats the later base - * `.ident-meta { display: flex }` rule regardless of source order. */ - .brand .ident-blurb, - .brand .ident-meta { - display: none; - } - .ident-title { - font-size: 18px; - } - .about-btn-mobile { - display: inline-flex; - } - /* Keep phone icons at the current compact size — the desktop bump above - * shouldn't leak into the mobile topbar. */ - .topbar-right .icon-btn { - width: 36px; - height: 36px; - } - .topbar-right .icon-btn svg { - width: 18px; - height: 18px; - } - /* Keep the account control compact on phones: avatar-only chip, shorter pill. */ - .account-chip { - height: 36px; - padding: 0 6px; - } - .account-handle { - display: none; - } - .signin-pill { - height: 36px; - padding: 0 12px; - } -} - -.topbar-right { - display: flex; - align-items: center; - gap: 10px; -} - -/* The HF user pill: compact avatar + handle. Rendered as a flex row so - * the avatar stays aligned with the text baseline even when the text - * wraps on narrow viewports. */ -.hf-user { - display: inline-flex; - align-items: center; - gap: 8px; - font-size: 13px; - color: var(--text-dim); - padding: 4px 10px 4px 4px; - border-radius: 999px; - background: var(--bg-elev); - border: 1px solid var(--border); - line-height: 1; -} - -.hf-avatar { - width: 24px; - height: 24px; - border-radius: 50%; - object-fit: cover; - background: color-mix(in srgb, var(--accent) 25%, var(--bg-elev-2)); - /* Hidden until an actual URL loads (see `setHfAvatar` in main.ts). - * Keeps the initial login flash from showing a broken image icon. */ - opacity: 0; - transition: opacity 0.25s ease; - flex: none; -} -.hf-avatar.loaded { - opacity: 1; -} - -.hf-user-name { - white-space: nowrap; -} - -/* ─── Transport pill ────────────────────────────────────────────── - * Surface the actual network path WebRTC picked for the robot peer - * connection (LAN, direct-through-NAT, or TURN-relayed). Lets the - * user spot at a glance when audio is going through the internet - * instead of staying on-prem. - */ -.transport-pill { - display: inline-flex; - align-items: center; - gap: 6px; - font-size: 12px; - font-weight: 500; - letter-spacing: 0.02em; - padding: 5px 10px; - border-radius: 999px; - background: var(--bg-elev); - border: 1px solid var(--border); - color: var(--text-dim); - user-select: none; - transition: background 0.2s ease, border-color 0.2s ease, color 0.2s ease; -} - -.transport-pill .transport-dot { - width: 7px; - height: 7px; - border-radius: 50%; - background: currentColor; - box-shadow: 0 0 0 3px color-mix(in srgb, currentColor 25%, transparent); - flex: none; -} - -/* Bitrate readout sits after the kind label. Dimmer + monospace so the - * kind (LAN / Direct / Relayed) stays the primary info and the numbers - * don't wiggle the layout as digits change. */ -.transport-pill .transport-bitrate { - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - color: color-mix(in srgb, currentColor 75%, var(--text-dim)); - opacity: 0.85; - padding-left: 6px; - border-left: 1px solid color-mix(in srgb, currentColor 25%, transparent); -} -.transport-pill .transport-bitrate:empty { - display: none; -} - -.transport-pill.transport-checking { - color: var(--text-dim); -} -.transport-pill.transport-checking .transport-dot { - animation: transport-blink 1.2s ease-in-out infinite; -} - -.transport-pill.transport-lan { - color: #4ade80; - border-color: color-mix(in srgb, #4ade80 35%, var(--border)); - background: color-mix(in srgb, #4ade80 10%, var(--bg-elev)); -} - -.transport-pill.transport-direct { - color: #60a5fa; - border-color: color-mix(in srgb, #60a5fa 35%, var(--border)); - background: color-mix(in srgb, #60a5fa 10%, var(--bg-elev)); -} - -.transport-pill.transport-relay { - color: #fbbf24; - border-color: color-mix(in srgb, #fbbf24 35%, var(--border)); - background: color-mix(in srgb, #fbbf24 10%, var(--bg-elev)); -} - -@keyframes transport-blink { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.35; } -} - -.icon-btn { - background: var(--bg-elev); - border: 1px solid var(--border); - color: var(--text-dim); - width: 36px; - height: 36px; - border-radius: var(--radius-sm); - display: inline-flex; - align-items: center; - justify-content: center; - cursor: pointer; - touch-action: manipulation; - transition: background 0.15s, color 0.15s, border-color 0.15s; -} -.icon-btn:hover { - background: var(--bg-elev-2); - color: var(--text); - border-color: var(--border-strong); -} -/* Topbar control cluster reads a touch bigger on desktop. Phones keep the - * compact 36px (reset in the mobile media query). */ -.topbar-right .icon-btn { - width: 40px; - height: 40px; -} -.topbar-right .icon-btn svg { - width: 20px; - height: 20px; -} - -/* ─── Account (HF login chip + sign-in pill + popover) ─────────────────────── */ -.account { - position: relative; - display: inline-flex; - align-items: center; -} -/* Signed-out: a compact pill that reads as the primary affordance. */ -.signin-pill { - display: inline-flex; - align-items: center; - gap: 7px; - height: 40px; - padding: 0 14px; - border-radius: var(--radius-sm); - border: 1px solid var(--border-strong); - background: var(--bg-elev); - color: var(--text); - font-size: 13px; - font-weight: 600; - text-decoration: none; - transition: background 0.15s, border-color 0.15s; -} -.signin-pill svg { - width: 16px; - height: 16px; - flex: none; -} -.signin-pill:hover { - background: var(--bg-elev-2); - border-color: var(--text); -} -/* Narrow viewports: collapse the pill to an icon-only square. */ -@media (max-width: 800px) { - .signin-pill { - gap: 0; - width: 40px; - padding: 0; - justify-content: center; - } - .signin-pill span { - display: none; - } -} -/* Signed-in: avatar + handle chip. */ -.account-chip { - display: inline-flex; - align-items: center; - gap: 8px; - height: 40px; - padding: 0 10px 0 6px; - border-radius: var(--radius-sm); - border: 1px solid var(--border); - background: var(--bg-elev); - color: var(--text-dim); - cursor: pointer; - transition: background 0.15s, color 0.15s, border-color 0.15s; -} -.account-chip:hover { - background: var(--bg-elev-2); - color: var(--text); - border-color: var(--border-strong); -} -.account-avatar { - width: 26px; - height: 26px; - border-radius: 50%; - object-fit: cover; - flex: none; -} -.account-avatar-fallback { - display: inline-flex; - align-items: center; - justify-content: center; - background: var(--bg-elev-2); - color: var(--text); - font-size: 12px; - font-weight: 700; -} -.account-handle { - font-size: 13px; - font-weight: 600; - max-width: 12ch; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; -} -.account-pro { - font-size: 10px; - font-weight: 700; - letter-spacing: 0.04em; - padding: 2px 5px; - border-radius: 5px; - background: var(--text); - color: var(--bg); -} -/* Org "Team" badge: same pill as PRO, accent-coloured so it reads as - * unlimited without claiming a paid PRO subscription. */ -.account-team { - background: var(--accent); - color: var(--bg); -} -.account-pop { - position: absolute; - top: calc(100% + 8px); - right: 0; - min-width: 200px; - padding: 6px; - border-radius: var(--radius-md); - border: 1px solid var(--border-strong); - background: var(--bg-elev); - box-shadow: var(--shadow-soft); - z-index: 50; -} -.account-pop[hidden] { - display: none; -} -.account-pop-row { - padding: 8px 10px; -} -.account-pop-name { - font-weight: 600; - font-size: 13px; -} -.account-pop-meta { - display: flex; - align-items: center; - justify-content: space-between; - gap: 10px; - padding-top: 0; - font-size: 12px; - color: var(--text-dim); -} -.account-tier { - font-weight: 600; - color: var(--text); -} -.account-pop-link { - display: block; - padding: 8px 10px; - border-radius: var(--radius-sm); - color: var(--text-dim); - font-size: 13px; - text-decoration: none; - transition: background 0.15s, color 0.15s; -} -.account-pop-link:hover { - background: var(--bg-elev-2); - color: var(--text); -} -.account-signout { - border-top: 1px solid var(--border); - margin-top: 4px; - padding-top: 10px; - border-radius: 0 0 var(--radius-sm) var(--radius-sm); -} - -/* ─── Daily-limit modal ────────────────────────────────────────────────────── */ -.limit-modal { - width: min(400px, 92vw); -} -.limit-card { - position: relative; - align-items: center; - text-align: center; - gap: 14px; - padding: 34px 28px 26px; -} -.limit-close { - position: absolute; - top: 12px; - right: 12px; -} -/* HF smiling face (brand yellow) on a neutral badge — friendly, not a stop sign. - * The logo is the single pop of colour, so the badge itself stays quiet. */ -.limit-badge { - width: 66px; - height: 66px; - border-radius: 50%; - display: grid; - place-items: center; - margin: 2px auto 2px; - background: var(--bg-elev-2); - border: 1px solid var(--border-strong); - box-shadow: 0 0 0 6px rgba(255, 210, 30, 0.06); -} -.limit-badge .hf-logo { - width: 46px; - height: auto; -} -.limit-title { - margin: 0; - font-size: 20px; - font-weight: 700; - letter-spacing: 0; -} -.limit-msg { - color: var(--text-dim); - font-size: 14px; - line-height: 1.55; - margin: 0; - max-width: 30ch; -} -.limit-cta { - display: inline-flex; - align-items: center; - justify-content: center; - gap: 8px; - margin-top: 4px; -} -.limit-cta .hf-logo { - width: 20px; - height: auto; - flex: none; -} -.limit-note { - margin: 0; - font-size: 12px; - color: var(--text-faint); -} -#limit-cta[hidden] { - display: none; -} - -/* ─── Stage ───────────────────────────────────────────────────────────── */ - -.stage { - position: relative; - z-index: 1; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - padding: 24px 24px 32px; - gap: 20px; - min-height: 0; - background: transparent; -} - -/* ─── Central circle ──────────────────────────────────────────────────── */ - -/* Wraps the orb and its two side controls so they stay aligned on one row. */ -.orb-wrap { - position: relative; - display: flex; - align-items: center; - justify-content: center; - gap: clamp(18px, 3vw, 32px); -} - -.circle { - position: relative; - /* Cap by height as well as width (min(vw, vh)-style): in a short container - * the orb shrinks instead of overflowing, so the page stays scroll-free even - * when embedded in a small box. On normal screens the 320px cap still wins, - * so the desktop size is unchanged. */ - width: clamp(140px, min(38vw, 46vh), 320px); - aspect-ratio: 1 / 1; - border-radius: 50%; - border: none; - background: transparent; - padding: 0; - cursor: pointer; - outline: none; - display: grid; - place-items: center; - color: var(--glow, var(--accent)); - transition: transform 0.18s ease, filter 0.18s ease; - -webkit-tap-highlight-color: transparent; - /* Treat taps as immediate clicks (no 300ms delay / double-tap-zoom). */ - touch-action: manipulation; -} -.circle:hover { - filter: brightness(1.08); -} -.circle:active { - transform: scale(0.97); - filter: brightness(0.92); -} -.circle:focus-visible .circle-core { - outline: 2px solid var(--glow, var(--accent)); - outline-offset: 6px; -} -.circle[disabled] { - cursor: default; - opacity: 0.75; -} - -.circle-glow { - position: absolute; - inset: 0; - border-radius: 50%; - background: radial-gradient(circle at center, var(--glow, var(--accent)) 0%, transparent 65%); - filter: blur(28px); - opacity: 0.5; - transform: scale(1); - transition: opacity 0.25s, background 0.25s, transform 1.4s ease-in-out; - pointer-events: none; -} - -/* Two nested circular rings: the inner tracks the core edge, the outer - * expands / fades to convey "audio radiating out" during speaking. */ -.circle-ring, -.circle-ring-outer { - position: absolute; - top: 50%; - left: 50%; - border-radius: 50%; - transform: translate(-50%, -50%); - pointer-events: none; - transition: opacity 0.4s ease, border-color 0.4s ease, transform 0.3s ease; -} -.circle-ring { - width: 82%; - height: 82%; - border: 1.5px solid color-mix(in srgb, var(--glow, var(--accent)) 35%, transparent); - opacity: 0.35; -} -.circle-ring-outer { - width: 94%; - height: 94%; - border: 1px solid color-mix(in srgb, var(--glow, var(--accent)) 22%, transparent); - opacity: 0; -} - -.circle-core { - position: relative; - width: 72%; - height: 72%; - border-radius: 50%; - background: radial-gradient( - circle at 35% 28%, - color-mix(in srgb, var(--glow, var(--accent)) 28%, transparent), - color-mix(in srgb, var(--glow, var(--accent)) 10%, transparent) 55%, - color-mix(in srgb, var(--glow, var(--accent)) 5%, transparent) - ); - border: 2px solid color-mix(in srgb, var(--glow, var(--accent)) 40%, transparent); - display: grid; - place-items: center; - box-shadow: - 0 0 32px color-mix(in srgb, var(--glow, var(--accent)) 22%, transparent), - inset 0 0 28px color-mix(in srgb, var(--glow, var(--accent)) 18%, transparent); - transition: background 0.4s ease, border-color 0.4s ease, box-shadow 0.4s ease, transform 0.4s ease; -} - -/* Indicator slot: a single SVG / spinner / bar group is visible at a time, - * driven by the state class on `.circle`. */ -.circle-indicator { - position: relative; - width: 44%; - height: 44%; - display: grid; - place-items: center; - color: var(--glow, var(--accent)); -} -.circle-indicator > .ind { - grid-area: 1 / 1; - opacity: 0; - transform: scale(0.85); - transition: opacity 0.25s ease, transform 0.25s ease; - pointer-events: none; -} -.circle-indicator > svg.ind { - width: 60%; - height: 60%; -} - -/* Spinner: a rotating ring gap, CSS-only. - * - * Note: the base `.circle-indicator > .ind` rule forces `transform: - * scale(.85)` / `scale(1)` on every indicator to drive the show/hide - * transition. If we only rotate here, the browser has to interpolate - * between `scale(1)` and `rotate(360deg)` (two different transform - * functions), which produces a broken, barely-moving animation. So we - * include the scale explicitly in the keyframes and bump specificity - * with `!important` so the spinner always wins over the state rule. */ -.ind-spinner { - width: 48%; - height: 48%; - border: 3px solid currentColor; - border-right-color: transparent; - border-radius: 50%; - opacity: 0; - animation: ind-spin 0.9s linear infinite; -} - -/* Thinking dots: 3 soft pulsing dots while the model is composing a - * response. Apple-style cadence: each dot scales up and brightens in - * turn, staggered by ~160 ms. Per-dot animation is on the child, so - * the parent's scale(.85 → 1) show/hide transform composes cleanly. */ -.ind-thinking { - display: flex; - align-items: center; - justify-content: center; - gap: 8px; - width: 60%; - height: 60%; -} -.ind-thinking .dot { - width: 10px; - height: 10px; - border-radius: 50%; - background: currentColor; - opacity: 0.3; - animation: thinking-dot 1.25s ease-in-out infinite; -} -.ind-thinking .dot:nth-child(1) { animation-delay: 0s; } -.ind-thinking .dot:nth-child(2) { animation-delay: 0.16s; } -.ind-thinking .dot:nth-child(3) { animation-delay: 0.32s; } - -/* Bars: 5 vertical pills driven by --bar0..--bar4 CSS vars. */ -.ind-bars { - display: flex; - align-items: center; - gap: 5px; - height: 42%; -} -.ind-bars .bar { - width: 4px; - min-height: 4px; - border-radius: 3px; - background: currentColor; - opacity: 0.7; - --h: var(--bar0, 0); - height: calc(4px + var(--h) * 36px); - transition: height 0.08s ease-out, opacity 0.08s ease-out; -} -.ind-bars .bar:nth-child(1) { --h: var(--bar0); } -.ind-bars .bar:nth-child(2) { --h: var(--bar1); } -.ind-bars .bar:nth-child(3) { --h: var(--bar2); } -.ind-bars .bar:nth-child(4) { --h: var(--bar3); } -.ind-bars .bar:nth-child(5) { --h: var(--bar4); } -.ind-bars .bar { - opacity: calc(0.55 + 0.45 * var(--h)); -} - -/* Active indicator per state. - * - * Note: `ai-speaking` deliberately has NO indicator here. The orb - * itself becomes the indicator by pulsing on Reachy's voice (see the - * `--ai-audio-level` rules further down), which is a lot clearer and - * less confusing than reusing the mic-bars (which viewers would read - * as "you are speaking"). */ -.state-signed-out .ind-connect, -.state-authenticated .ind-mic, -.state-ready .ind-mic, -.state-connecting .ind-spinner, -.state-connected .ind-spinner, -.state-auto-selecting .ind-spinner, -.state-starting .ind-spinner, -.state-queued .ind-spinner, -.state-processing .ind-thinking, -.state-listening .ind-bars, -.state-user-speaking .ind-bars, -.state-ai-speaking .ind-voice, -.state-error .ind-error { - opacity: 1; - transform: scale(1); -} - -/* AI speaking indicator: speaker + two sound waves. - * - * Each wave pulses outward (opacity + stroke grow) with a quarter- - * beat offset so it reads as sound radiating out. Kept as a pure - * CSS animation so the icon is always visually alive even between - * syllables when --ai-audio-level momentarily dips. */ -.ind-voice .wave { - transform-origin: 50% 50%; - animation: voice-wave 1.35s ease-out infinite; -} -.ind-voice .wave-1 { animation-delay: 0s; } -.ind-voice .wave-2 { animation-delay: 0.35s; } - -/* Idle indicators: a chain-link "connect" glyph for the signed-out - * step (invites the user to authenticate with HF) and a microphone - * once a session is ready. Both picked up by the generic - * `.circle-indicator > svg.ind` sizing rule (60% × 60% of the slot) - * and rely on per-state opacity transitions for show / hide. */ -.ind-connect, -.ind-mic { - color: color-mix(in srgb, var(--glow, var(--accent)) 85%, white); -} - -/* ─── Caption below the circle ────────────────────────────────────────── */ - -/* The caption under the orb is meant to whisper, not shout: micro-label - * vibe, uppercase, letter-spaced, muted. Only appears for actionable / - * transitional states (see STATE_VIEWS). During a live conversation the - * orb alone carries the state so we collapse this row entirely. */ -.circle-caption { - margin: 0; - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - letter-spacing: 0.14em; - text-transform: uppercase; - color: var(--text-faint); - min-height: 1.2em; - text-align: center; - opacity: 0.75; - transition: opacity 0.25s ease, color 0.25s ease, transform 0.25s ease, - min-height 0.25s ease; -} -.circle-caption.empty { - opacity: 0; - min-height: 0; - transform: translateY(-4px); - pointer-events: none; -} -.circle-caption.muted { - color: var(--text-faint); - opacity: 0.65; -} -.circle-caption.error { - color: var(--error); - opacity: 1; - letter-spacing: 0.08em; -} - -/* Warm, human line under the mono caption — sentence case, only while queued. - * The caption keeps the terse position; this reassures. */ -.circle-subcaption { - margin: 8px 0 0; - max-width: 32ch; - font-size: 13.5px; - line-height: 1.5; - color: var(--text-dim); - text-align: center; - text-wrap: balance; - transition: opacity 0.25s ease; -} -.circle-subcaption[hidden] { display: none; } - -/* Queue actions sit under the caption: "Join now" (primary, only when it's your - * turn) stacked above the quiet "Leave queue" escape hatch. */ -.queue-actions { - margin-top: 16px; - display: flex; - flex-direction: column; - align-items: center; - gap: 10px; -} -.queue-actions[hidden] { display: none; } - -/* "Join now": the one call to action in the queue flow, so it reads as inviting - * (filled accent) while everything around it stays quiet. */ -.join-queue-btn { - padding: 10px 26px; - font-family: var(--font-mono); - font-size: 12px; - font-weight: 600; - letter-spacing: 0.1em; - text-transform: uppercase; - color: #0b0b10; - background: var(--accent); - border: none; - border-radius: 999px; - cursor: pointer; - font-variant-numeric: tabular-nums; - transition: transform 0.12s ease, filter 0.2s ease; -} -.join-queue-btn:hover { filter: brightness(1.08); } -.join-queue-btn:active { transform: scale(0.97); } -.join-queue-btn:focus-visible { - outline: 2px solid var(--accent); - outline-offset: 3px; -} -.join-queue-btn[hidden] { display: none; } - -/* "Leave queue": a quiet outlined pill, understated so it reads as an escape - * hatch, not a CTA. */ -.leave-queue-btn { - padding: 7px 16px; - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - letter-spacing: 0.12em; - text-transform: uppercase; - color: var(--text-faint); - background: transparent; - border: 1px solid color-mix(in srgb, var(--text-faint) 35%, transparent); - border-radius: 999px; - cursor: pointer; - transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease; -} -.leave-queue-btn:hover { - color: var(--text); - border-color: color-mix(in srgb, var(--text-faint) 60%, transparent); - background: color-mix(in srgb, var(--text-faint) 8%, transparent); -} -.leave-queue-btn:focus-visible { - outline: 2px solid var(--accent); - outline-offset: 2px; -} -.leave-queue-btn[hidden] { display: none; } - -/* ─── Tool-call toaster ───────────────────────────────────────────────── */ - -/* A small, non-interactive pill that appears below the circle when the - * model invokes a tool (move_head, play_move). Sits just under the - * state caption, collapses to zero height when no toast is active. */ -.tool-toast { - display: inline-flex; - align-items: center; - gap: 8px; - margin-top: 10px; - padding: 6px 12px; - border-radius: 999px; - border: 1px solid var(--border-strong); - background: color-mix(in srgb, var(--bg-elev-2) 78%, transparent); - color: var(--text-dim); - font-size: 12px; - font-weight: 500; - letter-spacing: 0.01em; - line-height: 1; - white-space: nowrap; - max-width: 80vw; - overflow: hidden; - text-overflow: ellipsis; - - opacity: 0; - transform: translateY(-4px) scale(0.96); - pointer-events: none; - transition: opacity 0.22s ease, transform 0.22s ease; -} -.tool-toast.visible { - opacity: 1; - transform: translateY(0) scale(1); -} -.tool-toast-icon { - width: 14px; - height: 14px; - flex: none; - color: color-mix(in srgb, var(--voice-tool) 80%, white); - animation: tool-toast-spin 3.2s linear infinite; - animation-play-state: paused; -} -.tool-toast.visible .tool-toast-icon { - animation-play-state: running; -} -.tool-toast-text { - display: inline-block; - overflow: hidden; - text-overflow: ellipsis; -} -@keyframes tool-toast-spin { - from { transform: rotate(0deg); } - to { transform: rotate(360deg); } -} - -/* ─── Side controls (mic / stop) ──────────────────────────────────────── */ - -/* Mic button + its radial noise-gate arc. The wrapper centres the button; the - arc SVG is an absolute overlay larger than the button, revealed with the - live session (it has no layout footprint, so the idle collapse is unaffected). */ -.mic-gate { - position: relative; - flex: none; - display: inline-flex; - align-items: center; - justify-content: center; -} -.mic-gate-arc { - position: absolute; - left: 50%; - top: 50%; - width: 80px; - height: 80px; - transform: translate(-50%, -50%); - pointer-events: none; /* only the hit-path below catches drags */ - opacity: 0; - transition: opacity 0.25s ease; -} -.orb-wrap.live .mic-gate-arc { opacity: 1; } -.mga-track { - stroke: rgba(255, 255, 255, 0.1); /* hairline; recedes until needed */ - stroke-width: 1.5; - stroke-linecap: round; -} -.mga-fill { - stroke: var(--accent-2); - stroke-width: 2; - stroke-linecap: round; - opacity: 0.85; - transition: stroke-dashoffset 0.06s linear; -} -/* Threshold setpoint: a bead riding the ring. White with a thin dark outline so - it stays legible over the moving fill; it recolors to cyan with a soft glow - the moment the live level crosses it (the gate opening). */ -.mga-handle { - fill: var(--text); - stroke: none; - transition: fill 0.2s ease, filter 0.2s ease; -} -.mic-gate.gate-open .mga-handle { - fill: var(--accent-2); - filter: drop-shadow(0 0 3px var(--accent-2)); -} -.mga-hit { - stroke: transparent; - stroke-width: 16; - pointer-events: none; /* enabled only while live (below) */ - cursor: pointer; - touch-action: none; -} -/* Only catch drags during a live call; when idle the arc is hidden and must - not steal clicks near the collapsed mic button / orb. */ -.orb-wrap.live .mga-hit { pointer-events: stroke; } - -.side-btn { - flex: none; - width: 52px; - height: 52px; - border-radius: 50%; - border: 1px solid var(--border-strong); - /* Brighter background so the buttons actually stand out against the - * deep-blue stage gradient; previous var(--bg-elev) was too close to - * the page bg to be readable. */ - background: color-mix(in srgb, var(--bg-elev-2) 80%, #2a2e3c); - color: var(--text); - display: inline-flex; - align-items: center; - justify-content: center; - cursor: pointer; - touch-action: manipulation; - box-shadow: 0 6px 16px rgba(0, 0, 0, 0.35), - inset 0 1px 0 rgba(255, 255, 255, 0.06); - /* Hidden by default: take up no space until the session is live. The - * `width: 0` collapse keeps the orb centered on the idle screen. */ - opacity: 0; - transform: scale(0.55); - width: 0; - padding: 0; - pointer-events: none; - overflow: hidden; - transition: opacity 0.25s ease, transform 0.25s ease, width 0.25s ease, - background 0.15s, color 0.15s, border-color 0.15s; -} -.side-btn:hover { - background: color-mix(in srgb, var(--bg-elev-2) 60%, #323746); - border-color: var(--text-dim); -} -.side-btn svg { - width: 22px; - height: 22px; - flex: none; -} -.side-btn .mic-off { display: none; } -.side-btn.muted { - color: #fff; - border-color: var(--error); - background: color-mix(in srgb, var(--error) 70%, #1a0e13); -} -.side-btn.muted .mic-on { display: none; } -.side-btn.muted .mic-off { display: block; } - -/* Stop button: subtle warm tint so "end" reads as destructive. */ -#stop-btn:hover { - color: #fff; - border-color: color-mix(in srgb, var(--error) 60%, var(--border-strong)); - background: color-mix(in srgb, var(--error) 25%, var(--bg-elev-2)); -} - -/* Reveal when the session is live: buttons flank the orb on a flex row. */ -.orb-wrap.live .side-btn { - opacity: 1; - transform: scale(1); - width: 52px; - pointer-events: auto; -} - -/* Disable every transition / animation during the first paint so the - * orb doesn't fade-and-scale in when the page loads. `main.ts` removes - * the class after one animation frame. */ -body.booting, -body.booting *, -body.booting *::before, -body.booting *::after { - transition: none !important; - animation-duration: 0s !important; - animation-delay: 0s !important; -} - -/* ─── Circle animation keyframes ──────────────────────────────────────── */ - -/* Slow, subtle breathing for "warm idle" states. */ -@keyframes breathe { - 0%, 100% { transform: translate(-50%, -50%) scale(1); opacity: 0.4; } - 50% { transform: translate(-50%, -50%) scale(1.06); opacity: 0.15; } -} - -/* Outer ring expanding and fading - conveys "I am producing audio". */ -@keyframes ring-out { - 0% { transform: translate(-50%, -50%) scale(1); opacity: 0.35; } - 100% { transform: translate(-50%, -50%) scale(1.18); opacity: 0; } -} - -/* Soft inner scale for the core while talking. */ -@keyframes core-breathe { - 0%, 100% { transform: scale(1); } - 50% { transform: scale(1.04); } -} - -/* Subtle glow throb used for "thinking" — dimmer than speaking. */ -@keyframes thinking { - 0%, 100% { transform: scale(1); opacity: 0.7; } - 50% { transform: scale(0.96); opacity: 0.45; } -} - -/* Individual dot pulse for the 3-dot processing indicator. */ -@keyframes thinking-dot { - 0%, 60%, 100% { transform: scale(0.7); opacity: 0.3; } - 30% { transform: scale(1.15); opacity: 1; } -} - -/* Sound-wave pulse: arc fades in, scales up slightly, fades out. - * The transform-origin is the speaker's center (roughly x=12), so - * the waves feel like they're emanating from the cone. */ -@keyframes voice-wave { - 0% { opacity: 0; transform: scale(0.7); } - 30% { opacity: 1; transform: scale(1); } - 70% { opacity: 0.2; transform: scale(1.12); } - 100% { opacity: 0; transform: scale(0.7); } -} - -@keyframes ind-spin { - /* Scale kept at 1 so we don't fight with the indicator's base - * show/hide transform (see `.ind-spinner` for the rationale). */ - from { transform: scale(1) rotate(0deg); } - to { transform: scale(1) rotate(360deg); } -} - -/* ─── State-specific colors ──────────────────────────────────────────── */ - -.circle.state-signed-out { --glow: #8b7dff; } -.circle.state-authenticated, -.circle.state-ready { --glow: #34d399; } -.circle.state-connecting, -.circle.state-connected, -.circle.state-auto-selecting, -.circle.state-starting { --glow: #facc15; } -/* Queued: a calm slate glow, distinct from connecting's active yellow — this is - * waiting, not working. The spinner turns slowly and the ring breathes. */ -.circle.state-queued { --glow: #94a3b8; } -.circle.state-queued .ind-spinner { animation-duration: 2.4s; } -.circle.state-queued .circle-ring { animation: breathe 2.6s ease-in-out infinite; } -/* Your turn: a slot is held for you — the orb warms to the accent and breathes a - * little quicker, an invitation to join. */ -.circle.state-your-turn { --glow: var(--accent); } -.circle.state-your-turn .circle-ring { animation: breathe 1.6s ease-in-out infinite; } -.circle.state-listening, -.circle.state-user-speaking { --glow: var(--listening); } -.circle.state-processing { --glow: var(--processing); } -.circle.state-ai-speaking { --glow: var(--speaking); } -.circle.state-error { --glow: var(--error); } - -/* Idle / ready: gentle breathing of the inner ring. Kept out of the - * `signed-out` state so the very first paint on page load stays quiet - * (the orb now shows the Reachy head silhouette, no need to also pulse). */ -.circle.state-authenticated .circle-ring, -.circle.state-ready .circle-ring { - animation: breathe 2.4s ease-in-out infinite; -} - -/* Connecting flows: subtle glow throb so the orb feels thoughtful - * while the session is being negotiated. Kept off `processing` on - * purpose - the 3 thinking dots already pulse, and layering another - * throb on top competes with them for attention. */ -.circle.state-connecting .circle-core, -.circle.state-connected .circle-core, -.circle.state-auto-selecting .circle-core, -.circle.state-starting .circle-core { - animation: thinking 1.4s ease-in-out infinite; -} - -/* User is speaking: the mic RMS drives scale + opacity via --audio-level. */ -.circle.state-user-speaking .circle-ring { - animation: none; - opacity: calc(0.25 + 0.55 * var(--audio-level)); - transform: translate(-50%, -50%) scale(calc(1 + 0.08 * var(--audio-level))); - transition: transform 0.08s linear, opacity 0.08s linear; -} -.circle.state-user-speaking .circle-ring-outer { - opacity: calc(0.1 + 0.35 * var(--audio-level)); - transform: translate(-50%, -50%) scale(calc(1 + 0.12 * var(--audio-level))); - transition: transform 0.08s linear, opacity 0.08s linear; -} -.circle.state-listening .circle-ring { - animation: breathe 2s ease-in-out infinite; -} - -/* Assistant is speaking: the whole orb breathes in sync with Reachy's - * voice instead of running a fixed timer. `--ai-audio-level` (0-1) is - * updated at display rate by AiLevelMonitor from the OpenAI output - * track, so every syllable visibly moves the core + outer ring. This - * reads instantly as "the orb is the voice" and completely avoids the - * ambiguity of bars-vs-mic the user flagged. */ -.circle.state-ai-speaking .circle-core { - animation: none; - transform: scale(calc(1 + 0.09 * var(--ai-audio-level, 0))); - transition: transform 0.08s linear; -} -.circle.state-ai-speaking .circle-ring { - animation: none; - opacity: calc(0.3 + 0.5 * var(--ai-audio-level, 0)); - transform: translate(-50%, -50%) scale(calc(1 + 0.05 * var(--ai-audio-level, 0))); - transition: transform 0.08s linear, opacity 0.08s linear; -} -.circle.state-ai-speaking .circle-ring-outer { - animation: none; - opacity: calc(0.15 + 0.55 * var(--ai-audio-level, 0)); - transform: translate(-50%, -50%) scale(calc(1 + 0.18 * var(--ai-audio-level, 0))); - transition: transform 0.08s linear, opacity 0.08s linear; -} -.circle.state-ai-speaking .circle-glow { - opacity: calc(0.35 + 0.45 * var(--ai-audio-level, 0)); - transition: opacity 0.08s linear; -} - -/* ─── Robot picker ────────────────────────────────────────────────────── */ - -.robot-picker { - width: min(420px, 92vw); - background: var(--bg-elev); - border: 1px solid var(--border); - border-radius: var(--radius-md); - padding: 14px 16px; - box-shadow: var(--shadow-soft); -} - -.picker-title { - margin: 0 0 10px; - font-size: 11px; - font-weight: 600; - letter-spacing: 0.1em; - text-transform: uppercase; - color: var(--text-faint); -} - -.robot-list { - display: flex; - flex-direction: column; - gap: 8px; -} - -.robot-card { - display: flex; - justify-content: space-between; - align-items: center; - padding: 12px 14px; - border: 1px solid var(--border); - border-radius: var(--radius-sm); - background: var(--bg-elev-2); - cursor: pointer; - transition: border-color 0.15s, background 0.15s; -} -.robot-card:hover { - border-color: var(--border-strong); -} -.robot-card.selected { - border-color: var(--border-strong); - background: var(--bg-elev); -} - -.robot-card .name { - font-weight: 600; - font-size: 14px; -} -.robot-card .id { - font-family: var(--font-mono); - font-size: 12px; - color: var(--text-faint); -} - -.robot-empty { - padding: 14px; - text-align: center; - color: var(--text-faint); - font-size: 13px; -} - -/* ─── Footer ──────────────────────────────────────────────────────────── */ - -/* ─── Text input bar ─────────────────────────────────────────────────────── */ - -.text-input-bar { - display: flex; - align-items: center; - gap: 8px; - max-width: 560px; - margin: 20px auto 0; - padding: 0 20px; - width: 100%; -} - -.text-input { - flex: 1; - background: var(--bg-elev-2); - border: 1px solid var(--border); - border-radius: var(--radius-lg); - padding: 10px 16px; - font-family: inherit; - font-size: 14px; - color: var(--text); - outline: none; - transition: border-color 0.2s ease; -} - -.text-input::placeholder { - color: var(--text-faint); -} - -.text-input:focus { - border-color: var(--accent-2); -} - -.text-input:disabled { - opacity: 0.4; - cursor: not-allowed; -} - -.text-send-btn { - display: flex; - align-items: center; - justify-content: center; - width: 36px; - height: 36px; - border-radius: 50%; - border: none; - background: var(--accent); - color: #fff; - cursor: pointer; - transition: opacity 0.2s ease, background 0.2s ease; - flex-shrink: 0; -} - -.text-send-btn:hover:not(:disabled) { - background: var(--accent-2); -} - -.text-send-btn:disabled { - opacity: 0.35; - cursor: not-allowed; -} - -.footer { - padding: 12px 28px 18px; - font-size: 11px; - letter-spacing: 0.02em; - color: var(--text-faint); - display: flex; - justify-content: center; - opacity: 0.55; - transition: opacity 0.25s ease; -} -.footer:hover { - opacity: 0.9; -} -.footer a { - color: inherit; - text-decoration: none; - border-bottom: 1px dotted currentColor; -} -.footer a:hover { - color: var(--text-dim); -} -/* While the webcam preview sits bottom-left, push the credit to the - * bottom-right so the two don't collide. */ -body.cam-on .footer { - justify-content: flex-end; -} - -/* ─── Modal ───────────────────────────────────────────────────────────── */ - -.modal { - border: 1px solid var(--border); - border-radius: var(--radius-md); - padding: 0; - background: var(--bg-elev); - color: var(--text); - width: min(440px, 92vw); - box-shadow: var(--shadow-soft); -} -.modal::backdrop { - background: rgba(8, 9, 13, 0.65); - backdrop-filter: blur(4px); -} - -.modal-content { - display: flex; - flex-direction: column; - gap: 16px; - padding: 22px 24px 20px; -} - -.modal-header { - display: flex; - align-items: center; - justify-content: space-between; - margin-bottom: 4px; -} -.modal-header h2 { - margin: 0; - font-size: 16px; - font-weight: 600; - letter-spacing: 0.02em; -} - -.field[hidden] { display: none; } -.field { - display: flex; - flex-direction: column; - gap: 6px; - font-size: 13px; - font-weight: 500; - color: var(--text-dim); -} -.field > span { - color: var(--text); - font-weight: 600; - font-size: 12px; - letter-spacing: 0.04em; - text-transform: uppercase; -} -.field input, -.field select, -.field textarea { - font-family: inherit; - font-size: 14px; - color: var(--text); - background: var(--bg); - border: 1px solid var(--border); - border-radius: var(--radius-sm); - padding: 10px 12px; - outline: none; - transition: border-color 0.15s; - resize: vertical; -} -.field textarea { - min-height: 84px; -} -.field input:focus, -.field select:focus, -.field textarea:focus { - border-color: var(--text-dim); -} -/* Upload button for custom voice samples */ -.btn-upload { - display: inline-flex; - align-items: center; - justify-content: center; - gap: 4px; - font-family: inherit; - font-size: 13px; - font-weight: 600; - color: var(--accent); - background: var(--bg); - border: 1px dashed var(--accent); - border-radius: var(--radius-sm); - padding: 8px 12px; - cursor: pointer; - transition: background 0.15s, color 0.15s; - user-select: none; - margin-top: 2px; -} -.btn-upload:hover { - background: var(--accent); - color: #fff; -} -.btn-upload:focus-visible { - outline: 2px solid var(--accent); - outline-offset: 2px; -} - -/* Deploy-pinned server URL: shown for transparency, not editable. */ -.field input[readonly] { - color: var(--text-dim); - border-style: dashed; - cursor: default; -} -.field input[readonly]:focus { - border-color: var(--border); -} -.field small { - color: var(--text-faint); - font-size: 12px; - line-height: 1.4; -} -.field small.error { color: var(--error); } -.field small code { - background: var(--bg); - padding: 1px 5px; - border-radius: 4px; - border: 1px solid var(--border); -} - -/* Noise gate: a header with a live value, a level meter, and a range slider - that shares the meter's horizontal (dB) axis. */ -.field-head { - display: flex; - align-items: baseline; - justify-content: space-between; -} -.field-value { - font-weight: 500; - font-size: 12px; - letter-spacing: 0; - text-transform: none; - color: var(--text-dim); -} -/* The slider and the level meter are one widget: the range input is overlaid - on the meter track (its native track is transparent), so the live-level fill - shows through behind the thumb and the thumb itself is the threshold. */ -.gate { - display: flex; - flex-direction: column; - gap: 4px; -} -.gate-track { - position: relative; - height: 14px; - display: flex; - align-items: center; -} -.gate-track::before { - /* the visible meter track */ - content: ""; - position: absolute; - left: 0; - right: 0; - height: 8px; - border-radius: 999px; - background: var(--bg); - border: 1px solid var(--border); -} -.gate-meter-fill { - position: absolute; - left: 1px; - top: 50%; - transform: translateY(-50%); - height: 6px; - width: 0; - border-radius: 999px; - background: var(--accent-2); - transition: width 0.06s linear; - pointer-events: none; -} -.gate-ends { - display: flex; - justify-content: space-between; - font-size: 11px; - color: var(--text-faint); - letter-spacing: 0; - text-transform: none; -} -/* The range input sits transparently on top of the meter track. */ -.gate-track input[type="range"] { - position: relative; - z-index: 1; - width: 100%; - margin: 0; - -webkit-appearance: none; - appearance: none; - padding: 0; - border: none; - background: transparent; - height: 14px; - cursor: pointer; -} -.gate-track input[type="range"]::-webkit-slider-runnable-track { - height: 14px; - background: transparent; -} -.gate-track input[type="range"]::-moz-range-track { - height: 14px; - background: transparent; -} -.gate-track input[type="range"]::-webkit-slider-thumb { - -webkit-appearance: none; - appearance: none; - width: 6px; - height: 18px; - border-radius: 3px; - background: var(--text); - border: 2px solid var(--bg-elev); - box-shadow: 0 0 0 1px var(--border-strong); -} -.gate-track input[type="range"]::-moz-range-thumb { - width: 6px; - height: 18px; - border-radius: 3px; - background: var(--text); - border: 2px solid var(--bg-elev); - box-shadow: 0 0 0 1px var(--border-strong); -} -.gate-track input[type="range"]:focus { border: none; } - -.modal-footer { - display: flex; - justify-content: space-between; - gap: 12px; - padding-top: 6px; -} - -.btn { - padding: 10px 16px; - border-radius: var(--radius-sm); - border: 1px solid var(--border-strong); - background: var(--bg-elev-2); - color: var(--text); - font-weight: 600; - font-size: 13px; - cursor: pointer; - transition: background 0.15s, border-color 0.15s, transform 0.05s; -} -.btn:hover { - border-color: var(--text); -} -.btn:active { - transform: translateY(1px); -} -.btn.primary { - background: var(--text); - border-color: var(--text); - color: var(--bg); -} -.btn.primary:hover { - background: #fff; - border-color: #fff; -} -.btn.ghost { - background: transparent; - border-color: var(--border); - color: var(--text-dim); -} -.btn.wide { - width: 100%; - padding: 12px 16px; -} -.btn[disabled] { - opacity: 0.45; - cursor: not-allowed; -} -.btn[disabled]:hover { - border-color: var(--border-strong); -} - -/* ─── Settings tabs ───────────────────────────────────────────────────── */ - -.tabs { - display: flex; - gap: 4px; - padding: 4px; - background: var(--bg); - border: 1px solid var(--border); - border-radius: var(--radius-sm); -} -.tab { - flex: 1; - padding: 8px 12px; - border: none; - background: transparent; - color: var(--text-dim); - font-family: inherit; - font-size: 13px; - font-weight: 600; - letter-spacing: 0.02em; - border-radius: calc(var(--radius-sm) - 3px); - cursor: pointer; - transition: background 0.15s, color 0.15s; -} -.tab:hover { - color: var(--text); -} -.tab.active { - background: var(--bg-elev-2); - color: var(--text); - box-shadow: 0 1px 2px rgba(0, 0, 0, 0.3); -} - -.tab-panels { - display: flex; - flex-direction: column; -} -.tab-panel { - display: flex; - flex-direction: column; - gap: 16px; -} -.tab-panel[hidden] { - display: none; -} - -/* Horizontal layout for Voice + Model. */ -.field-row { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 12px; -} - -/* ─── Chat button + badge ────────────────────────────────────────────── */ - -#chat-btn { - position: relative; -} - -.chat-badge { - position: absolute; - top: 6px; - right: 6px; - width: 7px; - height: 7px; - border-radius: 50%; - background: var(--text); - border: 1.5px solid var(--bg-elev); - opacity: 0; - transform: scale(0); - transition: opacity 0.2s ease, transform 0.2s ease; - pointer-events: none; -} -.chat-badge.visible { - opacity: 1; - transform: scale(1); -} - -/* ─── Lyric-style ephemeral transcript ───────────────────────────────── - * A vertical lyrics stack like a music player: newest line is largest and - * sits at the bottom, older lines shrink and fade as they move up, and the - * oldest fades out at the top as new lines arrive. */ - -.bubble-stack { - position: fixed; - right: 40px; - bottom: 15%; - width: min(440px, calc(100vw - 64px)); - display: flex; - flex-direction: column; - justify-content: flex-end; - align-items: flex-end; - gap: 6px; - pointer-events: none; - z-index: 100; -} - -.bubble { - pointer-events: none; - max-width: 100%; - padding: 2px 0; - text-align: right; - line-height: 1.35; - word-break: break-word; - font-size: 12px; - font-weight: 400; - color: rgba(255, 255, 255, 0.85); - opacity: 0; - transform: translateY(12px); - transition: opacity 0.25s ease, transform 0.25s ease, font-size 0.25s ease; -} -.bubble.in { - opacity: 1; - transform: translateY(0); -} -.bubble.out { - opacity: 0; - transform: translateY(-10px); - transition: opacity 0.3s ease, transform 0.3s ease; -} - -/* Progressive lyric sizing — newest line (last child) is biggest, older - * lines shrink and dim as they climb the stack. */ -.bubble.in:last-child { - font-size: 26px; - font-weight: 700; -} -.bubble.in:nth-last-child(2) { - font-size: 21px; - font-weight: 600; - opacity: 0.85; -} -.bubble.in:nth-last-child(3) { - font-size: 17px; - font-weight: 500; - opacity: 0.7; -} -.bubble.in:nth-last-child(4) { - font-size: 14px; - font-weight: 400; - opacity: 0.55; -} -.bubble.in:nth-last-child(n+5) { - font-size: 12px; - font-weight: 400; - opacity: 0.35; -} - -.bubble.user, -.bubble.assistant, -.bubble.tool { - align-self: flex-end; -} -.bubble.user { - color: rgba(190, 235, 251, 0.82); -} -.bubble.assistant { - color: rgba(214, 208, 255, 0.82); -} -.bubble.tool { - display: flex; - align-items: center; - justify-content: flex-end; - gap: 8px; - color: rgba(255, 255, 255, 0.5); - font-family: var(--font-mono); - font-size: 12px !important; - font-weight: 400 !important; - opacity: 0.7; -} -.bubble.tool .bubble-tool-icon { - width: 13px; - height: 13px; - flex: none; - color: var(--voice-tool); -} - -.bubble-role { - display: none; -} - -/* ─── Conversation history panel ─────────────────────────────────────── */ - -.chat-panel { - position: fixed; - inset: 0; - z-index: 200; - pointer-events: none; -} -.chat-panel-backdrop { - position: absolute; - inset: 0; - background: rgba(8, 9, 13, 0.45); - backdrop-filter: blur(6px); - -webkit-backdrop-filter: blur(6px); - opacity: 0; - transition: opacity 0.25s ease; -} -.chat-panel.open .chat-panel-backdrop { - opacity: 1; - pointer-events: auto; -} -.chat-panel-inner { - position: absolute; - top: 0; - right: 0; - bottom: 0; - width: min(360px, 90vw); - background: var(--bg-elev); - border-left: 1px solid var(--border); - display: flex; - flex-direction: column; - transform: translateX(100%); - transition: transform 0.28s cubic-bezier(0.32, 0.72, 0, 1); - pointer-events: auto; - box-shadow: -8px 0 32px rgba(0, 0, 0, 0.35); -} -.chat-panel.open .chat-panel-inner { - transform: translateX(0); -} -.chat-panel-header { - display: flex; - align-items: center; - justify-content: space-between; - padding: 18px 20px; - border-bottom: 1px solid var(--border); - flex: none; -} -.chat-panel-header h3 { - margin: 0; - font-size: 14px; - font-weight: 600; - letter-spacing: 0.01em; -} -.chat-history { - flex: 1; - overflow-y: auto; - padding: 16px 20px; - display: flex; - flex-direction: column; - gap: 10px; - scroll-behavior: smooth; -} -.chat-history::-webkit-scrollbar { width: 3px; } -.chat-history::-webkit-scrollbar-track { background: transparent; } -.chat-history::-webkit-scrollbar-thumb { background: var(--border-strong); border-radius: 3px; } - -.chat-empty { - flex: 1; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - gap: 6px; - color: var(--text-faint); - padding: 48px 0; - opacity: 0.65; -} -.chat-empty svg { - margin-bottom: 6px; - opacity: 0.8; -} -.chat-empty-title { - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - letter-spacing: 0.14em; - text-transform: uppercase; -} -.chat-empty-hint { - font-size: 12px; - color: var(--text-faint); - opacity: 0.7; -} - -/* ─── History messages ───────────────────────────────────────────────── */ - -.hist-msg { - display: flex; - flex-direction: column; - gap: 3px; - max-width: 88%; -} -.hist-msg.user { align-self: flex-start; } -.hist-msg.assistant { align-self: flex-end; } -.hist-msg.tool { align-self: flex-start; max-width: 100%; } - -.hist-role { - font-family: var(--font-mono); - font-size: 10px; - font-weight: 500; - letter-spacing: 0.1em; - text-transform: uppercase; - color: var(--text-faint); - padding: 0 2px; -} -.hist-msg.user .hist-role { color: color-mix(in srgb, var(--voice-user) 75%, var(--text-faint)); } -.hist-msg.assistant .hist-role { color: color-mix(in srgb, var(--voice-assistant) 75%, var(--text-faint)); } -.hist-msg.tool .hist-role { color: color-mix(in srgb, var(--voice-tool) 75%, var(--text-faint)); } - -.hist-body { - padding: 9px 12px; - border-radius: var(--radius-md); - font-size: 13px; - line-height: 1.5; - border: 1px solid var(--border); - background: var(--bg-elev-2); - word-break: break-word; -} -/* Bodies share one neutral surface; alignment + the mono role label do the - * distinguishing, so the panel reads as one quiet column. */ -.hist-msg.user .hist-body.partial { opacity: 0.65; } - -/* ─── Tool call history item ─────────────────────────────────────────── */ - -.hist-tool-header { - display: flex; - align-items: center; - gap: 8px; - padding: 8px 12px; - border-radius: var(--radius-md) var(--radius-md) 0 0; - background: color-mix(in srgb, var(--processing) 10%, var(--bg-elev-2)); - border: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); - border-bottom: 1px solid color-mix(in srgb, var(--processing) 15%, var(--border)); - cursor: pointer; - color: var(--text); - font-family: inherit; - font-size: 13px; - font-weight: 500; - width: 100%; - text-align: left; - transition: background 0.15s; -} -.hist-tool-header:only-child { - border-radius: var(--radius-md); - border-bottom: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); -} -.hist-tool-header:hover { - background: color-mix(in srgb, var(--processing) 16%, var(--bg-elev-2)); -} -.hist-tool-icon { - width: 13px; - height: 13px; - flex: none; - color: var(--processing); -} -.hist-tool-name { - font-family: var(--font-mono); - font-size: 12px; - color: var(--voice-tool); - font-weight: 600; -} -.hist-tool-chevron { - margin-left: auto; - width: 13px; - height: 13px; - color: var(--text-faint); - transition: transform 0.2s ease; - flex: none; -} -.hist-tool-header[aria-expanded="true"] .hist-tool-chevron { - transform: rotate(180deg); -} -.hist-tool-body { - padding: 10px 12px 12px; - border-radius: 0 0 var(--radius-md) var(--radius-md); - background: var(--bg); - border: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); - border-top: none; - display: none; -} -.hist-tool-body.open { display: block; } -.hist-tool-label { - font-family: var(--font-mono); - font-size: 10px; - letter-spacing: 0.08em; - text-transform: uppercase; - color: var(--text-faint); - margin: 8px 0 4px; -} -.hist-tool-label:first-child { margin-top: 0; } -.hist-tool-block { - font-family: var(--font-mono); - font-size: 11px; - line-height: 1.65; - color: var(--text-dim); - white-space: pre-wrap; - word-break: break-word; - overflow-x: auto; -} -.hist-tool-output { color: var(--text); } - -/* ─── Phone layout ──────────────────────────────────────────────────────── - * On phones the floating bubble stream overlaps the orb and there isn't room - * for it, so we drop it entirely and rely on the conversation panel (opened - * from the top-right chat button) as the single place to read the transcript. - * The badge still pulses there when new messages arrive while it's closed. - * We also stack the mic / stop controls above and below the orb (instead of - * left/right) so the wide live row never overflows, and let the panel take - * the full width. */ -@media (max-width: 600px) { - .bubble-stack { - display: none; - } - - .topbar { - padding: 14px 16px; - } - - .stage { - padding: 16px 12px 24px; - } - - /* Stack vertically: mic above the orb, stop below it (DOM order is - * mic → circle → stop). */ - .orb-wrap { - flex-direction: column; - gap: 14px; - } - - .circle { - /* Height-capped like the base rule so a narrow AND short box still fits. */ - width: clamp(130px, min(56vw, 52vh), 240px); - } - - /* In the column layout the side controls must collapse by HEIGHT, not - * width, so they take no vertical space until the session is live. */ - .side-btn { - width: 44px; - height: 0; - transition: opacity 0.25s ease, transform 0.25s ease, height 0.25s ease, - background 0.15s, color 0.15s, border-color 0.15s; - } - .side-btn svg { - width: 19px; - height: 19px; - } - .orb-wrap.live .side-btn { - width: 44px; - height: 44px; - } - .mic-gate-arc { width: 70px; height: 70px; } - - /* Full-screen conversation on phones — feels more deliberate than a - * narrow slide-over. It already spans top-to-bottom (inset 0); make it - * span edge-to-edge too and drop the now-pointless border/shadow. */ - .chat-panel-inner { - width: 100vw; - border-left: none; - box-shadow: none; - } -} - -/* ─── About panel ───────────────────────────────────────────────────────── - * Opened from the (i) by the wordmark. Reuses the .modal shell. - * Strictly monochrome per DESIGN.md — the only "color" is the orb, never - * here. Machine identifiers (model IDs, role tags, usernames) ride Geist - * Mono; everything human stays Inter. */ -.about-modal { - /* Roomier on tablet/desktop; the 92vw cap keeps phones full-width. */ - width: min(680px, 92vw); - max-height: 88vh; -} -.about-modal .modal-content { - max-height: 88vh; - overflow-y: auto; - gap: 20px; - padding: 26px 30px 24px; -} -.about-modal .modal-content::-webkit-scrollbar { width: 3px; } -.about-modal .modal-content::-webkit-scrollbar-track { background: transparent; } -.about-modal .modal-content::-webkit-scrollbar-thumb { background: var(--border-strong); border-radius: 3px; } - -/* Links here are quiet: no permanent underline (the global dotted rule is - * too loud for a dense credit block), brighten + underline on hover only. */ -.about-modal a { - color: var(--text); - border-bottom: none; - text-decoration: none; - transition: color 0.15s ease; -} -.about-modal a:hover { - color: #fff; -} -.about-intro a:hover { - text-decoration: underline; - text-underline-offset: 2px; -} -.about-modal .ext { - width: 12px; - height: 12px; - flex: none; - opacity: 0.5; -} - -/* Title row: the demo name and the (i) read as one unit. */ -.ident-head { - display: flex; - align-items: center; - gap: 9px; -} -/* (i) trigger sits right after the title. A faint outline keeps it - * catchable without turning into a card; it fills in on hover. */ -.about-btn { - flex: none; - width: 34px; - height: 34px; - border-radius: 50%; - background: transparent; - border: 1px solid var(--border-strong); - color: var(--text-dim); - display: inline-flex; - align-items: center; - justify-content: center; - cursor: pointer; - touch-action: manipulation; - transition: background 0.15s ease, color 0.15s ease, border-color 0.15s ease; -} -.about-btn:hover { - background: var(--bg-elev); - border-color: var(--text-dim); - color: var(--text); -} -.about-btn svg { - width: 20px; - height: 20px; -} - -.about-btn-mobile { - display: inline-flex; -} - -/* ── Popup intro: a plain paragraph on the project + a repo link ── */ -.about-intro p { - margin: 0; - font-size: 14px; - line-height: 1.6; - color: var(--text-dim); -} -.about-repo { - display: inline-flex; - align-items: center; - gap: 5px; - margin-top: 12px; - font-size: 14px; - font-weight: 500; -} - -/* ── Corner identity (replaces the wordmark) ── - * A compact stack in the topbar: title, one-line blurb, two meta rows. - * Monochrome; only the role/name identifiers ride the machine typeface. */ -.ident { - display: flex; - flex-direction: column; - gap: 10px; - min-width: 0; -} -/* Reset the global underlined-anchor styling inside the identity block. */ -.ident a { - border-bottom: none; - color: inherit; - transition: color 0.15s ease; -} -.ident a:hover { - color: var(--text); - text-decoration: underline; - text-underline-offset: 2px; -} -.ident-title { - font-size: 22px; - font-weight: 600; - letter-spacing: 0.005em; - line-height: 1.1; - color: var(--text); -} -.ident-blurb { - margin: 0; - max-width: 48ch; - font-size: 14.5px; - line-height: 1.5; - font-weight: 400; - color: var(--text-dim); -} -.ident-meta { - display: flex; - flex-direction: column; - gap: 7px; - font-size: 14px; - font-weight: 400; - color: var(--text-dim); -} -.ident-row { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 4px 8px; -} -.ident-label { - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - letter-spacing: 0.1em; - text-transform: uppercase; - color: var(--text-faint); -} - -/* Shared credit bits — now used in the corner identity block. */ -.sep { - color: var(--text-faint); - opacity: 0.6; -} -.hf-credit { - display: inline-flex; - align-items: center; - gap: 5px; -} -/* Brand marks keep their own color — a deliberate exception to the - * monochrome rule, for the HF and Cerebras logos only. */ -.hf-mark { - width: 14px; - height: 14px; - flex: none; - color: #ffd21e; -} -.cerebras-credit { - display: inline-flex; - align-items: center; - gap: 5px; -} -.cerebras-mark { - width: 14px; - height: 14px; - flex: none; -} -/* Usernames are identifiers, so they ride the machine typeface. */ -.handle { - font-family: var(--font-mono); - font-size: 13.5px; -} - -/* ── Pipeline (the signal flow) ── */ -.about-pipeline { - border-top: 1px solid var(--border); - padding-top: 16px; -} -.pipeline-title { - margin: 0 0 14px; - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - letter-spacing: 0.14em; - text-transform: uppercase; - color: var(--text-faint); -} -.pipeline { - list-style: none; - margin: 0; - padding: 0; - position: relative; -} -/* One continuous rail threading every node, dot centers at x=8.5px. */ -.pipeline::before { - content: ""; - position: absolute; - left: 8px; - top: 7px; - bottom: 7px; - width: 1px; - background: var(--border-strong); -} -.pipeline > li { - position: relative; - padding-left: 30px; -} -/* Stages: solid node. Endpoints (you / orb): hollow node. */ -.pipe-stage::before { - content: ""; - position: absolute; - left: 5px; - top: 4px; - width: 7px; - height: 7px; - border-radius: 50%; - background: var(--text-dim); -} -.pipe-endpoint::before { - content: ""; - position: absolute; - left: 5px; - top: 3px; - width: 7px; - height: 7px; - border-radius: 50%; - border: 1px solid var(--text-faint); - background: var(--bg-elev); -} -.pipe-endpoint { - font-family: var(--font-mono); - font-size: 11px; - font-weight: 500; - letter-spacing: 0.12em; - text-transform: uppercase; - color: var(--text-faint); - padding-bottom: 12px; -} -.pipeline > li.pipe-endpoint:last-child { - padding-bottom: 0; -} -.pipe-stage { - padding-bottom: 16px; -} -.pipe-tag { - font-family: var(--font-mono); - font-size: 12px; - font-weight: 600; - letter-spacing: 0.08em; - color: var(--text); - margin-right: 9px; -} -.pipe-job { - font-size: 14px; - color: var(--text-dim); -} -/* Middot between the job and its model link, matching the separators - * used elsewhere. */ -.pipe-job::after { - content: "·"; - margin: 0 7px; - color: var(--text-faint); -} -.pipe-note { - color: var(--text-faint); -} -/* The Cerebras link stays as quiet as the note; brightens + underlines on hover. */ -.pipe-note a { - color: inherit; -} -.pipe-note a:hover { - color: var(--text); - text-decoration: underline; - text-underline-offset: 2px; -} -.pipe-model { - display: inline-flex; - align-items: center; - gap: 4px; - margin-top: 4px; - font-family: var(--font-mono); - font-size: 12px; - color: var(--text-dim); - word-break: break-all; -} -.pipe-model:hover { - color: #fff; - text-decoration: underline; - text-underline-offset: 2px; -} -.pipe-model .ext { - width: 11px; - height: 11px; -} - -/* "Interrupted" tag on an assistant reply the user barged in on. */ -.hist-note { - font-family: var(--font-mono); - font-size: 10px; - font-weight: 500; - letter-spacing: 0.1em; - text-transform: uppercase; - color: var(--text-faint); - padding: 0 2px; -} -.hist-msg.assistant .hist-note { - align-self: flex-end; -} -.hist-msg.interrupted .hist-body { - opacity: 0.7; -} - -/* Captured webcam frame shown in the transcript (camera tool result). */ -.hist-image { - display: block; - width: 100%; - max-width: 240px; - border-radius: var(--radius-sm); - border: 1px solid var(--border); - margin-top: 2px; -} - -/* ─── Tools panel ────────────────────────────────────────────────────────── - * Reuses the modal/field shell. Switches are monochrome (checked = near-white, - * the same high-contrast treatment as the primary button): color belongs to - * the voice, not the chrome. */ -.tools-intro { - margin: 0; - font-size: 13px; - line-height: 1.5; - color: var(--text-dim); -} -.tool-list { - display: flex; - flex-direction: column; -} -.tool-row { - display: flex; - align-items: center; - justify-content: space-between; - gap: 16px; - padding: 12px 0; -} -.tool-row-sep { - border-top: 1px solid var(--border); - margin-top: 4px; -} -.tool-info { - display: flex; - flex-direction: column; - gap: 3px; - min-width: 0; -} -.tool-name { - font-size: 14px; - font-weight: 600; - color: var(--text); -} -.tool-desc { - font-size: 12.5px; - color: var(--text-dim); -} -.tool-row.disabled .tool-name, -.tool-row.disabled .tool-desc { - opacity: 0.5; -} -.tool-hint { - display: block; - font-size: 12px; - line-height: 1.4; - color: var(--text-faint); -} -.tools-key { - margin: 0 0 4px; -} - -/* Toggle switch */ -.switch { - position: relative; - display: inline-flex; - flex: none; - width: 40px; - height: 24px; - cursor: pointer; -} -.switch input { - position: absolute; - inset: 0; - width: 100%; - height: 100%; - margin: 0; - opacity: 0; - cursor: pointer; -} -.switch-track { - position: absolute; - inset: 0; - border-radius: 999px; - background: var(--bg); - border: 1px solid var(--border-strong); - transition: background 0.15s, border-color 0.15s; -} -.switch-track::after { - content: ""; - position: absolute; - top: 50%; - left: 3px; - transform: translateY(-50%); - width: 16px; - height: 16px; - border-radius: 50%; - background: var(--text-dim); - transition: transform 0.18s ease, background 0.15s; -} -.switch input:checked + .switch-track { - background: var(--text); - border-color: var(--text); -} -.switch input:checked + .switch-track::after { - transform: translate(16px, -50%); - background: var(--bg); -} -.switch input:focus-visible + .switch-track { - outline: 2px solid var(--text-dim); - outline-offset: 2px; -} -.switch input:disabled { - cursor: not-allowed; -} -.switch input:disabled + .switch-track { - opacity: 0.5; -} - -/* ─── Webcam preview (camera tool) ────────────────────────────────────────── - * Floating self-view, bottom-left. Mirrored for the user; the frame sent to - * the model is drawn un-mirrored (see captureSnapshot). */ -.cam-pip { - position: fixed; - left: 20px; - bottom: 20px; - width: 280px; - aspect-ratio: 4 / 3; - border-radius: var(--radius-md); - overflow: hidden; - border: 1px solid var(--border-strong); - background: var(--bg-elev); - box-shadow: var(--shadow-soft); - z-index: 90; - opacity: 0; - transform: translateY(8px) scale(0.96); - pointer-events: none; - transition: opacity 0.22s ease, transform 0.22s ease; -} -.cam-pip.visible { - opacity: 1; - transform: none; -} -.cam-video { - display: block; - width: 100%; - height: 100%; - object-fit: cover; - transform: scaleX(-1); /* mirror the self-view only */ - background: var(--bg); -} -.cam-label { - position: absolute; - left: 8px; - bottom: 6px; - font-family: var(--font-mono); - font-size: 9.5px; - letter-spacing: 0.14em; - text-transform: uppercase; - color: var(--text); - opacity: 0.8; - text-shadow: 0 1px 3px rgba(0, 0, 0, 0.7); -} -.cam-flash { - position: absolute; - inset: 0; - background: #fff; - opacity: 0; - pointer-events: none; -} -.cam-pip.flash .cam-flash { - animation: cam-flash 0.4s ease; -} -@keyframes cam-flash { - 0% { opacity: 0; } - 12% { opacity: 0.85; } - 100% { opacity: 0; } -} -@media (max-width: 600px) { - /* Bottom-centred on phones. Auto margins centre it without touching the - * transform, so the slide/scale-in animation still works. */ - .cam-pip { - left: 0; - right: 0; - margin-inline: auto; - bottom: 16px; - width: min(188px, 52vw); - } - /* The credit would sit under the centred preview, so drop it while the - * camera is on. */ - body.cam-on .footer { - display: none; - } -} -@media (prefers-reduced-motion: reduce) { - .cam-pip { - transition: opacity 0.22s ease; - transform: none; - } - .cam-pip.flash .cam-flash { - animation: none; - } -} - -/* ─── Desktop type scale ──────────────────────────────────────────────────── - * A uniform step up (~+1px, title +2) for all UI text on larger screens. - * Scoped to min-width: 601px so it can't reach phones — the ≤600px layout - * keeps every size exactly as it was. */ -@media (min-width: 601px) { - .cam-label { font-size: 10.5px; } - - .bubble-role, - .hist-role { font-size: 11px; } - - .circle-caption, - .footer, - .chat-empty-title, - .hist-tool-block, - .ident-label, - .pipeline-title, - .pipe-endpoint { font-size: 12px; } - - .field > span, - .field small, - .bubble.tool, - .chat-empty-hint, - .hist-tool-name, - .pipe-tag, - .pipe-model, - .tool-hint { font-size: 13px; } - - .tool-desc { font-size: 13.5px; } - - .btn, - .bubble, - .field, - .hist-body, - .hist-tool-header, - .tools-intro { font-size: 14px; } - - .handle { font-size: 14.5px; } - - .about-intro p, - .about-repo, - .brand, - .chat-panel-header h3, - .field textarea, - .ident-meta, - .pipe-job, - .tool-name { font-size: 15px; } - - .ident-blurb { font-size: 15.5px; } - - .modal-header h2 { font-size: 17px; } - - .ident-title { font-size: 24px; } -} - -/* ─── Short viewports: fit without a vertical scrollbar ───────────────────── - * The base media queries only react to WIDTH, so a short container (e.g. the - * app embedded in a small box, or a laptop window once the browser chrome eats - * into the viewport) used to overflow vertically and get clipped. We KEEP the - * identity header, orb, caption and credits — we just undo the desktop type - * bump, tighten the vertical rhythm and let the height-capped orb shrink, so - * everything fits with comfortable margin. No opt-in needed. - * - * The single biggest space eater is the identity block (~180px at the desktop - * type scale), so this tier compacts it first. */ -@media (max-height: 780px) { - .topbar { padding: 14px 24px; } - .ident { gap: 5px; } - .ident-title { font-size: 20px; } - .ident-blurb { font-size: 13.5px; line-height: 1.4; } - .ident-meta { gap: 5px; font-size: 13px; } - .stage { padding: 12px 20px 16px; gap: 12px; } - .footer { padding: 8px 24px 10px; } - .circle { width: clamp(140px, min(38vw, 44vh), 300px); } -} - -@media (max-height: 560px) { - .topbar { padding: 8px 16px; } - .ident-title { font-size: 17px; } - /* Below this, room is tight enough that the one-line blurb and the footer - * (a verbatim duplicate of the header's "Powered by" credit) are dropped so - * the title + credits still read; the rest of the header stays. */ - .ident-blurb { display: none; } - .stage { padding: 8px 14px 10px; gap: 8px; } - .footer { display: none; } - .circle { width: clamp(120px, min(50vw, 46vh), 240px); } -} - -/* ─── Digital-human avatar as full-viewport background ────────────────── */ - -.avatar-stage { - position: fixed; - inset: 0; - width: 100vw; - height: 100vh; - z-index: 0; - overflow: hidden; - background: #111; -} - -.avatar-stage iframe { - width: 100%; - height: 100%; - border: none; - display: block; - pointer-events: none; -} diff --git a/public/voice-demo/ui/account.js b/public/voice-demo/ui/account.js deleted file mode 100644 index dc2d5b2..0000000 --- a/public/voice-demo/ui/account.js +++ /dev/null @@ -1,189 +0,0 @@ -// @ts-check -/** - * Account — the HF login chip and the daily-limit modal. - * - * Reads `/api/me` to learn the current tier (anonymous / signed-in / PRO) and - * remaining daily talk-time, renders a sign-in pill or a signed-in chip with a - * small popover (tier, remaining, sign out, upgrade), and shows the limit modal - * when a conversation is refused or cut. The time metering itself lives in - * main.js (heartbeat loop) + the server; this module is just the surface. - * - * Inert unless the deploy is in LB mode (`/api/me` → `{enabled:true}`). - */ - -import { $, escHtml } from "./dom.js"; - -const PRO_URL = "https://huggingface.co/subscribe/pro"; - -// Official multi-color Hugging Face logo, used in the badge + sign-in CTA. -const HF_MARK = ``; - -/** @param {number} sec @returns {string} "m:ss" */ -function fmt(sec) { - const s = Math.max(0, Math.round(sec)); - return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`; -} - -export class Account { - constructor() { - /** @type {HTMLElement} */ - this._root = $("#account"); - /** @type {HTMLDialogElement} */ - this._modal = $("#limit-modal"); - this._modalTitle = $("#limit-title"); - this._modalMsg = $("#limit-msg"); - this._modalNote = $("#limit-note"); - /** @type {HTMLAnchorElement} */ - this._modalCta = /** @type {any} */ ($("#limit-cta")); - - /** @type {{enabled:boolean, auth?:boolean, loggedIn?:boolean, username?:string, avatar?:string, tier?:string, remainingSec?:number|null, limitSec?:number|null, loginUrl?:string|null, logoutUrl?:string|null}} */ - this._me = { enabled: false }; - this._popoverOpen = false; - - $("#limit-close").addEventListener("click", () => this._modal.close()); - this._modal.addEventListener("click", (e) => { - if (e.target === this._modal) this._modal.close(); - }); - // Close the popover on an outside click. - document.addEventListener("click", (e) => { - if (this._popoverOpen && !this._root.contains(/** @type {Node} */ (e.target))) { - this._closePopover(); - } - }); - } - - get tier() { - return this._me.tier || "anon"; - } - - /** Fetch `/api/me` and (re)render the chip. Safe to call repeatedly (load, - * after the OAuth redirect, after a conversation ends). */ - async refresh() { - try { - const res = await fetch("api/me"); - this._me = res.ok ? await res.json() : { enabled: false }; - } catch { - this._me = { enabled: false }; - } - this._render(); - } - - _render() { - const me = this._me; - if (!me.enabled) { - this._root.hidden = true; - this._root.innerHTML = ""; - return; - } - this._root.hidden = false; - - if (!me.loggedIn) { - // Signed-out: a sign-in pill (only when OAuth is actually available). - if (me.auth && me.loginUrl) { - this._root.innerHTML = ``; - } else { - this._root.innerHTML = ""; - this._root.hidden = true; - } - return; - } - - // Signed-in: avatar + handle chip that toggles a popover. - const isPro = me.tier === "pro"; - // Org members get unlimited usage too, but aren't PRO — don't brand them so. - const isUnlimited = isPro || me.tier === "org"; - const avatar = me.avatar - ? `` - : ``; - const remaining = - isUnlimited || me.remainingSec == null - ? "Unlimited" - : `${fmt(me.remainingSec)} left today`; - const tierLabel = isPro ? "PRO" : isUnlimited ? "Team" : "Free"; - - this._root.innerHTML = ` - - `; - - const chip = $("#account-chip"); - chip.addEventListener("click", (e) => { - e.stopPropagation(); - this._popoverOpen ? this._closePopover() : this._openPopover(); - }); - } - - _openPopover() { - const pop = document.getElementById("account-pop"); - const chip = document.getElementById("account-chip"); - if (!pop || !chip) return; - pop.hidden = false; - chip.setAttribute("aria-expanded", "true"); - this._popoverOpen = true; - } - - _closePopover() { - const pop = document.getElementById("account-pop"); - const chip = document.getElementById("account-chip"); - if (pop) pop.hidden = true; - if (chip) chip.setAttribute("aria-expanded", "false"); - this._popoverOpen = false; - } - - /** - * Show the limit modal for a tier — used both when a conversation is refused - * at start (402) and when a live one is cut (heartbeat `expired`). - * @param {string} [tier] - */ - showLimit(tier = this.tier) { - const canSignIn = this._me.auth && this._me.loginUrl; - this._modalTitle.textContent = "Thanks for chatting!"; - if (tier === "anon") { - this._modalMsg.textContent = - "Guest conversations run for 5 minutes. Sign in with Hugging Face to get 10 minutes a day for free, and PRO members chat with no limit at all."; - this._modalNote.textContent = "Your free minutes refresh tomorrow."; - if (canSignIn) { - this._modalCta.innerHTML = `${HF_MARK}Sign in with Hugging Face`; - this._modalCta.href = /** @type {string} */ (this._me.loginUrl); - this._modalCta.hidden = false; - } else { - this._modalCta.hidden = true; - } - } else { - // Signed-in, non-PRO. - this._modalMsg.textContent = - "You've enjoyed your 10 minutes for today. Go PRO for unlimited conversations and to support open source AI."; - this._modalNote.textContent = "Or come back tomorrow. Your minutes reset daily."; - this._modalCta.innerHTML = "Upgrade to PRO"; - this._modalCta.href = PRO_URL; - this._modalCta.hidden = false; - } - if (!this._modal.open) this._modal.showModal(); - } - - /** Show a warm "we're at capacity" message when even the waiting line is full. - * Reuses the limit modal shell; no call-to-action, just reassurance. */ - showBusy() { - this._modalTitle.textContent = "Hugged to the limit 🤗"; - this._modalMsg.textContent = - "Every slot and the whole line are full right now. Too much love! Grab a coffee and pop back in a minute."; - this._modalNote.textContent = "A spot usually opens up within a few minutes."; - this._modalCta.hidden = true; - if (!this._modal.open) this._modal.showModal(); - } -} diff --git a/public/voice-demo/ui/chat.js b/public/voice-demo/ui/chat.js deleted file mode 100644 index b54a672..0000000 --- a/public/voice-demo/ui/chat.js +++ /dev/null @@ -1,422 +0,0 @@ -// @ts-check -/** - * ChatView — owns the whole conversation surface: the slide-in history panel, - * the ephemeral on-orb bubbles, and all the transcript/tool/streaming - * bookkeeping. main.js wires the realtime client's events straight to the - * `on*` methods here and otherwise doesn't touch chat state. - * - * Two parallel surfaces share one shape (see `_buildMessageEl`): - * - ephemeral bubbles (`.bubble` / `.bubble-*`) fade on a timer - * - persistent history (`.hist-msg` / `.hist-*`) the durable panel log - * - * Keying: - * - user transcripts by the server's `item_id` — a speculative continuation - * REUSES it, so both segments land in one row/bubble; deltas are CUMULATIVE - * (each carries the full sentence so far), so we replace text wholesale. - * - assistant transcripts by `response_id`, so a cancelled speculative reply - * can be marked interrupted without erasing what was already shown. - */ - -import { $, escHtml, DEBUG } from "./dom.js"; - -const WRENCH_PATH = ``; -const CHAT_BUBBLE_SVG = ``; -const EMPTY_STATE_HTML = `
${CHAT_BUBBLE_SVG}No messages yetTap the orb and start talking
`; - -export class ChatView { - constructor() { - /** @type {HTMLButtonElement} */ - this._chatBtn = $("#chat-btn"); - /** @type {HTMLSpanElement} */ - this._chatBadge = $("#chat-badge"); - /** @type {HTMLDivElement} */ - this._chatPanel = $("#chat-panel"); - /** @type {HTMLDivElement} */ - this._chatPanelBackdrop = $("#chat-panel-backdrop"); - /** @type {HTMLButtonElement} */ - this._chatPanelClose = $("#chat-panel-close"); - /** @type {HTMLDivElement} */ - this._chatHistory = $("#chat-history"); - /** @type {HTMLDivElement} */ - this._bubbleStack = $("#bubble-stack"); - - this._panelOpen = false; - this._scrollQueued = false; - - // ── User transcript state (keyed by item_id) ─────────────────────────── - /** @type {Map} */ - this._userHistByItem = new Map(); - /** @type {HTMLElement | null} */ - this._activeUserBubble = null; - this._activeUserItemId = ""; - // Monotonic counter for synthesizing unique keys when the server omits an - // item_id / response_id, so id-less messages never collapse onto each other. - this._anonSeq = 0; - - // ── Assistant transcript state (keyed by response_id) ────────────────── - /** @type {Map} */ - this._asstByResp = new Map(); - - // ── Ephemeral bubble auto-dismiss ────────────────────────────────────── - // Per-element expiry (epoch ms). A bubble fades once its expiry passes — - // but only in stack order (see _reapBubbles). Refreshing the expiry keeps a - // bubble alive while it updates (e.g. the live user utterance). - /** @type {WeakMap} */ - this._bubbleExpiry = new WeakMap(); - // Single pending reaper handle: one timer for the whole stack (not one per - // bubble) so dismissal is strictly oldest-first regardless of per-bubble delays. - this._reaperHandle = 0; - - this._chatBtn.addEventListener("click", () => (this._panelOpen ? this._closePanel() : this._openPanel())); - this._chatPanelClose.addEventListener("click", () => this._closePanel()); - this._chatPanelBackdrop.addEventListener("click", () => this._closePanel()); - document.addEventListener("keydown", (e) => { - if (e.key === "Escape" && this._panelOpen) this._closePanel(); - }); - } - - // ── Panel ─────────────────────────────────────────────────────────────── - - _openPanel() { - this._panelOpen = true; - this._chatPanel.classList.add("open"); - this._chatBadge.classList.remove("visible"); - this._scrollToBottom(); - } - - _closePanel() { - this._panelOpen = false; - this._chatPanel.classList.remove("open"); - } - - // Coalesce scroll-to-bottom: a burst of cumulative transcript deltas would - // otherwise queue one rAF per delta, all writing the same scrollTop. - _scrollToBottom() { - if (!this._panelOpen || this._scrollQueued) return; - this._scrollQueued = true; - requestAnimationFrame(() => { - this._scrollQueued = false; - this._chatHistory.scrollTop = this._chatHistory.scrollHeight; - }); - } - - _markUnread() { - if (this._panelOpen) { - this._scrollToBottom(); - return; - } - this._chatBadge.classList.add("visible"); - } - - // ── Shared rendering ────────────────────────────────────────────────────── - - /** - * Build a role-labelled message element. Ephemeral bubbles and persistent - * history rows share the same shape and differ only in their class prefix - * (`bubble`/`bubble-*` vs `hist-msg`/`hist-*`). - * @param {{ container: string, prefix: string, role: "user"|"assistant", text: string, partial?: boolean }} o - * @returns {HTMLElement} - */ - _buildMessageEl({ container, prefix, role, text, partial = false }) { - const el = document.createElement("div"); - el.className = `${container} ${role}`; - const label = role === "user" ? "You" : "Assistant"; - el.innerHTML = `
${label}
${escHtml(text)}
`; - return el; - } - - // ── Ephemeral bubbles ─────────────────────────────────────────────────── - - /** @param {"user"|"assistant"|"tool"} role @param {string} text @returns {HTMLElement} */ - _spawnBubble(role, text) { - let el; - if (role === "tool") { - el = document.createElement("div"); - el.className = "bubble tool"; - el.innerHTML = `${WRENCH_PATH}${escHtml(text)}`; - } else { - el = this._buildMessageEl({ container: "bubble", prefix: "bubble", role, text }); - } - this._bubbleStack.appendChild(el); - // Cap the stack at 8, but never evict the bubble the caller is still - // actively updating (the live user bubble) — drop the next-oldest instead. - const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]); - if (visible.length > 8) { - this._dismissBubble(visible.find((b) => b !== this._activeUserBubble) ?? visible[0]); - } - requestAnimationFrame(() => el.classList.add("in")); - return el; - } - - /** @param {HTMLElement} el @param {string} text */ - _updateBubbleText(el, text) { - const t = el.querySelector(".bubble-body"); - if (t) t.textContent = text; - } - - /** @param {HTMLElement} el */ - _dismissBubble(el) { - if (!el || el.classList.contains("out")) return; // idempotent - this._bubbleExpiry.delete(el); - el.classList.remove("in"); - el.classList.add("out"); - const remove = () => el.remove(); - el.addEventListener("transitionend", remove, { once: true }); - // Fallback: transitionend never fires if the bubble's visual state didn't - // change (dismissed pre-paint) or the tab is backgrounded. The transition - // is 0.3s, so force removal a little after. - setTimeout(remove, 400); - } - - /** - * Fade bubbles whose expiry has passed — strictly oldest-first. We walk the - * stack top (oldest) to bottom (newest) and stop at the first bubble still - * alive: nothing newer may leave while an older bubble is still on screen. A - * bubble that keeps updating pushes its own expiry forward, so it (and - * everything behind it) stays put until it finally goes quiet. - */ - _reapBubbles() { - this._reaperHandle = 0; - const now = Date.now(); - const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]); - let nextWake = Infinity; - for (const el of visible) { - const exp = this._bubbleExpiry.get(el) ?? now; // no expiry recorded → treat as due - if (exp <= now) { - this._dismissBubble(el); - } else { - // Oldest survivor isn't due yet; stop so nothing newer leaves before it. - nextWake = exp; - break; - } - } - if (nextWake !== Infinity) { - this._reaperHandle = setTimeout(() => this._reapBubbles(), Math.max(50, nextWake - Date.now())); - } - } - - /** - * (Re)arm a bubble's auto-dismiss by pushing its expiry out by `delay`. - * Calling it again resets the countdown — so a bubble that keeps updating - * stays on screen and only fades once it goes quiet. Removal is ordered by the - * shared reaper, so the oldest bubble always disappears first. - * @param {HTMLElement} el @param {number} [delay] - */ - _bumpDismiss(el, delay = 4000) { - this._bubbleExpiry.set(el, Date.now() + delay); - if (!this._reaperHandle) this._reaperHandle = setTimeout(() => this._reapBubbles(), delay); - } - - // ── History ─────────────────────────────────────────────────────────────── - - /** Render the empty-state placeholder into the history panel. */ - renderEmptyState() { - this._chatHistory.innerHTML = EMPTY_STATE_HTML; - } - - /** Reset the panel to the empty state and clear the unread badge. */ - clear() { - this.renderEmptyState(); - this._chatBadge.classList.remove("visible"); - } - - /** @param {"user"|"assistant"} role @param {string} text @param {boolean} partial @returns {HTMLElement} */ - _appendHistMsg(role, text, partial) { - const empty = this._chatHistory.querySelector(".chat-empty"); - if (empty) empty.remove(); - const el = this._buildMessageEl({ container: "hist-msg", prefix: "hist", role, text, partial }); - this._chatHistory.appendChild(el); - this._scrollToBottom(); - return el; - } - - /** @param {HTMLElement | null} el @param {string} text @param {boolean} partial */ - _updateHistMsg(el, text, partial) { - if (!el) return; - const body = /** @type {HTMLElement | null} */ (el.querySelector(".hist-body")); - if (!body) return; - body.textContent = text; - body.classList.toggle("partial", partial); - this._scrollToBottom(); - } - - /** - * Append a tool-call row to the conversation. We only add it once the tool - * has run, so the expandable toggle carries BOTH the call input and its result. - * @param {string} name @param {string} argsJson @param {string} output - */ - _appendHistTool(name, argsJson, output) { - const empty = this._chatHistory.querySelector(".chat-empty"); - if (empty) empty.remove(); - let pretty = argsJson; - try { pretty = JSON.stringify(JSON.parse(argsJson), null, 2); } catch {} - const el = document.createElement("div"); - el.className = "hist-msg tool"; - el.innerHTML = ` -
Tool call
- -
-
Input
-
${escHtml(pretty)}
-
Output
-
${escHtml(output || "(no output)")}
-
- `; - const header = /** @type {HTMLButtonElement} */ (el.querySelector(".hist-tool-header")); - const body = /** @type {HTMLDivElement} */ (el.querySelector(".hist-tool-body")); - header.addEventListener("click", () => { - const expanded = header.getAttribute("aria-expanded") === "true"; - header.setAttribute("aria-expanded", String(!expanded)); - body.classList.toggle("open", !expanded); - }); - this._chatHistory.appendChild(el); - this._scrollToBottom(); - } - - /** Tag an assistant history row as interrupted (user barged in mid-reply). - * @param {HTMLElement | null} hist */ - _markHistInterrupted(hist) { - if (!hist || hist.querySelector(".hist-note")) return; - hist.classList.add("interrupted"); - const note = document.createElement("div"); - note.className = "hist-note"; - note.textContent = "Interrupted"; - hist.appendChild(note); - } - - /** Render a captured webcam frame in the transcript (the camera tool result). - * @param {string} dataUrl */ - _appendHistImage(dataUrl) { - const empty = this._chatHistory.querySelector(".chat-empty"); - if (empty) empty.remove(); - const el = document.createElement("div"); - el.className = "hist-msg tool"; - el.innerHTML = `
Snapshot
Webcam snapshot sent to the model`; - const img = /** @type {HTMLImageElement} */ (el.querySelector("img")); - img.src = dataUrl; - this._chatHistory.appendChild(el); - this._scrollToBottom(); - } - - /** - * Reset all streaming bookkeeping for session start / teardown. Pass - * `dismiss` to also fade any bubbles still on screen. - * @param {{ dismiss?: boolean }} [opts] - */ - reset(opts) { - if (opts?.dismiss) { - if (this._activeUserBubble) this._dismissBubble(this._activeUserBubble); - for (const { bubble } of this._asstByResp.values()) this._dismissBubble(bubble); - } - this._userHistByItem.clear(); - this._activeUserBubble = null; - this._activeUserItemId = ""; - this._asstByResp.clear(); - } - - // ── Client event handlers ───────────────────────────────────────────────── - - /** - * A streamed transcript delta (user or assistant). - * @param {{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }} d - */ - onTranscript(d) { - if (DEBUG) console.debug(`[ui] transcript role=${d.role} partial=${d.partial} item=${d.itemId} resp=${d.responseId} text=${JSON.stringify(d.text)}`); - - if (d.role === "user") { - // Group by item_id: a speculative continuation reuses the same id, so it - // updates the same row/bubble. A missing id falls back to the active item - // (same utterance) or a fresh unique key, never a shared sentinel that - // would collapse distinct turns into one row. - const id = d.itemId || this._activeUserItemId || `_u${++this._anonSeq}`; - const text = d.text; - - let hist = this._userHistByItem.get(id); - if (!hist) { - hist = this._appendHistMsg("user", text, d.partial); - this._userHistByItem.set(id, hist); - } else { - this._updateHistMsg(hist, text, d.partial); - } - - // One ephemeral bubble per active item. Purely timer-based: the timer is - // refreshed on every delta, so it stays while the user keeps talking and - // fades a few seconds after they stop — no dependency on a response ever - // arriving, so it can never get stuck. - if (this._activeUserItemId !== id || !this._activeUserBubble) { - this._activeUserBubble = this._spawnBubble("user", text); - this._activeUserItemId = id; - } else { - this._updateBubbleText(this._activeUserBubble, text); - } - this._markUnread(); - } else if (d.role === "assistant") { - // Assistant transcript arrives once, as the full text, keyed by - // response_id so a cancelled speculative response can be removed later. A - // missing id gets a unique key so two id-less replies never collide. - const rid = d.responseId || `_a${++this._anonSeq}`; - const entry = this._asstByResp.get(rid); - if (!entry) { - const bubble = this._spawnBubble("assistant", d.text); - this._asstByResp.set(rid, { bubble, hist: this._appendHistMsg("assistant", d.text, false) }); - } else { - this._updateBubbleText(entry.bubble, d.text); - this._updateHistMsg(entry.hist, d.text, false); - } - this._markUnread(); - } - } - - /** - * A response closed (completed or cancelled). - * @param {{ responseId: string; status: string; audible?: boolean; transcript?: string }} detail - */ - onResponseFinished(detail) { - const { responseId, status, audible, transcript } = detail; - if (DEBUG) console.debug(`[ui] response-finished resp=${responseId} status=${status} audible=${audible} known=${this._asstByResp.has(responseId)}`); - // Without an id we can't target a specific response; the bubble will - // auto-dismiss on its own timer regardless. - if (!responseId) return; - const entry = this._asstByResp.get(responseId); - - if (status === "cancelled") { - // Keep every transcript that was received — mark it interrupted rather - // than erasing it. If the `*.transcript.done` never fired, build the row - // from the text carried in response.done. - let hist = entry?.hist ?? null; - if (!hist && transcript) { - hist = this._appendHistMsg("assistant", transcript, false); - } else if (hist && transcript) { - this._updateHistMsg(hist, transcript, false); - } - if (hist) this._markHistInterrupted(hist); - this._asstByResp.delete(responseId); - return; - } - - // Any other terminal close (completed / failed / incomplete / …): just - // release the map entry. The bubble already auto-dismisses on its timer and - // the history row persists as the conversation log. Crucially we do NOT - // touch user state here — that lifecycle is fully independent. - this._asstByResp.delete(responseId); - } - - /** The model called a tool — show an ephemeral "running" bubble. - * @param {string} name */ - onToolCall(name) { - this._bumpDismiss(this._spawnBubble("tool", name)); - this._markUnread(); - } - - /** The tool finished — append its call+result row (and any captured image). - * @param {string} name @param {string} argsJson @param {string} output @param {string} [image] */ - onToolResult(name, argsJson, output, image) { - this._appendHistTool(name, argsJson, output); - if (image) this._appendHistImage(image); // show the captured frame below the call - this._markUnread(); - } -} diff --git a/public/voice-demo/ui/dom.js b/public/voice-demo/ui/dom.js deleted file mode 100644 index d5a645a..0000000 --- a/public/voice-demo/ui/dom.js +++ /dev/null @@ -1,37 +0,0 @@ -// @ts-check -/** Small shared helpers used across the UI modules: a strict query selector, - * HTML escaping for text we drop into innerHTML, error-string trimming, and - * the opt-in debug flag. */ - -/** Opt-in tracing: `localStorage.setItem("s2s.debug", "1")` then reload. */ -export const DEBUG = (() => { - try { - return localStorage.getItem("s2s.debug") === "1"; - } catch { - return false; - } -})(); - -/** - * Query a single element, throwing if it's missing (so a broken selector fails - * loudly at startup rather than as a later null-deref). - * @template {HTMLElement} T - * @param {string} selector - * @returns {T} - */ -export function $(selector) { - const el = document.querySelector(selector); - if (!el) throw new Error(`Missing element: ${selector}`); - return /** @type {T} */ (el); -} - -/** @param {string} s @returns {string} */ -export function escHtml(s) { - return s.replace(/&/g, "&").replace(//g, ">"); -} - -/** Trim a long error message to fit the orb caption. @param {string} text */ -export function truncateError(text) { - if (text.length <= 90) return text; - return text.slice(0, 87) + "…"; -} diff --git a/public/voice-demo/worklets/audio-playback.js b/public/voice-demo/worklets/audio-playback.js deleted file mode 100644 index 469b904..0000000 --- a/public/voice-demo/worklets/audio-playback.js +++ /dev/null @@ -1,211 +0,0 @@ -// @ts-check -/** - * AudioWorkletProcessor that plays back Float32 mono samples received from - * the main thread, upsampling whatever incoming rate the server uses - * (typically 16 kHz PCM16) to the AudioContext rate (typically 48 kHz). - * - * Lifecycle / messaging: - * - * main -> worklet: - * { kind: "config", inputRate: 16000, muted: false } one-shot at startup - * { kind: "audio", samples: Float32Array } per chunk - * { kind: "clear" } wipe queue (barge-in) - * - * worklet -> main: - * { kind: "stats", queuedMs, played } every ~250 ms - * { kind: "underrun" } every time the queue - * runs dry mid-playback - * - * `muted` (default false): the Live Avatar (LiveTalking WebRTC) carries the - * same audio, so the page mutes its own playback to avoid double audio. When - * the avatar is NOT running (e.g. dev on macOS), keep muted=false so the page - * plays the backend audio itself. - * - * IMPORTANT: the read position only advances while audio is actually being - * played (`_playing`). Advancing during idle would drift `_readIdx` far past - * the next chunk's length and turn every read into NaN (silence) once audio - * arrives. - */ - -const STATS_INTERVAL_FRAMES = 12000; -const FADE_FRAMES = 32; - -class AudioPlaybackProcessor extends AudioWorkletProcessor { - constructor() { - super(); - this._inputRate = 16000; - this._stepRatio = this._inputRate / sampleRate; - this._muted = false; - this._queue = []; - this._readIdx = 0; - this._fracPos = 0; - this._playing = false; - this._lastSample = 0; - this._framesSinceStats = 0; - this._totalPlayed = 0; - this._dbgReceived = 0; - this._dbgOutPeak = 0; - this._fadeIn = 0; - this._fadeOut = 0; - - this.port.onmessage = (e) => { - const data = e.data; - if (!data || typeof data !== "object") return; - switch (data.kind) { - case "config": - if (typeof data.inputRate === "number" && data.inputRate > 0) { - this._inputRate = data.inputRate; - this._stepRatio = this._inputRate / sampleRate; - } - if (typeof data.muted === "boolean") this._muted = data.muted; - break; - case "audio": { - // Accept Float32Array, a transferred ArrayBuffer, or a plain - // {buffer, byteOffset, length} descriptor: cross-realm structured - // cloning can deliver the samples in any of these shapes. - let arr = data.samples; - let f32 = null; - if (arr instanceof Float32Array) { - f32 = arr; - } else if (arr instanceof ArrayBuffer) { - f32 = new Float32Array(arr); - } else if (arr && typeof arr === "object" && arr.buffer instanceof ArrayBuffer) { - f32 = new Float32Array(arr.buffer, arr.byteOffset || 0, (arr.byteLength || arr.buffer.byteLength) >> 2); - } - if (f32 && f32.length > 0) { - this._queue.push(f32); - this._dbgReceived += 1; - if (!this._playing) { - this._playing = true; - this._fadeIn = FADE_FRAMES; - this._fadeOut = 0; - } - } - break; - } - case "clear": - this._queue.length = 0; - this._readIdx = 0; - this._fracPos = 0; - this._playing = false; - this._lastSample = 0; - this._fadeOut = FADE_FRAMES; - break; - } - }; - } - - _queuedSamples() { - let total = -this._readIdx; - for (const buf of this._queue) total += buf.length; - return Math.max(0, total); - } - - /** Linear-interp read at the current fractional position. */ - _readInterpolated() { - if (this._queue.length === 0) return null; - const head = this._queue[0]; - const idx = this._readIdx; - const frac = this._fracPos; - - let a = head[idx]; - let b; - if (idx + 1 < head.length) { - b = head[idx + 1]; - } else if (this._queue.length > 1) { - b = this._queue[1][0]; - } else { - b = a; - } - return a + (b - a) * frac; - } - - /** Advance the read position by `stepRatio`; pop consumed buffers. */ - _advance() { - this._fracPos += this._stepRatio; - while (this._fracPos >= 1) { - this._fracPos -= 1; - this._readIdx += 1; - } - while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) { - this._readIdx -= this._queue[0].length; - this._queue.shift(); - } - } - - process(_, outputs) { - const channels = outputs[0]; - if (!channels || channels.length === 0) return true; - const out = channels[0]; - const stereo = channels.length > 1 ? channels[1] : null; - - // Muted mode (Live Avatar WebRTC carries the audio): output silence. - if (this._muted) { - for (let i = 0; i < out.length; i++) { - out[i] = 0; - if (stereo) stereo[i] = 0; - } - this._framesSinceStats += out.length; - return true; - } - - for (let i = 0; i < out.length; i++) { - let sample = 0; - - if (this._playing) { - const v = this._readInterpolated(); - if (v === null) { - // Underrun: ramp out cleanly to avoid clicks. - sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES); - this._lastSample = sample; - if (Math.abs(sample) < 1e-4) { - this._playing = false; - this._lastSample = 0; - this.port.postMessage({ kind: "underrun" }); - } - } else { - sample = v; - this._lastSample = v; - this._advance(); - } - - if (this._fadeIn > 0) { - const gain = 1 - this._fadeIn / FADE_FRAMES; - sample *= gain; - this._fadeIn -= 1; - } - if (this._fadeOut > 0) { - const gain = this._fadeOut / FADE_FRAMES; - sample *= gain; - this._fadeOut -= 1; - if (this._fadeOut === 0) { - this._playing = false; - this._lastSample = 0; - } - } - - this._totalPlayed += 1; - const abs = sample < 0 ? -sample : sample; - if (abs > this._dbgOutPeak) this._dbgOutPeak = abs; - } - - out[i] = sample; - if (stereo) stereo[i] = sample; - } - - this._framesSinceStats += out.length; - if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) { - this.port.postMessage({ - kind: "stats", - queuedMs: Math.round((this._queuedSamples() / this._inputRate) * 1000), - played: this._totalPlayed, - received: this._dbgReceived, - outPeak: this._dbgOutPeak, - }); - this._framesSinceStats = 0; - } - return true; - } -} - -registerProcessor("audio-playback", AudioPlaybackProcessor); diff --git a/public/voice-demo/worklets/mic-capture.js b/public/voice-demo/worklets/mic-capture.js deleted file mode 100644 index a48860b..0000000 --- a/public/voice-demo/worklets/mic-capture.js +++ /dev/null @@ -1,159 +0,0 @@ -// @ts-check -/** - * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz) - * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it - * back to the main thread in fixed-size chunks. - * - * The Hugging Face speech-to-speech WebSocket route expects the - * `input_audio_buffer.append` payload at 16 kHz PCM16 mono. - * - * Design notes: - * - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a - * cheap low-pass before decimating. Good enough for voice STT; we lose - * a tiny bit of >8 kHz content which the pipeline discards anyway. - * - Output frames are emitted at the cadence dictated by `chunkMs` - * (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime - * server batches incoming audio so the cadence is flexible; 20-100 ms - * is the sweet spot. - * - Float -> Int16 saturates to [-1, 1] before scaling. - * - Optional noise gate: per-chunk RMS decides open/closed against a - * threshold; the gain ramps (fast attack, hold, slow release) so word - * onsets aren't clipped and quiet tails don't click. The gate only - * affects the audio we SEND; the main-thread visualiser taps the raw - * mic separately. We post the chunk RMS up every frame so the Settings - * mic meter can show the live level against the threshold. - */ - -const TARGET_RATE = 16000; -const DEFAULT_CHUNK_MS = 40; -// Gate envelope timing (fixed; only the threshold is user-tunable). -const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive -const GATE_HOLD_MS = 250; // stay open this long after the level drops back under -const GATE_RELEASE_MS = 80; // then fade closed over this long (no click) - -class MicCaptureProcessor extends AudioWorkletProcessor { - constructor(options) { - super(); - const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS; - this._inputRate = sampleRate; - this._ratio = this._inputRate / TARGET_RATE; - this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000); - this._scratch = new Float32Array(0); - this._decimated = new Float32Array(this._chunkSamples16k); - this._enabled = true; - - // Noise gate state. Disabled by default (pure passthrough). - this._gateEnabled = false; - this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open - this._gateGain = 1; // smoothed gain currently applied - this._holdRemaining = 0; // samples left before the gate may start closing - this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE)); - this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE)); - this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE); - - this.port.onmessage = (e) => { - const data = e.data; - if (data?.kind === "enable") this._enabled = !!data.value; - else if (data?.kind === "gate") { - this._gateEnabled = !!data.enabled; - // dB -> linear amplitude. When off, threshold 0 keeps the gate open. - this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0; - } - }; - } - - /** - * Append `incoming` to the internal scratch buffer, then emit as many - * full output chunks as we have material for. - * @param {Float32Array} incoming - */ - _ingest(incoming) { - if (incoming.length === 0) return; - const next = new Float32Array(this._scratch.length + incoming.length); - next.set(this._scratch, 0); - next.set(incoming, this._scratch.length); - this._scratch = next; - this._maybeEmit(); - } - - _maybeEmit() { - const r = this._ratio; - const n = this._chunkSamples16k; - const needIn = Math.ceil(n * r); - const dec = this._decimated; - while (this._scratch.length >= needIn) { - // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter. - let sumSq = 0; - if (Math.abs(r - 3) < 1e-6) { - // 48 kHz -> 16 kHz fast path with boxcar lowpass. - for (let i = 0; i < n; i++) { - const idx = i * 3; - const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3; - dec[i] = s; - sumSq += s * s; - } - } else { - // Generic path: linear interpolation. Slower but works at any rate - // (e.g. some Windows boxes report sampleRate=44100). - for (let i = 0; i < n; i++) { - const srcPos = i * r; - const idx = Math.floor(srcPos); - const frac = srcPos - idx; - const a = this._scratch[idx]; - const b = this._scratch[idx + 1] ?? a; - const s = a + (b - a) * frac; - dec[i] = s; - sumSq += s * s; - } - } - const rms = Math.sqrt(sumSq / n); - - // 2. Decide the gate target for this chunk, then ramp sample-by-sample. - let target = 1; - if (this._gateEnabled) { - if (rms >= this._thresholdLin) { - this._holdRemaining = this._holdSamples; // re-arm the hold - } else if (this._holdRemaining > 0) { - this._holdRemaining -= n; // coasting through the hold window - } else { - target = 0; - } - } - - // 3. Apply the (smoothed) gain and pack to Int16. - const out = new Int16Array(n); - let gain = this._gateGain; - for (let i = 0; i < n; i++) { - const coef = target > gain ? this._attackCoef : this._releaseCoef; - gain = target + (gain - target) * coef; - const s = dec[i] * gain; - const clamped = s < -1 ? -1 : s > 1 ? 1 : s; - out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; - } - this._gateGain = gain; - - // Shift the scratch buffer to keep only the trailing unused samples. - const consumed = Math.floor(n * r); - this._scratch = this._scratch.slice(consumed); - - // Live input level for the Settings meter (raw RMS, pre-gate). - this.port.postMessage({ kind: "level", rms }); - - if (this._enabled) { - this.port.postMessage(out.buffer, [out.buffer]); - } - // When disabled (mic muted) we silently consume input so the worklet - // stays alive and the buffer never grows unbounded. - } - } - - process(inputs) { - const input = inputs[0]; - if (!input || input.length === 0 || !input[0]) return true; - const mono = input[0]; - if (mono.length > 0) this._ingest(mono); - return true; - } -} - -registerProcessor("mic-capture", MicCaptureProcessor); diff --git a/public/voice-demo/ws/codec.js b/public/voice-demo/ws/codec.js deleted file mode 100644 index 511ad9e..0000000 --- a/public/voice-demo/ws/codec.js +++ /dev/null @@ -1,57 +0,0 @@ -// @ts-check -/** - * Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM - * conversion for the audio frames on the wire, transcript extraction from a - * `response.done` payload, and a tiny URL helper. Kept separate from the client - * so the protocol/state logic stays readable. - */ - -/** @param {string} url */ -export function trimTrailingSlash(url) { - return url.endsWith("/") ? url.slice(0, -1) : url; -} - -/** - * Pull the assistant transcript out of a `response.done` payload. The text - * lives in `response.output[].content[].transcript` (audio) or `.text`. Used as - * the source of truth for interrupted replies, where the dedicated - * `*.transcript.done` event may never arrive. - * @param {any} response - * @returns {string} - */ -export function extractResponseTranscript(response) { - const output = response?.output; - if (!Array.isArray(output)) return ""; - /** @type {string[]} */ - const parts = []; - for (const item of output) { - for (const part of item?.content ?? []) { - const text = part?.transcript ?? part?.text; - if (typeof text === "string" && text.trim()) parts.push(text.trim()); - } - } - return parts.join(" ").trim(); -} - -/** @param {ArrayBuffer} buf */ -export function base64FromArrayBuffer(buf) { - const bytes = new Uint8Array(buf); - // Chunked encoding so we don't blow up the call stack on long buffers. - let binary = ""; - const chunk = 0x8000; - for (let i = 0; i < bytes.length; i += chunk) { - binary += String.fromCharCode.apply(null, /** @type {number[]} */ ( - /** @type {unknown} */ (bytes.subarray(i, i + chunk)) - )); - } - return btoa(binary); -} - -/** @param {string} b64 */ -export function base64ToBytes(b64) { - const binary = atob(b64); - const len = binary.length; - const out = new Uint8Array(len); - for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i); - return out; -} diff --git a/public/voice-demo/ws/orb-visualizer.js b/public/voice-demo/ws/orb-visualizer.js deleted file mode 100644 index 082f601..0000000 --- a/public/voice-demo/ws/orb-visualizer.js +++ /dev/null @@ -1,98 +0,0 @@ -// @ts-check -/** - * Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the - * mic input and the TTS output) and maps the low-frequency speech energy onto - * the orb's CSS custom properties: - * - `--bar0`..`--bar4` the 5-band level meter - * - `--ai-audio-level` the global "Reachy talks" glow / scale pulse - * - * The bottom of the FFT is where speech energy lives, so the band edges stay - * low — that keeps the bars dancing on voice rather than on noise. While the AI - * is speaking we source the bars from the OUTPUT analyser so the orb pulses with - * Reachy's voice instead of sitting dead while the user is silent. - */ - -// Exported so the client can size its AnalyserNodes to match our buffer. -export const VIS_FFT_SIZE = 256; -const VIS_BAND_COUNT = 5; -const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52]; -const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy) -const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade) - -export class OrbVisualiser { - /** - * @param {AnalyserNode} micAnalyser - * @param {AnalyserNode} outAnalyser - * @param {() => boolean} isAiSpeaking Source the bars from the AI output when - * true, otherwise from the mic. - */ - constructor(micAnalyser, outAnalyser, isAiSpeaking) { - this._mic = micAnalyser; - this._out = outAnalyser; - this._isAiSpeaking = isAiSpeaking; - this._buf = new Uint8Array(micAnalyser.frequencyBinCount); - this._bands = new Float32Array(VIS_BAND_COUNT); - this._aiLevel = 0; - /** @type {number | null} */ - this._frame = null; - } - - /** Begin the rAF loop (idempotent). */ - start() { - if (this._frame !== null) return; - const root = document.documentElement; - const tick = () => { - this._frame = requestAnimationFrame(tick); - this._update(root); - }; - this._frame = requestAnimationFrame(tick); - } - - /** Stop the loop and clear the CSS vars so the orb returns to rest. */ - stop() { - if (this._frame !== null) { - cancelAnimationFrame(this._frame); - this._frame = null; - } - const root = document.documentElement; - for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`); - root.style.removeProperty("--ai-audio-level"); - } - - /** @param {HTMLElement} root */ - _update(root) { - // Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars. - const source = this._isAiSpeaking() ? this._out : this._mic; - source.getByteFrequencyData(this._buf); - - for (let b = 0; b < VIS_BAND_COUNT; b++) { - const lo = VIS_BAND_EDGES[b]; - const hi = VIS_BAND_EDGES[b + 1]; - let sum = 0; - let n = 0; - for (let i = lo; i < hi && i < this._buf.length; i++) { - sum += this._buf[i]; - n += 1; - } - const target = n > 0 ? sum / (n * 255) : 0; - const prev = this._bands[b]; - const k = target > prev ? VIS_ATTACK : VIS_RELEASE; - const next = prev + (target - prev) * k; - this._bands[b] = next; - root.style.setProperty(`--bar${b}`, next.toFixed(3)); - } - - // Global AI audio level: peak of the output analyser, used by the CSS to - // make the orb's glow / scale react to Reachy's voice. - this._out.getByteFrequencyData(this._buf); - let peak = 0; - const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]); - for (let i = 0; i < limit; i++) { - if (this._buf[i] > peak) peak = this._buf[i]; - } - const aiTarget = peak / 255; - const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE; - this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k; - root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3)); - } -} diff --git a/public/voice-demo/ws/s2s-ws-client.js b/public/voice-demo/ws/s2s-ws-client.js deleted file mode 100644 index 1843c72..0000000 --- a/public/voice-demo/ws/s2s-ws-client.js +++ /dev/null @@ -1,1176 +0,0 @@ -// @ts-check -/** - * Minimal WebSocket client for the Hugging Face speech-to-speech load balancer. - * - * Two-step handshake (same /session route as the WebRTC client): - * - * 1. POST `/session` -> JSON `{ connect_url: wss:///v1/realtime?session_token=, ... }` - * 2. Open a WebSocket directly on `connect_url` (no rewrite, unlike the WebRTC client). - * - * Once the socket is open we follow the OpenAI Realtime GA WebSocket - * protocol: - * - * - Server pushes `session.created` immediately after upgrade. - * - We send `session.update` (GA schema: `session.audio.{input,output}`, - * `session.output_modalities`, ...). - * - We stream mic audio as PCM16 16 kHz mono base64 chunks via - * `input_audio_buffer.append`. - * - The server pushes `response.output_audio.delta` (PCM16 24 kHz mono - * base64) and transcript deltas. - * - * Audio is handled internally via two AudioWorklet processors so the - * client owns the full mic-in / speaker-out pipeline. The main app only - * sees high-level lifecycle events (`status`, `transcript`, `error`, - * `session`), the same shape as the WebRTC client. - * - * @typedef {"idle" | "creating-session" | "queued" | "your-turn" | "connecting" | - * "connected" | "user-speaking" | "processing" | "ai-speaking" | - * "closed" | "error" - * } WsStatus - * - * @typedef {Object} WsSessionInfo - * @property {string} sessionId - * @property {string} connectUrl - * @property {string} websocketUrl - * @property {string} sessionToken - * @property {number} pendingTimeoutS - * @property {string} [tier] Login tier from the session proxy ("anon"|"free"|"pro"). - * @property {boolean} [limited] Whether this session is metered (heartbeat needed). - * @property {number} [heartbeatSec] Suggested heartbeat cadence in seconds. - * @property {number} [remainingSec] Daily budget left after this grant (display). - * - * @typedef {Object} WsClientOptions - * @property {string} [sessionUrl] URL to POST for the session handshake (returns - * `{ connect_url, ... }`). Usually a same-origin proxy like `api/session` so the - * load-balancer address stays server-side. Provide this OR `directUrl`. - * @property {string} [loadBalancerUrl] Load-balancer base URL. Legacy/direct - * alternative to `sessionUrl`: the client POSTs `/session` itself. Prefer - * `sessionUrl` so the LB address isn't exposed to the browser. - * @property {string} [directUrl] Full WebSocket URL of an s2s realtime endpoint - * (e.g. `ws://localhost:8080/v1/realtime`). When set, the client skips the - * session POST and dials it directly — no load balancer in between. - * @property {string} voice - * @property {string} instructions - * @property {MediaStream} [micStream] Live mic stream. Provide this OR `acquireMic`. - * @property {() => Promise} [acquireMic] Lazily obtain the mic stream, - * called only once a session is actually granted (after any queue wait). Lets the - * caller prime mic permission up front but not hold the mic 'in use' indicator on - * while waiting in line. Ignored if `micStream` is already set. - * @property {AudioContext} [audioContext] Pre-created (and resumed) context. - * iOS Safari only lets an AudioContext start from within a user gesture, so - * the caller creates/resumes it synchronously on the orb tap and hands it - * here; otherwise it stays suspended (silent) after the mic/session awaits. - * @property {ToolDef[]} [tools] Function tools declared to the backend in the - * initial `session.update`. The model decides when to call them; the caller - * executes and replies via `sendToolOutput` + `requestResponse`. - * @property {NoiseGate} [noiseGate] Client-side noise gate applied to the mic - * before it's sent. Tunable live via `setNoiseGate`. - * - * @typedef {Object} NoiseGate - * @property {boolean} enabled - * @property {number} thresholdDb Open threshold in dBFS (e.g. -45). - * - * @typedef {Object} ToolDef - * @property {"function"} type - * @property {string} name - * @property {string} description - * @property {object} parameters JSON Schema for the call arguments. - * - * @typedef {Object} TranscriptEvent - * @property {"user" | "assistant"} role - * @property {string} text - * @property {boolean} partial - */ - -import { - base64FromArrayBuffer, - base64ToBytes, - extractResponseTranscript, - trimTrailingSlash, -} from "./codec.js"; -import { OrbVisualiser, VIS_FFT_SIZE } from "./orb-visualizer.js"; - -/** Build an Error carrying a `code` (and optional extra fields) so callers can - * branch on the failure kind: "limit" | "queue-full" | "queue-expired" | "aborted". - * @param {string} message @param {string} code @param {object} [extra] */ -function _codedError(message, code, extra) { - const err = /** @type {Error & { code?: string }} */ (new Error(message)); - err.code = code; - if (extra) Object.assign(err, extra); - return err; -} - -// The s2s pipeline runs internally at 16 kHz mono PCM. The WebRTC transport -// resamples to 48 kHz for Opus, but the WebSocket transport emits the -// native pipeline rate. We don't (can't) override it via `audio.output.format` -// because the server's pydantic validator rejects the whole `session.update` -// as soon as a sub-field shape it doesn't know about appears. -const OUTPUT_SAMPLE_RATE = 16000; -const MIC_CHUNK_MS = 40; - -export class S2sWsRealtimeClient extends EventTarget { - /** @param {WsClientOptions} options */ - constructor(options) { - super(); - /** @type {WsClientOptions} */ - this.options = options; - /** Audio debug counters (see getAudioDebug). */ - this._dbgDeltas = 0; - this._dbgBytes = 0; - this._dbgPlayed = 0; - /** @type {ToolDef[]} Function tools declared to the backend. */ - this._tools = options.tools ?? []; - /** @type {string} Direct realtime WS URL (set => skip the LB session POST). */ - this._directUrl = options.directUrl ?? ""; - /** @type {string} Where to POST for the session handshake. Prefer the - * explicit `sessionUrl`; fall back to `/session` for callers - * that still pass the LB address directly. */ - this._sessionUrl = options.sessionUrl - ? options.sessionUrl - : options.loadBalancerUrl - ? `${trimTrailingSlash(options.loadBalancerUrl)}/session` - : ""; - /** @type {(() => Promise) | null} Lazy mic acquisition (post-grant). */ - this._acquireMic = options.acquireMic ?? null; - /** @type {boolean} Set by close() to abort a queue wait in progress. */ - this._closed = false; - /** @type {string} The active queue ticket id while waiting (else ""). */ - this._queueId = ""; - /** @type {(() => void) | null} Wakes the queue poll sleep early on close(). */ - this._queueWake = null; - /** @type {ReturnType | 0} */ - this._queueTimer = 0; - // Join gate: after waiting in line the caller must explicitly `join()` before - // we dial, so a slot isn't spent on someone who walked away. Resolved by - // join(), rejected on timeout (the LB reclaims the slot) or close(). - /** @type {(() => void) | null} */ - this._joinResolve = null; - /** @type {((err: Error) => void) | null} */ - this._joinReject = null; - /** @type {ReturnType | 0} */ - this._joinTimer = 0; - /** @type {NoiseGate} Mic noise gate; off by default. */ - this._noiseGate = options.noiseGate ?? { enabled: false, thresholdDb: -45 }; - /** @type {WebSocket | null} */ - this._ws = null; - /** @type {AudioContext | null} */ - this._ctx = null; - /** @type {MediaStreamAudioSourceNode | null} */ - this._micSrc = null; - /** @type {AudioWorkletNode | null} */ - this._captureNode = null; - /** @type {AudioWorkletNode | null} */ - this._playbackNode = null; - /** @type {GainNode | null} */ - this._captureSink = null; - /** @type {AnalyserNode | null} */ - this._micAnalyser = null; - /** @type {AnalyserNode | null} */ - this._outAnalyser = null; - /** @type {OrbVisualiser | null} */ - this._visualiser = null; - /** @type {WsStatus} */ - this._status = "idle"; - this._aiSpeaking = false; - /** @type {Set} response_ids that have actually played audio, so the - * UI can tell a barge-in cut (keep it) from a never-heard speculative - * response (drop it). */ - this._audibleResponses = new Set(); - /** @type {Map} The CURRENT assistant transcript segment per - * response, accumulated from streamed deltas (reset on each segment's done). */ - this._asstTranscriptByResp = new Map(); - /** @type {Map} Completed assistant transcript segments per - * response, space-joined. A single response can emit several - * `*.transcript.done` events; we concatenate them until response.done. */ - this._asstFullByResp = new Map(); - this._muted = false; - // ── Response lock ──────────────────────────────────────────────────── - // The backend allows only ONE response in flight: creating a second while - // one is active fails with `conversation_already_has_active_response`. So - // we serialize response.create. `_openResponses` counts responses the - // server has confirmed (response.created) but not yet finished - // (response.done) — it's cumulative, so every create maps to one done. - // `_createInFlight` covers the window after we send a create but before its - // response.created echo. Any requestResponse() made while locked is queued - // and replayed, one at a time, as each response.done frees the slot. - this._openResponses = 0; - this._createInFlight = false; - /** @type {{ image?: string }[]} Pending response.create payloads, one per - * queued requestResponse(). A payload may carry an image to send just - * before its create (so the frame travels with the create, not eagerly). */ - this._createQueue = []; - /** @type {Promise | null} */ - this._readyPromise = null; - this._sessionConfigured = false; - this._debug = (() => { try { return localStorage.getItem("s2s.debug") === "1"; } catch { return false; } })(); - } - - get status() { - return this._status; - } - - /** @param {WsStatus} status */ - _setStatus(status) { - if (this._status === status) return; - this._status = status; - this.dispatchEvent(new CustomEvent("status", { detail: { status } })); - } - - /** Full assistant transcript so far for a response: the completed segments - * plus the in-progress one, all space-joined. - * @param {string} rid @returns {string} */ - _asstDisplay(rid) { - const full = this._asstFullByResp.get(rid) || ""; - const seg = this._asstTranscriptByResp.get(rid) || ""; - if (!seg) return full; - return full ? `${full} ${seg}` : seg; - } - - _markAudible() { - if (this._status === "ai-speaking") return; - if (this._status === "closed" || this._status === "error") return; - this._setStatus("ai-speaking"); - } - - /** - * Full handshake. Resolves once the WS is open AND the audio pipeline is - * ready to send/receive samples. - * @returns {Promise} - */ - async connect() { - if (this._ws) throw new Error("Already connected"); - - let connectUrl; - if (this._directUrl) { - // Direct mode: no load balancer, no /session POST — dial the realtime - // endpoint straight away (e.g. a local s2s server). - connectUrl = this._directUrl; - this._setStatus("connecting"); - } else { - if (!this._sessionUrl) { - throw new Error("No session endpoint or direct URL configured"); - } - this._setStatus("creating-session"); - const { grant, waited } = await this._createSessionOrQueue(); - if (this._closed) throw _codedError("connect aborted", "aborted"); - // If we waited in line, don't dial until the user explicitly joins — this - // keeps a freed slot from being spent on someone who stepped away, and the - // click is a fresh gesture (re-arms the AudioContext on iOS). - if (waited) { - await this._awaitJoin(grant); - if (this._closed) throw _codedError("connect aborted", "aborted"); - } - this.dispatchEvent(new CustomEvent("session", { detail: { info: grant } })); - connectUrl = grant.connectUrl; - this._setStatus("connecting"); - } - - // Acquire the mic now — only once a slot is actually ours. The caller primed - // permission up front, so this is silent and the 'in use' indicator lights - // only for a real, connecting session (never during a queue wait). - if (!this.options.micStream && this._acquireMic) { - this.options.micStream = await this._acquireMic(); - } - - // Spin up the AudioContext + worklets in parallel with the WS dial. - const audioReady = this._setupAudio(); - const wsReady = this._openWebSocket(connectUrl); - await Promise.all([audioReady, wsReady]); - } - - /** - * POST the session handshake; if the pool is busy, wait in the queue (polling - * position) until a slot is claimed. Resolves to a grant plus whether we had to - * wait (which decides if an explicit join is required before dialing). - * @returns {Promise<{ grant: WsSessionInfo, waited: boolean }>} - */ - async _createSessionOrQueue() { - const first = await this._postSession(); - if (first.state === "queued") { - this._setStatus("queued"); - const grant = await this._pollQueue(first); - return { grant, waited: true }; - } - return { grant: first.grant, waited: false }; - } - - /** - * Hold at the front of the line until the user clicks join (resolves the gate) - * or the grant lapses. Announces "your-turn" + a deadline the UI counts down. - * @param {WsSessionInfo} grant - * @returns {Promise} - */ - _awaitJoin(grant) { - // The LB reclaims an unclaimed slot at its pending timeout; expire the gate a - // touch earlier so we never dial a session the LB just reaped. - const windowS = Math.max(3, (grant.pendingTimeoutS || 60) - 3); - this._setStatus("your-turn"); - this.dispatchEvent( - new CustomEvent("ready-to-join", { detail: { info: grant, expiresSec: windowS } }), - ); - return new Promise((resolve, reject) => { - this._joinResolve = resolve; - this._joinReject = reject; - this._joinTimer = setTimeout(() => { - this._joinResolve = null; - this._joinReject = null; - reject(_codedError("Your spot expired", "join-expired")); - }, windowS * 1000); - }); - } - - /** Accept the held slot and let connect() proceed to dial. Called from the - * "Join now" click, so it's a user gesture: re-resume the AudioContext, which - * iOS may have suspended while we waited. */ - join() { - if (this._joinTimer) { - clearTimeout(this._joinTimer); - this._joinTimer = 0; - } - try { - void this.options.audioContext?.resume(); - } catch { - // best-effort; _setupAudio resumes again - } - const resolve = this._joinResolve; - this._joinResolve = null; - this._joinReject = null; - resolve?.(); - } - - /** - * POST /session once. Returns either a granted session or a queue ticket. - * @returns {Promise<{ state: "granted", grant: WsSessionInfo } | { state: "queued", queueId: string, position: number, pollIntervalS: number }>} - */ - async _postSession() { - const url = this._sessionUrl; - console.log("[ws] POST", url); - const response = await fetch(url, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: "{}", - }); - if (response.status === 402) { - // The session proxy refused: today's per-tier time budget is spent. Surface - // it as a typed error so the UI shows the limit modal, not a crash. - const body = await response.json().catch(() => ({})); - throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier }); - } - if (response.status === 503) { - const body = await response.json().catch(() => ({})); - if (body?.state === "at_capacity") { - throw _codedError("The queue is full — try again shortly.", "queue-full"); - } - throw new Error("/session failed (503)"); - } - if (!response.ok) { - const text = await response.text().catch(() => ""); - throw new Error(`/session failed (${response.status}): ${text}`); - } - const json = await response.json(); - if (json.state === "queued") { - return { - state: "queued", - queueId: json.queue_id, - position: json.position, - pollIntervalS: json.poll_interval_s, - }; - } - return { state: "granted", grant: this._parseGrant(json) }; - } - - /** - * Poll the waiting queue until this ticket claims a slot. Emits `queue` events - * ({ position }) as the line advances. Throws on limit (402), expiry (404), or - * close(). Transient network/5xx blips are ignored and retried next tick. - * @param {{ queueId: string, position: number, pollIntervalS: number }} ticket - * @returns {Promise} - */ - async _pollQueue(ticket) { - const intervalMs = Math.max(1, ticket.pollIntervalS || 2) * 1000; - this._queueId = ticket.queueId; - this._emitQueue(ticket.position); - - while (true) { - await this._queueSleep(intervalMs); - if (this._closed) throw _codedError("queue wait aborted", "aborted"); - - let response; - try { - response = await fetch(`api/queue/${encodeURIComponent(this._queueId)}`, { - headers: { "Content-Type": "application/json" }, - }); - } catch { - continue; // network blip — keep our place, retry next tick - } - - if (response.status === 402) { - const body = await response.json().catch(() => ({})); - throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier }); - } - if (response.status === 404) { - throw _codedError("Queue timed out", "queue-expired"); - } - if (!response.ok) continue; // 502/503 — transient, retry - - const json = await response.json().catch(() => null); - if (!json) continue; - if (json.state === "queued") { - this._emitQueue(json.position); - continue; - } - // Reached the front and claimed a slot. - this._queueId = ""; - return this._parseGrant(json); - } - } - - /** @param {number} position */ - _emitQueue(position) { - this.dispatchEvent( - new CustomEvent("queue", { detail: { position, queueId: this._queueId } }), - ); - } - - /** A sleep that close() can cut short so a queued client tears down promptly. - * @param {number} ms */ - _queueSleep(ms) { - return new Promise((resolve) => { - this._queueWake = resolve; - this._queueTimer = setTimeout(() => { - this._queueWake = null; - resolve(); - }, ms); - }); - } - - /** @param {any} json @returns {WsSessionInfo} */ - _parseGrant(json) { - return { - sessionId: json.session_id, - connectUrl: json.connect_url, - websocketUrl: json.websocket_url, - sessionToken: json.session_token, - pendingTimeoutS: json.pending_timeout_s, - tier: json.tier, - limited: json.limited, - heartbeatSec: json.heartbeatSec, - remainingSec: json.remainingSec, - }; - } - - async _setupAudio() { - // Prefer a context the caller already created + resumed inside the tap - // gesture (required on iOS). Fall back to creating one here for callers - // that don't (desktop is lenient about the gesture timing). - // Most desktops give us 48 kHz, mobiles can give 44.1/24/16 kHz; the - // capture worklet handles any rate (linear interp fallback). - const ctx = this.options.audioContext ?? new AudioContext({ latencyHint: "interactive" }); - this._ctx = ctx; - - // Resume if still suspended. This is best-effort here — on iOS the resume - // that actually counts is the one the caller did synchronously on tap. - if (ctx.state === "suspended") { - try { - await ctx.resume(); - } catch (err) { - console.warn("[ws] AudioContext resume failed:", err); - } - } - - // The worklets live at the repo root, one level up from this module. - // ?v=2 busts the browser cache so the fixed (unmuted) playback worklet - // always loads instead of a stale muted copy. - const base = new URL("../worklets/", import.meta.url); - await ctx.audioWorklet.addModule(new URL("mic-capture.js?v=3", base).href); - await ctx.audioWorklet.addModule(new URL("audio-playback.js?v=3", base).href); - - const captureNode = new AudioWorkletNode(ctx, "mic-capture", { - numberOfInputs: 1, - numberOfOutputs: 0, - processorOptions: { chunkMs: MIC_CHUNK_MS }, - }); - captureNode.port.onmessage = (e) => { - const data = e.data; - if (data instanceof ArrayBuffer) { - this._onMicChunk(data); - } else if (data?.kind === "level") { - // Raw pre-gate mic RMS for the Settings meter. - this.dispatchEvent(new CustomEvent("input-level", { detail: { rms: data.rms } })); - } - }; - // Push the initial gate config now that the worklet exists. - captureNode.port.postMessage({ kind: "gate", ...this._noiseGate }); - this._captureNode = captureNode; - - const micSrc = ctx.createMediaStreamSource(this.options.micStream); - micSrc.connect(captureNode); - this._micSrc = micSrc; - - // Mic analyser: tap the mic in parallel with the worklet so we get the - // raw (un-resampled, un-clipped) signal for the visualiser. - const micAnalyser = ctx.createAnalyser(); - micAnalyser.fftSize = VIS_FFT_SIZE; - micAnalyser.smoothingTimeConstant = 0; - micSrc.connect(micAnalyser); - this._micAnalyser = micAnalyser; - - const playbackNode = new AudioWorkletNode(ctx, "audio-playback", { - numberOfInputs: 0, - numberOfOutputs: 1, - outputChannelCount: [1], - }); - playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE, muted: this._playbackMuted === true }); - playbackNode.port.onmessage = (e) => this._onPlaybackMessage(e.data); - - // Output analyser sits between the playback worklet and the speakers. - const outAnalyser = ctx.createAnalyser(); - outAnalyser.fftSize = VIS_FFT_SIZE; - outAnalyser.smoothingTimeConstant = 0.3; - playbackNode.connect(outAnalyser); - outAnalyser.connect(ctx.destination); - this._outAnalyser = outAnalyser; - this._playbackNode = playbackNode; - - this._visualiser = new OrbVisualiser(micAnalyser, outAnalyser, () => this._aiSpeaking); - this._visualiser.start(); - } - - /** @param {string} connectUrl */ - _openWebSocket(connectUrl) { - return new Promise((resolve, reject) => { - const ws = new WebSocket(connectUrl); - ws.binaryType = "arraybuffer"; - this._ws = ws; - - const onceOpen = () => { - ws.removeEventListener("open", onceOpen); - ws.removeEventListener("error", onceErr); - resolve(); - }; - const onceErr = (e) => { - ws.removeEventListener("open", onceOpen); - ws.removeEventListener("error", onceErr); - reject(new Error(`WebSocket failed to open: ${e?.type ?? "error"}`)); - }; - ws.addEventListener("open", onceOpen); - ws.addEventListener("error", onceErr); - - ws.addEventListener("message", (e) => this._onWsMessage(e.data)); - ws.addEventListener("close", (e) => this._onWsClose(e)); - ws.addEventListener("error", (e) => { - console.error("[ws] socket error", e); - }); - }); - } - - /** - * @param {{ kind: string; queuedMs?: number; played?: number }} data - */ - _onPlaybackMessage(data) { - if (data?.kind === "underrun") { - // Server stopped sending audio mid-response. Most likely the turn - // ended cleanly (a response.done usually arrives just before/after - // this). We let the state machine fall back to "connected" via the - // response.done event handler. - } else if (data?.kind === "stats") { - if (typeof data.played === "number") this._dbgPlayed = data.played; - if (typeof data.received === "number") this._dbgReceived = data.received; - if (typeof data.outPeak === "number") this._dbgOutPeak = data.outPeak; - } - } - - /** - * Audio pipeline debug readout for the status overlay. - * @returns {{ deltas: number, bytes: number, played: number, ctx: string, ws: number }} - */ - getAudioDebug() { - return { - deltas: this._dbgDeltas, - bytes: this._dbgBytes, - received: this._dbgReceived ?? 0, - played: this._dbgPlayed, - peak: this._dbgPeak ?? 0, - outPeak: this._dbgOutPeak ?? 0, - ctx: this._ctx ? this._ctx.state : "none", - ws: this._ws ? this._ws.readyState : -1, - }; - } - - /** - * Mic worklet just sent us a ~40 ms PCM16 16 kHz mono chunk. - * Base64-encode and forward via the WS. - * @param {ArrayBuffer} pcm16Buffer - */ - _onMicChunk(pcm16Buffer) { - if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; - if (!this._sessionConfigured) return; // Server rejects audio before session.update. - if (this._muted) return; - const b64 = base64FromArrayBuffer(pcm16Buffer); - this._send({ type: "input_audio_buffer.append", audio: b64 }); - } - - /** - * @param {string | ArrayBuffer | Blob} raw - */ - async _onWsMessage(raw) { - let text; - if (typeof raw === "string") { - text = raw; - } else if (raw instanceof ArrayBuffer) { - text = new TextDecoder("utf-8").decode(raw); - } else if (raw instanceof Blob) { - text = await raw.text(); - } else { - return; - } - - let event; - try { - event = JSON.parse(text); - } catch { - return; - } - - const type = event?.type; - if (typeof type !== "string") return; - // Opt-in event tracing for diagnosing turn/transcript issues. Enable with - // `localStorage.setItem("s2s.debug", "1")` in the browser console. - if (this._debug) { - const extra = type.startsWith("conversation.item.input_audio_transcription") - ? ` item=${event.item_id} ci=${event.content_index} ${event.delta ?? event.transcript ?? ""}` - : type.startsWith("response.") - ? ` resp=${event.response_id ?? event.response?.id ?? ""} status=${event.response?.status ?? ""} ${event.transcript ?? ""}` - : ""; - console.debug(`[ws] ${type}${extra}`); - } - - switch (type) { - case "session.created": - // Server-side defaults for the s2s pipeline are already what we - // want (server_vad, whisper-1 transcription, PCM16 16k in / 24k - // out). We only push the user-tunable bits: voice + instructions. - this._sendSessionUpdate(); - this._sessionConfigured = true; - if (this._status === "connecting") this._setStatus("connected"); - break; - - case "session.updated": - // Acknowledged by server, nothing to do. - break; - - case "input_audio_buffer.speech_started": - // User started speaking — stop any audio still playing OR queued, every - // time. We clear unconditionally (not just when `_aiSpeaking`): after a - // reply or a tool result the worklet's ring buffer can still be draining - // even though we already flipped `_aiSpeaking` off, and that tail would - // otherwise keep playing over the user's barge-in. - this._playbackNode?.port.postMessage({ kind: "clear" }); - this._aiSpeaking = false; - this._setStatus("user-speaking"); - break; - - case "input_audio_buffer.speech_stopped": - if (this._status === "user-speaking") this._setStatus("processing"); - break; - - case "response.created": - // A response now owns the slot — count it and clear our create guard - // (this confirms either our create or a server-initiated one). - this._openResponses++; - this._createInFlight = false; - if (this._status === "connected" || this._status === "user-speaking") { - this._setStatus("processing"); - } - break; - - case "response.output_item.added": - if (this._status === "connected" || this._status === "user-speaking") { - this._setStatus("processing"); - } - break; - - case "response.audio.delta": - case "response.output_audio.delta": { - this._pushAudioDelta(event.delta); - const rid = event.response_id ?? event.response?.id; - if (rid) this._audibleResponses.add(rid); - if (!this._aiSpeaking) { - this._aiSpeaking = true; - this._markAudible(); - } - break; - } - - case "response.content_part.added": { - const part = event.part; - if (part?.type === "audio" || part?.type === "output_audio") { - this._markAudible(); - } - break; - } - - case "response.done": { - this._aiSpeaking = false; - // This response freed the slot (completion OR cancellation both arrive - // as response.done). Decrement and, if a create was waiting, replay it. - this._openResponses = Math.max(0, this._openResponses - 1); - if (this._status === "ai-speaking" || this._status === "processing") { - this._setStatus("connected"); - } - // A response closes here for BOTH normal completion and cancellation - // (the s2s server signals a speculative-turn interrupt as - // `response.done` with status "cancelled" — there is no separate - // `response.cancelled` event). Surface the id + status so the UI can - // drop a cancelled response's transcript and commit a completed one. - const status = event.response?.status ?? "completed"; - const responseId = event.response?.id ?? ""; - // Did this response ever play audio? Distinguishes a barge-in cut (the - // user heard part of it) from a speculative response that never played. - const audible = responseId ? this._audibleResponses.has(responseId) : false; - this._audibleResponses.delete(responseId); - // Pull whatever transcript the response carries, falling back to the - // segments we concatenated from the `*.transcript.done` events (plus any - // in-progress delta). For an interrupted reply the response payload may - // be empty, so this is the last chance to capture the text. - const transcript = - extractResponseTranscript(event.response) || - this._asstDisplay(responseId) || - ""; - // Response finished — clear both transcript accumulators for it. - this._asstTranscriptByResp.delete(responseId); - this._asstFullByResp.delete(responseId); - this.dispatchEvent(new CustomEvent("response-finished", { - detail: { responseId, status, audible, transcript }, - })); - // The slot is free now — replay a queued create (e.g. a tool follow-up - // that arrived while this response was still running). - this._flushQueuedCreate(); - break; - } - - case "response.function_call_arguments.done": { - const name = typeof event.name === "string" ? event.name : ""; - const args = typeof event.arguments === "string" ? event.arguments : "{}"; - const callId = typeof event.call_id === "string" ? event.call_id : ""; - if (name) { - this.dispatchEvent(new CustomEvent("toolcall", { - detail: { name, arguments: args, callId }, - })); - } else { - // A nameless call can't be executed, so no function_call_output is - // ever sent and the model would wait forever for a result. The - // backend shouldn't emit these; warn loudly rather than stall silently. - console.warn(`[ws] function_call_arguments.done with no name (call_id=${callId}); cannot run tool — turn may stall`); - } - break; - } - - case "conversation.item.input_audio_transcription.delta": { - const delta = typeof event.delta === "string" ? event.delta : ""; - if (delta) { - // `itemId` is REUSED across a speculative continuation, so the UI - // groups both segments into one message. The delta carries the full - // cumulative transcript so far (not an increment). - this.dispatchEvent( - new CustomEvent("transcript", { - detail: { - role: "user", - text: delta, - partial: true, - itemId: typeof event.item_id === "string" ? event.item_id : "", - }, - }), - ); - } - break; - } - - case "conversation.item.input_audio_transcription.completed": { - const transcript = typeof event.transcript === "string" ? event.transcript : ""; - if (transcript) { - this.dispatchEvent( - new CustomEvent("transcript", { - detail: { - role: "user", - text: transcript, - partial: false, - itemId: typeof event.item_id === "string" ? event.item_id : "", - }, - }), - ); - } - break; - } - - case "response.audio_transcript.delta": - case "response.output_audio_transcript.delta": { - // Stream the assistant transcript live: accumulate the incremental - // deltas and push the running text to the UI. Every transcribe event we - // receive reaches the conversation, so an interrupted reply already has - // its partial text even if the `.done` never fires. - this._markAudible(); - const rid = typeof event.response_id === "string" ? event.response_id : ""; - const delta = typeof event.delta === "string" ? event.delta : ""; - if (delta) { - this._asstTranscriptByResp.set(rid, (this._asstTranscriptByResp.get(rid) || "") + delta); - // Show completed segments + the segment streaming in right now. - this.dispatchEvent( - new CustomEvent("transcript", { - detail: { role: "assistant", text: this._asstDisplay(rid), partial: true, responseId: rid }, - }), - ); - } - break; - } - - case "response.audio_transcript.done": - case "response.output_audio_transcript.done": { - const rid = typeof event.response_id === "string" ? event.response_id : ""; - // This is ONE completed segment. A response can emit several; concatenate - // them, space-separated, until response.done clears the accumulator. - const segment = - (typeof event.transcript === "string" && event.transcript) || - this._asstTranscriptByResp.get(rid) || - ""; - this._asstTranscriptByResp.delete(rid); // segment finished; next one starts fresh - if (segment) { - const prev = this._asstFullByResp.get(rid) || ""; - this._asstFullByResp.set(rid, prev ? `${prev} ${segment}` : segment); - } - const full = this._asstFullByResp.get(rid) || ""; - if (full) { - this.dispatchEvent( - new CustomEvent("transcript", { - detail: { role: "assistant", text: full, partial: false, responseId: rid }, - }), - ); - } - break; - } - - case "error": { - const err = event.error; - console.error("[ws] server error:", err); - // The "another response is already active" race: our optimistic create - // collided with a still-running response. Don't surface it — clear the - // in-flight guard and re-queue, so the create replays on the next - // response.done (never retried immediately, which would just collide - // again). - if (err?.type === "conversation_already_has_active_response" || - err?.code === "conversation_already_has_active_response") { - if (this._createInFlight) { - this._createInFlight = false; - // Re-queue a BARE create: any image on the original payload was - // already sent before this (rejected) create, so don't resend it. - this._createQueue.push({}); - } - break; - } - // Every other server error is non-fatal: surface it for logging but - // NEVER tear the socket down. Only transport failures (close / failed - // open) are fatal, and those come through their own paths. - this.dispatchEvent( - new CustomEvent("server-error", { detail: { error: new Error(err?.message ?? "Server error") } }), - ); - break; - } - } - } - - /** @param {string} b64 */ - _pushAudioDelta(b64) { - if (!this._playbackNode) return; - if (!b64) return; - this._dbgDeltas += 1; - this._dbgBytes += (b64.length * 3) >> 2; // approx decoded bytes - const bytes = base64ToBytes(b64); - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - const samples = new Float32Array(bytes.byteLength / 2); - let peak = 0; - for (let i = 0; i < samples.length; i++) { - const s = view.getInt16(i * 2, true); - samples[i] = s < 0 ? s / 0x8000 : s / 0x7fff; - const a = samples[i] < 0 ? -samples[i] : samples[i]; - if (a > peak) peak = a; - } - this._dbgPeak = Math.max(this._dbgPeak || 0, peak); - // Clone (no transfer list): structured cloning across the main->worklet - // boundary is more robust than transferring the buffer — some browser - // versions mis-shape the received object and the worklet would drop it. - this._playbackNode.port.postMessage({ kind: "audio", samples }); - } - - /** @param {CloseEvent} ev */ - _onWsClose(ev) { - console.log("[ws] socket closed:", ev.code, ev.reason); - if (this._status === "closed" || this._status === "error") return; - if (ev.code === 1000) { - this._setStatus("closed"); - } else { - this.dispatchEvent( - new CustomEvent("error", { - detail: { error: new Error(`WebSocket closed (${ev.code}) ${ev.reason || ""}`.trim()) }, - }), - ); - this._setStatus("error"); - } - } - - _sendSessionUpdate() { - // Minimal payload: only the bits the user is allowed to configure. - // The s2s server already defaults to server_vad, whisper-1 - // transcription, 16 kHz PCM input and 24 kHz PCM output, so we don't - // need (and must not send) `audio.input.format`, `audio.input.transcription`, - // `audio.input.turn_detection` or `audio.output.format`: the pydantic - // validator on the server rejects the whole event if any unknown or - // future-shaped sub-field shows up. - /** @type {Record} */ - const session = { - type: "realtime", - instructions: this.options.instructions, - audio: { - output: { voice: this.options.voice }, - }, - }; - // Tools are declared here; the backend already accepts them in - // session.update and emits response.function_call_arguments.done when the - // model decides to call one. Only include the keys when we actually have - // tools — the server's pydantic validator is strict about shapes. - if (this._tools.length) { - session.tools = this._tools; - session.tool_choice = "auto"; - } - this._send({ type: "session.update", session }); - } - - /** Update voice/instructions on a live session without tearing down. */ - /** @param {{ voice?: string; instructions?: string }} patch */ - updateSession(patch) { - /** @type {Record} */ - const session = { type: "realtime" }; - if (patch.instructions) session.instructions = patch.instructions; - if (patch.voice) session.audio = { output: { voice: patch.voice } }; - if (Object.keys(session).length > 1) { - this._send({ type: "session.update", session }); - } - } - - /** - * Replace the declared tool set on a live session (e.g. the user flipped a - * tool switch mid-conversation). Always sends `tools` — an empty array - * clears them — so toggling the last tool off actually removes it. - * @param {ToolDef[]} tools - */ - setTools(tools) { - this._tools = tools; - this._send({ - type: "session.update", - session: { type: "realtime", tools, tool_choice: tools.length ? "auto" : "none" }, - }); - } - - /** - * Return a tool's result to the model. Pairs with the `toolcall` event's - * `callId`. Caller follows this with `requestResponse()` so the model speaks. - * @param {string} callId - * @param {string} output Plain text / JSON string the model will read. - */ - sendToolOutput(callId, output) { - if (!callId) return; // Can't target a result without the call id. - this._send({ - type: "conversation.item.create", - item: { type: "function_call_output", call_id: callId, output }, - }); - } - - /** - * Add an image to the conversation as user content, so the vision-language - * model can see it (used by the camera tool). `dataUrl` is a - * `data:image/jpeg;base64,...` string. - * @param {string} dataUrl - */ - sendUserImage(dataUrl) { - this._send({ - type: "conversation.item.create", - item: { - type: "message", - role: "user", - content: [{ type: "input_image", image_url: dataUrl }], - }, - }); - } - - /** - * Add a text message to the conversation as user content, and trigger a - * response. Use this to type messages instead of speaking. The text is - * sent as a user message and the model generates a reply. - * @param {string} text - */ - sendUserText(text) { - if (!text) return; - this._send({ - type: "conversation.item.create", - item: { - type: "message", - role: "user", - content: [{ type: "input_text", text }], - }, - }); - this.requestResponse(); - } - - /** - * Ask the model to generate a response now (after feeding tool results). - * Serialized: if a response is already in flight we queue this request and - * replay it once the active response finishes, so we never trip the - * backend's `conversation_already_has_active_response` guard. - * - * @param {{ image?: string }} [opts] Optional `image` (a data URL) sent as a - * user `input_image` immediately before this response.create — so the frame - * travels with the create (and is deferred together with it if queued), - * rather than being added to the conversation eagerly. Used by the camera - * tool so the model sees the snapshot in the response it's about to speak. - */ - requestResponse(opts = {}) { - if (this._responseActive()) { - this._createQueue.push(opts); - if (this._debug) console.debug(`[ws] response.create queued (a response is active); pending=${this._createQueue.length}`); - return; - } - this._createResponseNow(opts); - } - - /** True while a response occupies the single backend slot. */ - _responseActive() { - return this._openResponses > 0 || this._createInFlight; - } - - /** Send a response.create immediately and arm the in-flight guard. Any image - * on the payload is added as user content right before the create. - * @param {{ image?: string }} [opts] */ - _createResponseNow(opts = {}) { - if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; - if (opts.image) this.sendUserImage(opts.image); - this._createInFlight = true; - this._send({ type: "response.create" }); - } - - /** Replay one queued response.create if the slot is now free. Called on every - * response.done, so queued creates drain one-per-completion. */ - _flushQueuedCreate() { - if (this._createQueue.length > 0 && !this._responseActive()) { - const opts = this._createQueue.shift(); - if (this._debug) console.debug(`[ws] replaying queued response.create; remaining=${this._createQueue.length}`); - this._createResponseNow(opts); - } - } - - /** @param {boolean} muted */ - setMuted(muted) { - this._muted = muted; - } - - /** - * Mute/unmute the page's own TTS playback. When the Live Avatar's WebRTC - * stream carries the same audio (digital human running), mute the page to - * avoid double audio; otherwise (no avatar) keep it playing. - * @param {boolean} muted - */ - setPlaybackMuted(muted) { - this._playbackMuted = muted; - if (this._playbackNode) { - this._playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE, muted }); - } - } - - /** - * Update the mic noise gate live (the user moved the Settings cursor). - * @param {NoiseGate} gate - */ - setNoiseGate(gate) { - this._noiseGate = gate; - this._captureNode?.port.postMessage({ kind: "gate", ...gate }); - } - - /** @param {Record} event */ - _send(event) { - if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; - this._ws.send(JSON.stringify(event)); - } - - async close() { - // Abort a queue wait in progress: flag it and wake the poll sleep so - // `_pollQueue` throws "aborted" and connect() unwinds cleanly. - this._closed = true; - if (this._queueWake) { - clearTimeout(this._queueTimer); - const wake = this._queueWake; - this._queueWake = null; - wake(); - } - if (this._joinTimer) { - clearTimeout(this._joinTimer); - this._joinTimer = 0; - } - if (this._joinReject) { - const reject = this._joinReject; - this._joinResolve = null; - this._joinReject = null; - reject(_codedError("join aborted", "aborted")); - } - this._visualiser?.stop(); - this._visualiser = null; - try { - if (this._ws && this._ws.readyState <= WebSocket.OPEN) { - this._ws.close(1000, "client closed"); - } - } catch { - // ignored - } - this._ws = null; - - try { - this._captureNode?.port.close?.(); - } catch { - // ignored - } - try { - this._micSrc?.disconnect(); - } catch { - // ignored - } - try { - this._captureNode?.disconnect(); - } catch { - // ignored - } - try { - this._micAnalyser?.disconnect(); - } catch { - // ignored - } - try { - this._outAnalyser?.disconnect(); - } catch { - // ignored - } - try { - this._playbackNode?.disconnect(); - } catch { - // ignored - } - try { - await this._ctx?.close(); - } catch { - // ignored - } - this._ctx = null; - this._captureNode = null; - this._playbackNode = null; - this._micSrc = null; - this._micAnalyser = null; - this._outAnalyser = null; - this._setStatus("closed"); - } -} diff --git a/src/pages/Screen.jsx b/src/pages/Screen.jsx deleted file mode 100644 index 95302ad..0000000 --- a/src/pages/Screen.jsx +++ /dev/null @@ -1,550 +0,0 @@ -import { useState, useEffect, useRef, useCallback } from 'react'; -import { useNavigate } from 'react-router-dom'; -import * as api from '../utils/api'; -import { API_BASE } from '../utils/api'; -import { getCurrentWindow } from '@tauri-apps/api/window'; -import { enable, disable } from '@tauri-apps/plugin-autostart'; -import '../styles/screen.css'; - -/* ============ Loader ============ */ -function Loader({ hidden, text }) { - return ( -
-
-
-
-
-
-
PineSound
-
-
-
-
-
-
-
-
- ); -} - -/* ========================================================= - Screen Page - ========================================================= */ -export default function Screen() { - const navigate = useNavigate(); - const [list, setList] = useState([]); - const [index, setIndex] = useState(0); - const [volume, setVolume] = useState(0.8); - const [playMode, setPlayMode] = useState('sequential'); - const [imageDuration, setImageDuration] = useState(5); - const [paused, setPaused] = useState(false); - const [soundBlocked, setSoundBlocked] = useState(false); - const [loaded, setLoaded] = useState(false); - const [loaderText, setLoaderText] = useState('加载播放列表中...'); - const [mediaKey, setMediaKey] = useState(0); // 强制重新渲染，用于单项目循环 - const [showVideo, setShowVideo] = useState(true); - const [showLoginModal, setShowLoginModal] = useState(false); - const [loginUsername, setLoginUsername] = useState(''); - const [loginPassword, setLoginPassword] = useState(''); - const [loginError, setLoginError] = useState(''); - - const videoRef = useRef(null); - const imgRef = useRef(null); - const timerRef = useRef(null); - const loginTimerRef = useRef(null); - const loginInputRef = useRef(null); - - // ref 存储可变值，避免回调中的闭包过期 - const listRef = useRef(list); - const indexRef = useRef(index); - const pausedRef = useRef(paused); - const playModeRef = useRef(playMode); - const imageDurationRef = useRef(imageDuration); - const soundBlockedRef = useRef(soundBlocked); - - // 从后端同步的显示设置（Screen 上无需 UI 控制，但需要响应 SSE） - const [fullscreenMode, setFullscreenMode] = useState(true); - const [autostartEnabled, setAutostartEnabled] = useState(false); - - useEffect(() => { listRef.current = list; }, [list]); - useEffect(() => { indexRef.current = index; }, [index]); - useEffect(() => { pausedRef.current = paused; }, [paused]); - useEffect(() => { playModeRef.current = playMode; }, [playMode]); - useEffect(() => { imageDurationRef.current = imageDuration; }, [imageDuration]); - useEffect(() => { soundBlockedRef.current = soundBlocked; }, [soundBlocked]); - - const getUrl = useCallback((item) => { - return item.source === 'url' ? item.relative_path : `${API_BASE}/file/${item.relative_path}`; - }, []); - - // ============ 核心播放控制 ============ - - /** 稳定的 next — 始终推进索引（不检查 paused），永不停止 */ - const next = useCallback(() => { - if (timerRef.current) clearTimeout(timerRef.current); - const len = listRef.current.length; - if (len === 0) return; - const mode = playModeRef.current; - setIndex(prev => { - if (mode === 'random') { - return Math.floor(Math.random() * len); - } - // 关键：始终循环，(prev + 1) % len 保证永不越界 - return (prev + 1) % len; - }); - // 单项目时 index 不变（(0+1)%1=0），用 mediaKey 强制渲染 - setMediaKey(k => k + 1); - }, []); - - const skip = useCallback((delta) => { - if (timerRef.current) clearTimeout(timerRef.current); - const len = listRef.current.length; - if (len === 0) return; - const mode = playModeRef.current; - setIndex(prev => { - if (mode === 'random') { - return Math.floor(Math.random() * len); - } - return (prev + delta + len) % len; - }); - setMediaKey(k => k + 1); - }, []); - - // ============ 播放效果 ============ - - // index / mediaKey 变化时播放对应项 - useEffect(() => { - if (!loaded || list.length === 0 || paused) return; - const item = list[index]; - if (!item) return; - - // 同步状态到后端（字段名 type 与 Rust PlaybackState 的 #[serde(rename = "type")] 匹配） - api.updateState({ - status: 'playing', - index, - name: item.name || '', - type: item.type || '', - }).catch(() => {}); - - if (item.type === 'video') { - setShowVideo(true); - const video = videoRef.current; - if (!video) return; - const url = getUrl(item); - video.src = url; - video.load(); - video.muted = soundBlocked; - video.volume = volume; - const p = video.play(); - if (p !== undefined) { - p.catch(() => { - setSoundBlocked(true); - video.muted = true; - video.play().catch(() => {}); - }); - } - } else { - // 图片 - setShowVideo(false); - const url = getUrl(item); - imgRef.current.src = url; - if (timerRef.current) clearTimeout(timerRef.current); - timerRef.current = setTimeout(next, imageDuration * 1000); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [index, mediaKey, loaded, paused]); - - // 恢复播放：paused 从 true → false 时触发 - useEffect(() => { - if (paused || !loaded || list.length === 0) return; - const item = list[index]; - if (!item || item.type !== 'video') return; - const video = videoRef.current; - if (video && video.paused && video.src) { - video.play().catch(() => { - setSoundBlocked(true); - video.muted = true; - video.play().catch(() => {}); - }); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [paused]); - - // ============ 事件监听 ============ - - // Video ended → next（稳定，永不失效） - useEffect(() => { - const video = videoRef.current; - if (!video) return; - const handler = () => next(); - video.addEventListener('ended', handler); - return () => video.removeEventListener('ended', handler); - // mediaKey 变化时 video 元素被 key 强制重建，需要重新绑定 - }, [next, mediaKey]); - - // 加载播放列表 - useEffect(() => { - const load = async () => { - try { - const data = await api.getPlaylist(); - const files = data.files || []; - const vol = (data.volume || 80) / 100; - setList(files); - setVolume(vol); - setPlayMode(data.play_mode || 'sequential'); - setImageDuration(data.image_duration || 5); - if (videoRef.current) videoRef.current.volume = vol; - - if (files.length === 0) { - setLoaderText('播放列表为空，前往管理后台添加'); - return; - } - setLoaded(true); - - // 尝试启用声音 - const video = videoRef.current; - if (video) { - video.muted = false; - const testPlay = video.play(); - if (testPlay !== undefined) { - testPlay.then(() => { - setSoundBlocked(false); - video.pause(); - }).catch(() => { - setSoundBlocked(true); - video.muted = true; - }); - } - } - } catch {/* ignore */} - }; - load(); - }, []); - - // SSE 连接 - useEffect(() => { - const es = new EventSource(`${API_BASE}/api/events`); - es.onmessage = (e) => { - try { - const msg = JSON.parse(e.data); - switch (msg.action) { - case 'pause': - setPaused(true); - if (timerRef.current) clearTimeout(timerRef.current); - if (videoRef.current) videoRef.current.pause(); - break; - case 'play': - setPaused(false); - // 由 paused → false 的 useEffect 自动恢复播放 - break; - case 'next': - setPaused(false); - skip(1); - break; - case 'prev': - setPaused(false); - skip(-1); - break; - case 'minimize_window': - getCurrentWindow().minimize().catch(() => {}); - break; - case 'playlist_changed': - reloadPlaylist(); - break; - case 'settings_changed': - applySettings(msg); - break; - } - } catch {/* ignore */} - }; - return () => es.close(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [skip]); - - const reloadPlaylist = useCallback(async () => { - try { - const data = await api.getPlaylist(); - const newList = data.files || []; - const currentItem = listRef.current[indexRef.current] || null; - - if (newList.length === 0) { - setList([]); - setLoaded(false); - setLoaderText('播放列表已清空，前往管理后台添加'); - const video = videoRef.current; - if (video) video.pause(); - return; - } - - setLoaded(true); - - let newIndex = -1; - if (currentItem) { - newIndex = newList.findIndex(item => item.relative_path === currentItem.relative_path); - } - if (newIndex >= 0) { - setList(newList); - setIndex(newIndex); - } else { - setList(newList); - setIndex(prev => Math.min(prev, newList.length - 1)); - } - } catch {/* ignore */} - }, []); - - const applySettings = useCallback((msg) => { - if (msg.volume !== undefined) { - const vol = msg.volume / 100; - setVolume(vol); - const video = videoRef.current; - if (video) { - video.volume = vol; - if (soundBlockedRef.current) { - video.muted = false; - video.play().then(() => { - setSoundBlocked(false); - const hint = document.getElementById('soundHint'); - if (hint) hint.style.display = 'none'; - }).catch(() => { - video.muted = true; - video.play().catch(() => {}); - }); - } - } - } - if (msg.play_mode !== undefined) setPlayMode(msg.play_mode); - if (msg.image_duration !== undefined) setImageDuration(msg.image_duration); - if (msg.fullscreen !== undefined) { - setFullscreenMode(msg.fullscreen); - getCurrentWindow().setFullscreen(msg.fullscreen).catch(() => {}); - } - if (msg.autostart !== undefined) { - setAutostartEnabled(msg.autostart); - if (msg.autostart) { - enable().catch(() => {}); - } else { - disable().catch(() => {}); - } - } - }, []); - - const enableSound = useCallback(() => { - const video = videoRef.current; - if (!video) return; - video.muted = false; - video.play().then(() => { - setSoundBlocked(false); - const hint = document.getElementById('soundHint'); - if (hint) hint.style.display = 'none'; - }).catch(() => { - video.muted = true; - video.play().catch(() => {}); - }); - }, []); - - // ============ 点击标题弹出登录弹窗 ============ - const handleTitleClick = useCallback(() => { - setLoginUsername(''); - setLoginPassword(''); - setLoginError(''); - setShowLoginModal(true); - // 30 秒无操作自动关闭 - if (loginTimerRef.current) clearTimeout(loginTimerRef.current); - loginTimerRef.current = setTimeout(() => { - setShowLoginModal(false); - }, 30000); - // 下一帧聚焦输入框 - setTimeout(() => { - if (loginInputRef.current) loginInputRef.current.focus(); - }, 100); - }, []); - - const handleCloseLogin = useCallback(() => { - if (loginTimerRef.current) clearTimeout(loginTimerRef.current); - setShowLoginModal(false); - }, []); - - const handleLoginSubmit = useCallback(async () => { - try { - const data = await api.login(loginUsername, loginPassword); - if (data.success) { - if (loginTimerRef.current) clearTimeout(loginTimerRef.current); - localStorage.setItem('token', 'admin_logged'); - navigate('/admin?from=screen'); - } else { - setLoginError('账号或密码错误'); - } - } catch { - setLoginError('登录失败，请重试'); - } - }, [loginUsername, loginPassword, navigate]); - - const handleLoginKeyDown = useCallback((e) => { - if (e.key === 'Enter') handleLoginSubmit(); - }, [handleLoginSubmit]); - - // ============ LIVE 点击切换播放/暂停 ============ - const togglePlayPause = useCallback(() => { - if (pausedRef.current) { - // 恢复播放 - setPaused(false); - } else { - // 暂停 - setPaused(true); - const video = videoRef.current; - if (video) video.pause(); - if (timerRef.current) clearTimeout(timerRef.current); - } - }, []); - - // Global click for sound unblock - useEffect(() => { - const handler = () => { - if (soundBlockedRef.current) enableSound(); - }; - document.addEventListener('click', handler); - return () => document.removeEventListener('click', handler); - }, [enableSound]); - - // 加载显示设置（大屏/小窗口、开机自启动） - useEffect(() => { - const load = async () => { - try { - const data = await api.getSettings(); - setFullscreenMode(data.fullscreen ?? true); - setAutostartEnabled(data.autostart ?? false); - } catch {/* ignore */} - }; - load(); - }, []); - - // Escape 退出 Tauri 窗口全屏 / 关闭登录弹窗 - useEffect(() => { - const handler = (e) => { - if (e.key === 'Escape') { - if (showLoginModal) { - handleCloseLogin(); - } else { - getCurrentWindow().setFullscreen(false).catch(() => {}); - } - } - }; - document.addEventListener('keydown', handler); - return () => document.removeEventListener('keydown', handler); - }, [showLoginModal, handleCloseLogin]); - - // Cleanup timers on unmount - useEffect(() => { - return () => { - if (timerRef.current) clearTimeout(timerRef.current); - if (loginTimerRef.current) clearTimeout(loginTimerRef.current); - }; - }, []); - - const currentItem = list[index] || null; - - return ( -
- {/* Perspective Grid */} -
- {/* Ambient Orbs */} -
-
-
- - {/* Loader */} -
- ); -} diff --git a/src/pages/voice-embed.css b/src/pages/voice-embed.css deleted file mode 100644 index faf0c87..0000000 --- a/src/pages/voice-embed.css +++ /dev/null @@ -1,12 +0,0 @@ -/* 实时语音对话页：iframe 占满内容区（页眉由 ScreenLayout 注入） */ -.voice-embed { - width: 100%; - height: 100%; -} -.voice-embed iframe { - width: 100%; - height: 100%; - border: none; - display: block; - background: #0d1117; /* 与原对话页深色背景一致 */ -}